gibheer
/
bleve
0
0
Fork 0
Code

Merge branch 'master' into firestorm

This commit is contained in:
Marty Schoch 2015-10-28 11:26:01 -04:00
parent 66aa1b020a 1a978a4591
commit 01526e971f
128 changed files with 4631 additions and 4277 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -16,7 +16,7 @@ Try out bleve live by [searching our wiki](http://wikisearch.blevesearch.com/sea
* Term, Phrase, Match, Match Phrase, Prefix
* Conjunction, Disjunction, Boolean
* Numeric Range, Date Range
* Simple query [syntax](https://github.com/blevesearch/bleve/wiki/Query-String-Query) for human entry
* Simple query [syntax](http://www.blevesearch.com/docs/Query-String-Query/) for human entry
* tf-idf Scoring
* Search result match highlighting
* Supports Aggregating Facets:

8
analysis/freq.go
View File

@ -9,6 +9,10 @@
package analysis
// TokenLocation represents one occurrence of a term at a particular location in
// a field. Start, End and Position have the same meaning as in analysis.Token.
// Field and ArrayPositions identify the field value in the source document.
// See document.Field for details.
type TokenLocation struct {
Field string
ArrayPositions []uint64
@ -17,6 +21,8 @@ type TokenLocation struct {
Position int
}
// TokenFreq represents all the occurrences of a term in all fields of a
// document.
type TokenFreq struct {
Term []byte
Locations []*TokenLocation
@ -26,6 +32,8 @@ func (tf *TokenFreq) Frequency() int {
return len(tf.Locations)
}
// TokenFrequencies maps document terms to their combined frequencies from all
// fields.
type TokenFrequencies map[string]*TokenFreq
func (tfs TokenFrequencies) MergeAll(remoteField string, other TokenFrequencies) {

18
analysis/language/ja/ja_morph_kagome.go
View File

@ -13,23 +13,27 @@ import (
"github.com/blevesearch/bleve/analysis"
"github.com/blevesearch/bleve/registry"
"github.com/ikawaha/kagome"
"github.com/ikawaha/kagome/tokenizer"
)
const TokenizerName = "kagome"
type KagomeMorphTokenizer struct {
tok *kagome.Tokenizer
tok tokenizer.Tokenizer
}
func init() {
_ = tokenizer.SysDic() // prepare system dictionary
}
func NewKagomeMorphTokenizer() *KagomeMorphTokenizer {
return &KagomeMorphTokenizer{
tok: kagome.NewTokenizer(),
tok: tokenizer.New(),
}
}
func NewKagomeMorphTokenizerWithUserDic(userdic *kagome.UserDic) *KagomeMorphTokenizer {
k := kagome.NewTokenizer()
func NewKagomeMorphTokenizerWithUserDic(userdic tokenizer.UserDic) *KagomeMorphTokenizer {
k := tokenizer.New()
k.SetUserDic(userdic)
return &KagomeMorphTokenizer{
tok: k,
@ -38,7 +42,7 @@ func NewKagomeMorphTokenizerWithUserDic(userdic *kagome.UserDic) *KagomeMorphTok
func (t *KagomeMorphTokenizer) Tokenize(input []byte) analysis.TokenStream {
var (
morphs []kagome.Token
morphs []tokenizer.Token
prevstart int
)
@ -47,7 +51,7 @@ func (t *KagomeMorphTokenizer) Tokenize(input []byte) analysis.TokenStream {
return rv
}
morphs = t.tok.Tokenize(string(input))
morphs = t.tok.Analyze(string(input), tokenizer.Search)
for i, m := range morphs {
if m.Surface == "EOS" || m.Surface == "BOS" {

15
analysis/type.go
View File

@ -30,10 +30,19 @@ const (
Double
)
// Token represents one occurrence of a term at a particular location in a
// field.
type Token struct {
Start int `json:"start"`
End int `json:"end"`
Term []byte `json:"term"`
// Start specifies the byte offset of the beginning of the term in the
// field.
Start int `json:"start"`
// End specifies the byte offset of the end of the term in the field.
End int `json:"end"`
Term []byte `json:"term"`
// Position specifies the 1-based index of the token in the sequence of
// occurrences of its term in the field.
Position int `json:"position"`
Type TokenType `json:"type"`
KeyWord bool `json:"keyword"`

1
config/config.go
View File

@ -86,7 +86,6 @@ import (
_ "github.com/blevesearch/bleve/index/store/boltdb"
_ "github.com/blevesearch/bleve/index/store/goleveldb"
_ "github.com/blevesearch/bleve/index/store/gtreap"
_ "github.com/blevesearch/bleve/index/store/inmem"
// index types
_ "github.com/blevesearch/bleve/index/firestorm"

5
docs/build_children.sh
View File

@ -4,8 +4,3 @@
BUILD_NUM=$(curl -s 'https://api.travis-ci.org/repos/blevesearch/beer-search/builds' | grep -o '^\[{"id":[0-9]*,' | grep -o '[0-9]' | tr -d '\n')
# Restart last child project build
curl -X POST https://api.travis-ci.org/builds/$BUILD_NUM/restart --header "Authorization: token "$AUTH_TOKEN
# Get last child project build number
BUILD_NUM=$(curl -s 'https://api.travis-ci.org/repos/blevesearch/bleve-wiki-indexer/builds' | grep -o '^\[{"id":[0-9]*,' | grep -o '[0-9]' | tr -d '\n')
# Restart last child project build
curl -X POST https://api.travis-ci.org/builds/$BUILD_NUM/restart --header "Authorization: token "$AUTH_TOKEN

7
document/field.go
View File

@ -14,7 +14,14 @@ import (
)
type Field interface {
// Name returns the path of the field from the root DocumentMapping.
// A root field path is "field", a subdocument field is "parent.field".
Name() string
// ArrayPositions returns the intermediate document and field indices
// required to resolve the field value in the document. For example, if the
// field path is "doc1.doc2.field" where doc1 and doc2 are slices or
// arrays, ArrayPositions returns 2 indices used to resolve "doc2" value in
// "doc1", then "field" in "doc2".
ArrayPositions() []uint64
Options() IndexingOptions
Analyze() (int, analysis.TokenFrequencies)

34
error.go
View File

@ -25,6 +25,7 @@ const (
ErrorAliasMulti
ErrorAliasEmpty
ErrorUnknownIndexType
ErrorEmptyID
)
// Error represents a more strongly typed bleve error for detecting
@ -32,22 +33,23 @@ const (
type Error int
func (e Error) Error() string {
return errorMessages[int(e)]
return errorMessages[e]
}
var errorMessages = map[int]string{
int(ErrorIndexPathExists): "cannot create new index, path already exists",
int(ErrorIndexPathDoesNotExist): "cannot open index, path does not exist",
int(ErrorIndexMetaMissing): "cannot open index, metadata missing",
int(ErrorIndexMetaCorrupt): "cannot open index, metadata corrupt",
int(ErrorDisjunctionFewerThanMinClauses): "disjunction query has fewer than the minimum number of clauses to satisfy",
int(ErrorBooleanQueryNeedsMustOrShouldOrNotMust): "boolean query must contain at least one must or should or not must clause",
int(ErrorNumericQueryNoBounds): "numeric range query must specify min or max",
int(ErrorPhraseQueryNoTerms): "phrase query must contain at least one term",
int(ErrorUnknownQueryType): "unknown query type",
int(ErrorUnknownStorageType): "unknown storage type",
int(ErrorIndexClosed): "index is closed",
int(ErrorAliasMulti): "cannot perform single index operation on multiple index alias",
int(ErrorAliasEmpty): "cannot perform operation on empty alias",
int(ErrorUnknownIndexType): "unknown index type",
var errorMessages = map[Error]string{
ErrorIndexPathExists: "cannot create new index, path already exists",
ErrorIndexPathDoesNotExist: "cannot open index, path does not exist",
ErrorIndexMetaMissing: "cannot open index, metadata missing",
ErrorIndexMetaCorrupt: "cannot open index, metadata corrupt",
ErrorDisjunctionFewerThanMinClauses: "disjunction query has fewer than the minimum number of clauses to satisfy",
ErrorBooleanQueryNeedsMustOrShouldOrNotMust: "boolean query must contain at least one must or should or not must clause",
ErrorNumericQueryNoBounds: "numeric range query must specify min or max",
ErrorPhraseQueryNoTerms: "phrase query must contain at least one term",
ErrorUnknownQueryType: "unknown query type",
ErrorUnknownStorageType: "unknown storage type",
ErrorIndexClosed: "index is closed",
ErrorAliasMulti: "cannot perform single index operation on multiple index alias",
ErrorAliasEmpty: "cannot perform operation on empty alias",
ErrorUnknownIndexType: "unknown index type",
ErrorEmptyID: "document ID cannot be empty",
}

4
examples_test.go
View File

@ -253,7 +253,7 @@ func ExampleNewFacetRequest() {
fmt.Println(searchResults.Facets["facet name"].Total)
// numer of docs with no value for this field
fmt.Println(searchResults.Facets["facet name"].Missing)
// term with highest occurences in field name
// term with highest occurrences in field name
fmt.Println(searchResults.Facets["facet name"].Terms[0].Term)
// Output:
// 5
@ -339,7 +339,7 @@ func ExampleSearchRequest_AddFacet() {
fmt.Println(searchResults.Facets["facet name"].Total)
// numer of docs with no value for this field
fmt.Println(searchResults.Facets["facet name"].Missing)
// term with highest occurences in field name
// term with highest occurrences in field name
fmt.Println(searchResults.Facets["facet name"].Terms[0].Term)
// Output:
// 5

79
index.go
View File

@ -30,6 +30,9 @@ type Batch struct {
// batch. NOTE: the bleve Index is not updated
// until the batch is executed.
func (b *Batch) Index(id string, data interface{}) error {
if id == "" {
return ErrorEmptyID
}
doc := document.NewDocument(id)
err := b.index.Mapping().mapDocument(doc, data)
if err != nil {
@ -43,7 +46,9 @@ func (b *Batch) Index(id string, data interface{}) error {
// batch. NOTE: the bleve Index is not updated until
// the batch is executed.
func (b *Batch) Delete(id string) {
b.internal.Delete(id)
if id != "" {
b.internal.Delete(id)
}
}
// SetInternal adds the specified set internal
@ -81,14 +86,84 @@ func (b *Batch) Reset() {
// An Index implements all the indexing and searching
// capabilities of bleve. An Index can be created
// using the New() and Open() methods.
//
// Index() takes an input value, deduces a DocumentMapping for its type,
// assigns string paths to its fields or values then applies field mappings on
// them.
//
// If the value is a []byte, the indexer attempts to convert it to something
// else using the ByteArrayConverter registered as
// IndexMapping.ByteArrayConverter. By default, it interprets the value as a
// JSON payload and unmarshals it to map[string]interface{}.
//
// The DocumentMapping used to index a value is deduced by the following rules:
// 1) If value implements Classifier interface, resolve the mapping from Type().
// 2) If value has a string field or value at IndexMapping.TypeField.
// (defaulting to "_type"), use it to resolve the mapping. Fields addressing
// is described below.
// 3) If IndexMapping.DefaultType is registered, return it.
// 4) Return IndexMapping.DefaultMapping.
//
// Each field or nested field of the value is identified by a string path, then
// mapped to one or several FieldMappings which extract the result for analysis.
//
// Struct values fields are identified by their "json:" tag, or by their name.
// Nested fields are identified by prefixing with their parent identifier,
// separated by a dot.
//
// Map values entries are identified by their string key. Entries not indexed
// by strings are ignored. Entry values are identified recursively like struct
// fields.
//
// Slice and array values are identified by their field name. Their elements
// are processed sequentially with the same FieldMapping.
//
// String, float64 and time.Time values are identified by their field name.
// Other types are ignored.
//
// Each value identifier is decomposed in its parts and recursively address
// SubDocumentMappings in the tree starting at the root DocumentMapping. If a
// mapping is found, all its FieldMappings are applied to the value. If no
// mapping is found and the root DocumentMapping is dynamic, default mappings
// are used based on value type and IndexMapping default configurations.
//
// Finally, mapped values are analyzed, indexed or stored. See
// FieldMapping.Analyzer to know how an analyzer is resolved for a given field.
//
// Examples:
//
// type Date struct {
// Day string `json:"day"`
// Month string
// Year string
// }
//
// type Person struct {
// FirstName string `json:"first_name"`
// LastName string
// BirthDate Date `json:"birth_date"`
// }
//
// A Person value FirstName is mapped by the SubDocumentMapping at
// "first_name". Its LastName is mapped by the one at "LastName". The day of
// BirthDate is mapped to the SubDocumentMapping "day" of the root
// SubDocumentMapping "birth_date". It will appear as the "birth_date.day"
// field in the index. The month is mapped to "birth_date.Month".
type Index interface {
// Index analyzes, indexes or stores mapped data fields. Supplied
// identifier is bound to analyzed data and will be retrieved by search
// requests. See Index interface documentation for details about mapping
// rules.
Index(id string, data interface{}) error
Delete(id string) error
NewBatch() *Batch
Batch(b *Batch) error
// Document returns specified document or nil if the document is not
// indexed or stored.
Document(id string) (*document.Document, error)
// DocCount returns the number of documents in the index.
DocCount() (uint64, error)
Search(req *SearchRequest) (*SearchResult, error)
@ -113,6 +188,8 @@ type Index interface {
SetInternal(key, val []byte) error
DeleteInternal(key []byte) error
// Advanced returns the indexer and data store, exposing lower level
// methods to enumerate records and access data.
Advanced() (index.Index, store.KVStore, error)
}

5
index/analysis.go
View File

@ -12,7 +12,12 @@ package index
import "github.com/blevesearch/bleve/document"
type IndexRow interface {
KeySize() int
KeyTo([]byte) (int, error)
Key() []byte
ValueSize() int
ValueTo([]byte) (int, error)
Value() []byte
}

36
index/firestorm/analysis_test.go
View File

@ -16,38 +16,53 @@ import (
"github.com/blevesearch/bleve/analysis/analyzers/standard_analyzer"
"github.com/blevesearch/bleve/document"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store/inmem"
"github.com/blevesearch/bleve/index/store/gtreap"
"github.com/blevesearch/bleve/index/store/null"
"github.com/blevesearch/bleve/registry"
)
func TestAnalysis(t *testing.T) {
kv, err := inmem.New()
aq := index.NewAnalysisQueue(1)
f, err := NewFirestorm(gtreap.Name, nil, aq)
if err != nil {
t.Fatal(err)
}
err = f.Open()
if err != nil {
t.Fatal(err)
}
aq := index.NewAnalysisQueue(1)
f := NewFirestorm(kv, aq)
rows := []index.IndexRow{
NewFieldRow(0, IDFieldName),
}
kvwriter, err := f.store.Writer()
kvwriter, err := f.(*Firestorm).store.Writer()
if err != nil {
t.Fatal(err)
}
for _, row := range rows {
err := kvwriter.Set(row.Key(), row.Value())
wb := kvwriter.NewBatch()
wb.Set(row.Key(), row.Value())
err := kvwriter.ExecuteBatch(wb)
if err != nil {
t.Fatal(err)
}
}
err = kvwriter.Close()
if err != nil {
t.Fatal(err)
}
kvreader, err := f.(*Firestorm).store.Reader()
if err != nil {
t.Fatal(err)
}
// warmup to load field cache and set maxRead correctly
f.warmup(kvwriter)
f.(*Firestorm).warmup(kvreader)
tests := []struct {
d *document.Document
@ -77,7 +92,7 @@ func TestAnalysis(t *testing.T) {
}
}
err = kvwriter.Close()
err = kvreader.Close()
if err != nil {
t.Fatal(err)
}
@ -91,12 +106,11 @@ func BenchmarkAnalyze(b *testing.B) {
b.Fatal(err)
}
s, err := null.New()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewFirestorm(null.Name, nil, analysisQueue)
if err != nil {
b.Fatal(err)
}
analysisQueue := index.NewAnalysisQueue(1)
idx := NewFirestorm(s, analysisQueue)
d := document.NewDocument("1")
f := document.NewTextFieldWithAnalyzer("desc", nil, bleveWikiArticle1K, analyzer)

35
index/firestorm/benchmark_boltdb_test.go
View File

@ -10,68 +10,61 @@
package firestorm
import (
"os"
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/index/store/boltdb"
)
func CreateBoltDB() (store.KVStore, error) {
s := boltdb.New("test", "bleve")
return s, nil
}
func DestroyBoltDB() error {
return os.RemoveAll("test")
var boltTestConfig = map[string]interface{}{
"path": "test",
}
func BenchmarkBoltDBIndexing1Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateBoltDB, DestroyBoltDB, 1)
CommonBenchmarkIndex(b, boltdb.Name, boltTestConfig, DestroyTest, 1)
}
func BenchmarkBoltDBIndexing2Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateBoltDB, DestroyBoltDB, 2)
CommonBenchmarkIndex(b, boltdb.Name, boltTestConfig, DestroyTest, 2)
}
func BenchmarkBoltDBIndexing4Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateBoltDB, DestroyBoltDB, 4)
CommonBenchmarkIndex(b, boltdb.Name, boltTestConfig, DestroyTest, 4)
}
// batches
func BenchmarkBoltDBIndexing1Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 1, 10)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 1, 10)
}
func BenchmarkBoltDBIndexing2Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 2, 10)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 2, 10)
}
func BenchmarkBoltDBIndexing4Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 4, 10)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 4, 10)
}
func BenchmarkBoltDBIndexing1Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 1, 100)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 1, 100)
}
func BenchmarkBoltDBIndexing2Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 2, 100)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 2, 100)
}
func BenchmarkBoltDBIndexing4Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 4, 100)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 4, 100)
}
func BenchmarkBoltBIndexing1Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 1, 1000)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 1, 1000)
}
func BenchmarkBoltBIndexing2Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 2, 1000)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 2, 1000)
}
func BenchmarkBoltBIndexing4Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 4, 1000)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 4, 1000)
}

21
index/firestorm/benchmark_common_test.go
View File

@ -10,13 +10,13 @@
package firestorm
import (
"os"
"strconv"
"testing"
_ "github.com/blevesearch/bleve/analysis/analyzers/standard_analyzer"
"github.com/blevesearch/bleve/document"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
)
@ -33,10 +33,13 @@ var benchmarkDocBodies = []string{
"The expansion ratio of a liquefied and cryogenic substance is the volume of a given amount of that substance in liquid form compared to the volume of the same amount of substance in gaseous form, at room temperature and normal atmospheric pressure.",
}
type KVStoreCreate func() (store.KVStore, error)
type KVStoreDestroy func() error
func CommonBenchmarkIndex(b *testing.B, create KVStoreCreate, destroy KVStoreDestroy, analysisWorkers int) {
func DestroyTest() error {
return os.RemoveAll("test")
}
func CommonBenchmarkIndex(b *testing.B, storeName string, storeConfig map[string]interface{}, destroy KVStoreDestroy, analysisWorkers int) {
cache := registry.NewCache()
analyzer, err := cache.AnalyzerNamed("standard")
@ -50,12 +53,11 @@ func CommonBenchmarkIndex(b *testing.B, create KVStoreCreate, destroy KVStoreDes
b.ResetTimer()
b.StopTimer()
for i := 0; i < b.N; i++ {
s, err := create()
analysisQueue := index.NewAnalysisQueue(analysisWorkers)
idx, err := NewFirestorm(storeName, storeConfig, analysisQueue)
if err != nil {
b.Fatal(err)
}
analysisQueue := index.NewAnalysisQueue(analysisWorkers)
idx := NewFirestorm(s, analysisQueue)
err = idx.Open()
if err != nil {
@ -81,7 +83,7 @@ func CommonBenchmarkIndex(b *testing.B, create KVStoreCreate, destroy KVStoreDes
}
}
func CommonBenchmarkIndexBatch(b *testing.B, create KVStoreCreate, destroy KVStoreDestroy, analysisWorkers, batchSize int) {
func CommonBenchmarkIndexBatch(b *testing.B, storeName string, storeConfig map[string]interface{}, destroy KVStoreDestroy, analysisWorkers, batchSize int) {
cache := registry.NewCache()
analyzer, err := cache.AnalyzerNamed("standard")
@ -93,12 +95,11 @@ func CommonBenchmarkIndexBatch(b *testing.B, create KVStoreCreate, destroy KVSto
b.StopTimer()
for i := 0; i < b.N; i++ {
s, err := create()
analysisQueue := index.NewAnalysisQueue(analysisWorkers)
idx, err := NewFirestorm(storeName, storeConfig, analysisQueue)
if err != nil {
b.Fatal(err)
}
analysisQueue := index.NewAnalysisQueue(analysisWorkers)
idx := NewFirestorm(s, analysisQueue)
err = idx.Open()
if err != nil {

31
index/firestorm/benchmark_cznicb_test.go
View File

@ -12,64 +12,59 @@ package firestorm
import (
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/index/store/cznicb"
"github.com/blevesearch/blevex/cznicb"
)
func CreateCznicB() (store.KVStore, error) {
return cznicb.StoreConstructor(nil)
}
func DestroyCznicB() error {
return nil
}
func BenchmarkCznicBIndexing1Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateCznicB, DestroyCznicB, 1)
CommonBenchmarkIndex(b, cznicb.Name, nil, DestroyCznicB, 1)
}
func BenchmarkCznicBIndexing2Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateCznicB, DestroyCznicB, 2)
CommonBenchmarkIndex(b, cznicb.Name, nil, DestroyCznicB, 2)
}
func BenchmarkCznicBIndexing4Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateCznicB, DestroyCznicB, 4)
CommonBenchmarkIndex(b, cznicb.Name, nil, DestroyCznicB, 4)
}
// batches
func BenchmarkCznicBIndexing1Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 1, 10)
CommonBenchmarkIndexBatch(b, cznicb.Name, nil, DestroyCznicB, 1, 10)
}
func BenchmarkCznicBIndexing2Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 2, 10)
CommonBenchmarkIndexBatch(b, cznicb.Name, nil, DestroyCznicB, 2, 10)
}
func BenchmarkCznicBIndexing4Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 4, 10)
CommonBenchmarkIndexBatch(b, cznicb.Name, nil, DestroyCznicB, 4, 10)
}
func BenchmarkCznicBIndexing1Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 1, 100)
CommonBenchmarkIndexBatch(b, cznicb.Name, nil, DestroyCznicB, 1, 100)
}
func BenchmarkCznicBIndexing2Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 2, 100)
CommonBenchmarkIndexBatch(b, cznicb.Name, nil, DestroyCznicB, 2, 100)
}
func BenchmarkCznicBIndexing4Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 4, 100)
CommonBenchmarkIndexBatch(b, cznicb.Name, nil, DestroyCznicB, 4, 100)
}
func BenchmarkCznicBIndexing1Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 1, 1000)
CommonBenchmarkIndexBatch(b, cznicb.Name, nil, DestroyCznicB, 1, 1000)
}
func BenchmarkCznicBIndexing2Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 2, 1000)
CommonBenchmarkIndexBatch(b, cznicb.Name, nil, DestroyCznicB, 2, 1000)
}
func BenchmarkCznicBIndexing4Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 4, 1000)
CommonBenchmarkIndexBatch(b, cznicb.Name, nil, DestroyCznicB, 4, 1000)
}

35
index/firestorm/benchmark_goleveldb_test.go
View File

@ -10,71 +10,62 @@
package firestorm
import (
"os"
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/index/store/goleveldb"
)
var goLevelDBTestOptions = map[string]interface{}{
"create_if_missing": true,
}
func CreateGoLevelDB() (store.KVStore, error) {
return goleveldb.New("test", goLevelDBTestOptions)
}
func DestroyGoLevelDB() error {
return os.RemoveAll("test")
"path": "test",
}
func BenchmarkGoLevelDBIndexing1Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateGoLevelDB, DestroyGoLevelDB, 1)
CommonBenchmarkIndex(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 1)
}
func BenchmarkGoLevelDBIndexing2Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateGoLevelDB, DestroyGoLevelDB, 2)
CommonBenchmarkIndex(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 2)
}
func BenchmarkGoLevelDBIndexing4Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateGoLevelDB, DestroyGoLevelDB, 4)
CommonBenchmarkIndex(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 4)
}
// batches
func BenchmarkGoLevelDBIndexing1Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 1, 10)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 1, 10)
}
func BenchmarkGoLevelDBIndexing2Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 2, 10)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 2, 10)
}
func BenchmarkGoLevelDBIndexing4Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 4, 10)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 4, 10)
}
func BenchmarkGoLevelDBIndexing1Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 1, 100)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 1, 100)
}
func BenchmarkGoLevelDBIndexing2Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 2, 100)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 2, 100)
}
func BenchmarkGoLevelDBIndexing4Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 4, 100)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 4, 100)
}
func BenchmarkGoLevelDBIndexing1Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 1, 1000)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 1, 1000)
}
func BenchmarkGoLevelDBIndexing2Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 2, 1000)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 2, 1000)
}
func BenchmarkGoLevelDBIndexing4Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 4, 1000)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 4, 1000)
}

29
index/firestorm/benchmark_gtreap_test.go
View File

@ -12,64 +12,59 @@ package firestorm
import (
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/index/store/gtreap"
)
func CreateGTreap() (store.KVStore, error) {
return gtreap.StoreConstructor(nil)
}
func DestroyGTreap() error {
return nil
}
func BenchmarkGTreapIndexing1Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateGTreap, DestroyGTreap, 1)
CommonBenchmarkIndex(b, gtreap.Name, nil, DestroyGTreap, 1)
}
func BenchmarkGTreapIndexing2Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateGTreap, DestroyGTreap, 2)
CommonBenchmarkIndex(b, gtreap.Name, nil, DestroyGTreap, 2)
}
func BenchmarkGTreapIndexing4Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateGTreap, DestroyGTreap, 4)
CommonBenchmarkIndex(b, gtreap.Name, nil, DestroyGTreap, 4)
}
// batches
func BenchmarkGTreapIndexing1Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 1, 10)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyGTreap, 1, 10)
}
func BenchmarkGTreapIndexing2Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 2, 10)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyGTreap, 2, 10)
}
func BenchmarkGTreapIndexing4Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 4, 10)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyGTreap, 4, 10)
}
func BenchmarkGTreapIndexing1Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 1, 100)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyGTreap, 1, 100)
}
func BenchmarkGTreapIndexing2Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 2, 100)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyGTreap, 2, 100)
}
func BenchmarkGTreapIndexing4Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 4, 100)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyGTreap, 4, 100)
}
func BenchmarkGTreapIndexing1Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 1, 1000)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyGTreap, 1, 1000)
}
func BenchmarkGTreapIndexing2Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 2, 1000)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyGTreap, 2, 1000)
}
func BenchmarkGTreapIndexing4Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 4, 1000)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyGTreap, 4, 1000)
}

75
index/firestorm/benchmark_inmem_test.go
View File

@ -1,75 +0,0 @@
// Copyright (c) 2015 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package firestorm
import (
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/index/store/inmem"
)
func CreateInMem() (store.KVStore, error) {
return inmem.New()
}
func DestroyInMem() error {
return nil
}
func BenchmarkInMemIndexing1Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateInMem, DestroyInMem, 1)
}
func BenchmarkInMemIndexing2Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateInMem, DestroyInMem, 2)
}
func BenchmarkInMemIndexing4Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateInMem, DestroyInMem, 4)
}
// batches
func BenchmarkInMemIndexing1Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 1, 10)
}
func BenchmarkInMemIndexing2Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 2, 10)
}
func BenchmarkInMemIndexing4Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 4, 10)
}
func BenchmarkInMemIndexing1Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 1, 100)
}
func BenchmarkInMemIndexing2Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 2, 100)
}
func BenchmarkInMemIndexing4Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 4, 100)
}
func BenchmarkInMemIndexing1Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 1, 1000)
}
func BenchmarkInMemIndexing2Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 2, 1000)
}
func BenchmarkInMemIndexing4Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 4, 1000)
}

29
index/firestorm/benchmark_null_test.go
View File

@ -12,64 +12,59 @@ package firestorm
import (
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/index/store/null"
)
func CreateNull() (store.KVStore, error) {
return null.New()
}
func DestroyNull() error {
return nil
}
func BenchmarkNullIndexing1Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateNull, DestroyNull, 1)
CommonBenchmarkIndex(b, null.Name, nil, DestroyNull, 1)
}
func BenchmarkNullIndexing2Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateNull, DestroyNull, 2)
CommonBenchmarkIndex(b, null.Name, nil, DestroyNull, 2)
}
func BenchmarkNullIndexing4Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateNull, DestroyNull, 4)
CommonBenchmarkIndex(b, null.Name, nil, DestroyNull, 4)
}
// batches
func BenchmarkNullIndexing1Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 1, 10)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyNull, 1, 10)
}
func BenchmarkNullIndexing2Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 2, 10)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyNull, 2, 10)
}
func BenchmarkNullIndexing4Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 4, 10)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyNull, 4, 10)
}
func BenchmarkNullIndexing1Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 1, 100)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyNull, 1, 100)
}
func BenchmarkNullIndexing2Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 2, 100)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyNull, 2, 100)
}
func BenchmarkNullIndexing4Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 4, 100)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyNull, 4, 100)
}
func BenchmarkNullIndexing1Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 1, 1000)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyNull, 1, 1000)
}
func BenchmarkNullIndexing2Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 2, 1000)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyNull, 2, 1000)
}
func BenchmarkNullIndexing4Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 4, 1000)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyNull, 4, 1000)
}

2
index/firestorm/dict_updater.go
View File

@ -108,7 +108,7 @@ func (d *DictUpdater) update() {
wb.Merge([]byte(term), dictionaryTermDelta)
}
err = wb.Execute()
err = writer.ExecuteBatch(wb)
if err != nil {
writer.Close()
logger.Printf("dict updater fatal: %v", err)

30
index/firestorm/dict_updater_test.go
View File

@ -13,21 +13,17 @@ import (
"testing"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store/inmem"
"github.com/blevesearch/bleve/index/store/gtreap"
)
func TestDictUpdater(t *testing.T) {
kv, err := inmem.New()
aq := index.NewAnalysisQueue(1)
f, err := NewFirestorm(gtreap.Name, nil, aq)
if err != nil {
t.Fatal(err)
}
kv.SetMergeOperator(&mergeOperator)
aq := index.NewAnalysisQueue(1)
f := NewFirestorm(kv, aq)
err = kv.Open()
err = f.Open()
if err != nil {
t.Fatal(err)
}
@ -39,13 +35,13 @@ func TestDictUpdater(t *testing.T) {
string([]byte{'d', 1, 0, 'c', 'a', 't'}): 3,
}
f.dictUpdater.NotifyBatch(dictBatch)
f.(*Firestorm).dictUpdater.NotifyBatch(dictBatch)
// invoke updater manually
f.dictUpdater.update()
f.(*Firestorm).dictUpdater.update()
// assert that dictionary rows are correct
reader, err := f.store.Reader()
reader, err := f.(*Firestorm).store.Reader()
if err != nil {
t.Fatal(err)
}
@ -78,13 +74,13 @@ func TestDictUpdater(t *testing.T) {
string([]byte{'d', 1, 0, 'c', 'a', 't'}): 4,
}
f.dictUpdater.NotifyBatch(dictBatch)
f.(*Firestorm).dictUpdater.NotifyBatch(dictBatch)
// invoke updater manually
f.dictUpdater.update()
f.(*Firestorm).dictUpdater.update()
// assert that dictionary rows are correct
reader, err = f.store.Reader()
reader, err = f.(*Firestorm).store.Reader()
if err != nil {
t.Fatal(err)
}
@ -117,13 +113,13 @@ func TestDictUpdater(t *testing.T) {
string([]byte{'d', 1, 0, 'c', 'a', 't'}): 2,
}
f.dictUpdater.NotifyBatch(dictBatch)
f.(*Firestorm).dictUpdater.NotifyBatch(dictBatch)
// invoke updater manually
f.dictUpdater.update()
f.(*Firestorm).dictUpdater.update()
// assert that dictionary rows are correct
reader, err = f.store.Reader()
reader, err = f.(*Firestorm).store.Reader()
if err != nil {
t.Fatal(err)
}

28
index/firestorm/dictionary.go
View File

@ -88,17 +88,35 @@ func (dr *DictionaryRow) SetCount(count uint64) {
dr.value.Count = proto.Uint64(count)
}
func (dr *DictionaryRow) Key() []byte {
buf := make([]byte, 3+len(dr.term))
func (dr *DictionaryRow) KeySize() int {
return 3 + len(dr.term)
}
func (dr *DictionaryRow) KeyTo(buf []byte) (int, error) {
copy(buf[0:], DictionaryKeyPrefix)
binary.LittleEndian.PutUint16(buf[1:3], dr.field)
copy(buf[3:], dr.term)
return buf
return 3 + len(dr.term), nil
}
func (dr *DictionaryRow) Key() []byte {
buf := make([]byte, dr.KeySize())
n, _ := dr.KeyTo(buf)
return buf[:n]
}
func (dr *DictionaryRow) ValueSize() int {
return dr.value.Size()
}
func (dr *DictionaryRow) ValueTo(buf []byte) (int, error) {
return dr.value.MarshalTo(buf)
}
func (dr *DictionaryRow) Value() []byte {
rv, _ := dr.value.Marshal()
return rv
buf := make([]byte, dr.ValueSize())
n, _ := dr.ValueTo(buf)
return buf[:n]
}
func DictionaryRowKey(field uint16, term []byte) []byte {

23
index/firestorm/dump_test.go
View File

@ -10,32 +10,25 @@
package firestorm
import (
"os"
"testing"
"time"
"github.com/blevesearch/bleve/document"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store/boltdb"
"github.com/blevesearch/bleve/index/store/gtreap"
)
var dictWaitDuration = 5 * time.Second
func TestDump(t *testing.T) {
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
s := boltdb.New("test", "bleve")
s.SetMergeOperator(&mergeOperator)
analysisQueue := index.NewAnalysisQueue(1)
idx := NewFirestorm(s, analysisQueue)
err := idx.Open()
idx, err := NewFirestorm(gtreap.Name, nil, analysisQueue)
if err != nil {
t.Errorf("error opening index: %v", err)
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Fatalf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
@ -112,7 +105,7 @@ func TestDump(t *testing.T) {
t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount)
}
idx.dictUpdater.waitTasksDone(dictWaitDuration)
idx.(*Firestorm).dictUpdater.waitTasksDone(dictWaitDuration)
// 1 version
// fieldsCount field rows

28
index/firestorm/field.go
View File

@ -76,16 +76,34 @@ func NewFieldRowKV(key, value []byte) (*FieldRow, error) {
return &rv, nil
}
func (fr *FieldRow) Key() []byte {
buf := make([]byte, 3)
func (fr *FieldRow) KeySize() int {
return 3
}
func (fr *FieldRow) KeyTo(buf []byte) (int, error) {
buf[0] = 'f'
binary.LittleEndian.PutUint16(buf[1:3], fr.index)
return buf
return 3, nil
}
func (fr *FieldRow) Key() []byte {
buf := make([]byte, fr.KeySize())
n, _ := fr.KeyTo(buf)
return buf[:n]
}
func (fr *FieldRow) ValueSize() int {
return fr.value.Size()
}
func (fr *FieldRow) ValueTo(buf []byte) (int, error) {
return fr.value.MarshalTo(buf)
}
func (fr *FieldRow) Value() []byte {
rv, _ := fr.value.Marshal()
return rv
buf := make([]byte, fr.ValueSize())
n, _ := fr.ValueTo(buf)
return buf[:n]
}
func (fr *FieldRow) Index() uint16 {

69
index/firestorm/firestorm.go
View File

@ -27,6 +27,8 @@ const Name = "firestorm"
var UnsafeBatchUseDetected = fmt.Errorf("bleve.Batch is NOT thread-safe, modification after execution detected")
type Firestorm struct {
storeName string
storeConfig map[string]interface{}
store store.KVStore
compensator *Compensator
analysisQueue *index.AnalysisQueue
@ -39,10 +41,11 @@ type Firestorm struct {
stats *indexStat
}
func NewFirestorm(s store.KVStore, analysisQueue *index.AnalysisQueue) *Firestorm {
func NewFirestorm(storeName string, storeConfig map[string]interface{}, analysisQueue *index.AnalysisQueue) (index.Index, error) {
initialCount := uint64(0)
rv := Firestorm{
store: s,
storeName: storeName,
storeConfig: storeConfig,
compensator: NewCompensator(),
analysisQueue: analysisQueue,
fieldCache: index.NewFieldCache(),
@ -53,45 +56,51 @@ func NewFirestorm(s store.KVStore, analysisQueue *index.AnalysisQueue) *Firestor
rv.garbageCollector = NewGarbageCollector(&rv)
rv.lookuper = NewLookuper(&rv)
rv.dictUpdater = NewDictUpdater(&rv)
return &rv
return &rv, nil
}
func (f *Firestorm) Open() (err error) {
// install the merge operator
f.store.SetMergeOperator(&mergeOperator)
// open the kv store
storeConstructor := registry.KVStoreConstructorByName(f.storeName)
if storeConstructor == nil {
err = index.ErrorUnknownStorageType
return
}
// now open the kv store
err = f.store.Open()
// now open the store
f.store, err = storeConstructor(&mergeOperator, f.storeConfig)
if err != nil {
return
}
// start a writer for the open process
var kvwriter store.KVWriter
kvwriter, err = f.store.Writer()
// start a reader
var kvreader store.KVReader
kvreader, err = f.store.Reader()
if err != nil {
return
}
defer func() {
if cerr := kvwriter.Close(); err == nil && cerr != nil {
err = cerr
}
}()
// assert correct version, and find out if this is new index
var newIndex bool
newIndex, err = f.checkVersion(kvwriter)
newIndex, err = f.checkVersion(kvreader)
if err != nil {
return
}
if !newIndex {
// process existing index before opening
f.warmup(kvreader)
}
err = kvreader.Close()
if err != nil {
return
}
if newIndex {
// prepare a new index
f.bootstrap(kvwriter)
} else {
// process existing index before opening
f.warmup(kvwriter)
f.bootstrap()
}
// start the garbage collector
@ -197,7 +206,7 @@ func (f *Firestorm) batchRows(writer store.KVWriter, rows []index.IndexRow, dele
}
// write out the batch
err := wb.Execute()
err := writer.ExecuteBatch(wb)
if err != nil {
return nil, err
}
@ -331,7 +340,11 @@ func (f *Firestorm) SetInternal(key, val []byte) (err error) {
err = cerr
}
}()
return writer.Set(internalRow.Key(), internalRow.Value())
wb := writer.NewBatch()
wb.Set(internalRow.Key(), internalRow.Value())
return writer.ExecuteBatch(wb)
}
func (f *Firestorm) DeleteInternal(key []byte) (err error) {
@ -346,7 +359,11 @@ func (f *Firestorm) DeleteInternal(key []byte) (err error) {
err = cerr
}
}()
return writer.Delete(internalRow.Key())
wb := writer.NewBatch()
wb.Delete(internalRow.Key())
return writer.ExecuteBatch(wb)
}
func (f *Firestorm) DumpAll() chan interface{} {
@ -427,10 +444,6 @@ func (f *Firestorm) Stats() json.Marshaler {
}
func IndexTypeConstructor(store store.KVStore, analysisQueue *index.AnalysisQueue) (index.Index, error) {
return NewFirestorm(store, analysisQueue), nil
}
func init() {
registry.RegisterIndexType(Name, IndexTypeConstructor)
registry.RegisterIndexType(Name, NewFirestorm)
}

174
index/firestorm/firestorm_test.go
View File

@ -20,7 +20,6 @@ import (
"github.com/blevesearch/bleve/document"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store/boltdb"
"github.com/blevesearch/bleve/index/store/null"
"github.com/blevesearch/bleve/registry"
)
@ -34,11 +33,12 @@ func TestIndexOpenReopen(t *testing.T) {
}
}()
store := boltdb.New("test", "bleve")
store.SetMergeOperator(&mergeOperator)
analysisQueue := index.NewAnalysisQueue(1)
idx := NewFirestorm(store, analysisQueue)
err := idx.Open()
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
@ -58,9 +58,10 @@ func TestIndexOpenReopen(t *testing.T) {
t.Fatal(err)
}
store = boltdb.New("test", "bleve")
store.SetMergeOperator(&mergeOperator)
idx = NewFirestorm(store, analysisQueue)
idx, err = NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
@ -81,11 +82,12 @@ func TestIndexInsert(t *testing.T) {
}
}()
store := boltdb.New("test", "bleve")
store.SetMergeOperator(&mergeOperator)
analysisQueue := index.NewAnalysisQueue(1)
idx := NewFirestorm(store, analysisQueue)
err := idx.Open()
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
@ -113,7 +115,7 @@ func TestIndexInsert(t *testing.T) {
}
expectedCount++
idx.lookuper.waitTasksDone(lookupWaitDuration)
idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration)
docCount, err = idx.DocCount()
if err != nil {
@ -133,11 +135,12 @@ func TestIndexInsertThenDelete(t *testing.T) {
}
}()
store := boltdb.New("test", "bleve")
store.SetMergeOperator(&mergeOperator)
analysisQueue := index.NewAnalysisQueue(1)
idx := NewFirestorm(store, analysisQueue)
err := idx.Open()
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
@ -165,7 +168,7 @@ func TestIndexInsertThenDelete(t *testing.T) {
}
expectedCount++
idx.lookuper.waitTasksDone(lookupWaitDuration)
idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration)
doc2 := document.NewDocument("2")
doc2.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
@ -175,7 +178,7 @@ func TestIndexInsertThenDelete(t *testing.T) {
}
expectedCount++
idx.lookuper.waitTasksDone(lookupWaitDuration)
idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration)
docCount, err = idx.DocCount()
if err != nil {
@ -191,7 +194,7 @@ func TestIndexInsertThenDelete(t *testing.T) {
}
expectedCount--
idx.lookuper.waitTasksDone(lookupWaitDuration)
idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration)
docCount, err = idx.DocCount()
if err != nil {
@ -207,7 +210,7 @@ func TestIndexInsertThenDelete(t *testing.T) {
}
expectedCount--
idx.lookuper.waitTasksDone(lookupWaitDuration)
idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration)
docCount, err = idx.DocCount()
if err != nil {
@ -226,11 +229,12 @@ func TestIndexInsertThenUpdate(t *testing.T) {
}
}()
store := boltdb.New("test", "bleve")
store.SetMergeOperator(&mergeOperator)
analysisQueue := index.NewAnalysisQueue(1)
idx := NewFirestorm(store, analysisQueue)
err := idx.Open()
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
@ -266,7 +270,7 @@ func TestIndexInsertThenUpdate(t *testing.T) {
t.Errorf("Error deleting entry from index: %v", err)
}
idx.lookuper.waitTasksDone(lookupWaitDuration)
idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration)
docCount, err := idx.DocCount()
if err != nil {
@ -286,11 +290,12 @@ func TestIndexInsertMultiple(t *testing.T) {
}
}()
store := boltdb.New("test", "bleve")
store.SetMergeOperator(&mergeOperator)
analysisQueue := index.NewAnalysisQueue(1)
idx := NewFirestorm(store, analysisQueue)
err := idx.Open()
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
@ -319,9 +324,10 @@ func TestIndexInsertMultiple(t *testing.T) {
t.Fatal(err)
}
store = boltdb.New("test", "bleve")
store.SetMergeOperator(&mergeOperator)
idx = NewFirestorm(store, analysisQueue)
idx, err = NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
@ -341,7 +347,7 @@ func TestIndexInsertMultiple(t *testing.T) {
}
expectedCount++
idx.lookuper.waitTasksDone(lookupWaitDuration)
idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration)
docCount, err := idx.DocCount()
if err != nil {
@ -360,11 +366,12 @@ func TestIndexInsertWithStore(t *testing.T) {
}
}()
store := boltdb.New("test", "bleve")
store.SetMergeOperator(&mergeOperator)
analysisQueue := index.NewAnalysisQueue(1)
idx := NewFirestorm(store, analysisQueue)
err := idx.Open()
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
@ -392,7 +399,7 @@ func TestIndexInsertWithStore(t *testing.T) {
}
expectedCount++
idx.lookuper.waitTasksDone(lookupWaitDuration)
idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration)
docCount, err = idx.DocCount()
if err != nil {
@ -438,11 +445,12 @@ func TestIndexInternalCRUD(t *testing.T) {
}
}()
store := boltdb.New("test", "bleve")
store.SetMergeOperator(&mergeOperator)
analysisQueue := index.NewAnalysisQueue(1)
idx := NewFirestorm(store, analysisQueue)
err := idx.Open()
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
@ -531,11 +539,12 @@ func TestIndexBatch(t *testing.T) {
}
}()
store := boltdb.New("test", "bleve")
store.SetMergeOperator(&mergeOperator)
analysisQueue := index.NewAnalysisQueue(1)
idx := NewFirestorm(store, analysisQueue)
err := idx.Open()
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
@ -596,7 +605,7 @@ func TestIndexBatch(t *testing.T) {
}
}()
idx.lookuper.waitTasksDone(lookupWaitDuration)
idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration)
docCount := indexReader.DocCount()
if docCount != expectedCount {
@ -635,11 +644,12 @@ func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) {
}
}()
store := boltdb.New("test", "bleve")
store.SetMergeOperator(&mergeOperator)
analysisQueue := index.NewAnalysisQueue(1)
idx := NewFirestorm(store, analysisQueue)
err := idx.Open()
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
@ -673,7 +683,7 @@ func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) {
}
expectedCount++
idx.lookuper.waitTasksDone(lookupWaitDuration)
idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration)
docCount, err = idx.DocCount()
if err != nil {
@ -791,7 +801,7 @@ func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) {
err = idx.Delete("1")
expectedCount--
idx.lookuper.waitTasksDone(lookupWaitDuration)
idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration)
// expected doc count shouldn't have changed
docCount, err = idx.DocCount()
@ -811,11 +821,12 @@ func TestIndexInsertFields(t *testing.T) {
}
}()
store := boltdb.New("test", "bleve")
store.SetMergeOperator(&mergeOperator)
analysisQueue := index.NewAnalysisQueue(1)
idx := NewFirestorm(store, analysisQueue)
err := idx.Open()
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
@ -870,11 +881,12 @@ func TestIndexUpdateComposites(t *testing.T) {
}
}()
store := boltdb.New("test", "bleve")
store.SetMergeOperator(&mergeOperator)
analysisQueue := index.NewAnalysisQueue(1)
idx := NewFirestorm(store, analysisQueue)
err := idx.Open()
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
@ -940,11 +952,12 @@ func TestIndexFieldsMisc(t *testing.T) {
}
}()
store := boltdb.New("test", "bleve")
store.SetMergeOperator(&mergeOperator)
analysisQueue := index.NewAnalysisQueue(1)
idx := NewFirestorm(store, analysisQueue)
err := idx.Open()
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
@ -963,15 +976,15 @@ func TestIndexFieldsMisc(t *testing.T) {
t.Errorf("Error updating index: %v", err)
}
fieldName1 := idx.fieldCache.FieldIndexed(1)
fieldName1 := idx.(*Firestorm).fieldCache.FieldIndexed(1)
if fieldName1 != "name" {
t.Errorf("expected field named 'name', got '%s'", fieldName1)
}
fieldName2 := idx.fieldCache.FieldIndexed(2)
fieldName2 := idx.(*Firestorm).fieldCache.FieldIndexed(2)
if fieldName2 != "title" {
t.Errorf("expected field named 'title', got '%s'", fieldName2)
}
fieldName3 := idx.fieldCache.FieldIndexed(3)
fieldName3 := idx.(*Firestorm).fieldCache.FieldIndexed(3)
if fieldName3 != "" {
t.Errorf("expected field named '', got '%s'", fieldName3)
}
@ -986,11 +999,12 @@ func TestIndexTermReaderCompositeFields(t *testing.T) {
}
}()
store := boltdb.New("test", "bleve")
store.SetMergeOperator(&mergeOperator)
analysisQueue := index.NewAnalysisQueue(1)
idx := NewFirestorm(store, analysisQueue)
err := idx.Open()
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
@ -1046,11 +1060,12 @@ func TestIndexDocumentFieldTerms(t *testing.T) {
}
}()
store := boltdb.New("test", "bleve")
store.SetMergeOperator(&mergeOperator)
analysisQueue := index.NewAnalysisQueue(1)
idx := NewFirestorm(store, analysisQueue)
err := idx.Open()
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
@ -1101,12 +1116,11 @@ func BenchmarkBatch(b *testing.B) {
b.Fatal(err)
}
s, err := null.New()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
b.Fatal(err)
}
analysisQueue := index.NewAnalysisQueue(1)
idx := NewFirestorm(s, analysisQueue)
err = idx.Open()
if err != nil {
b.Fatal(err)

10
index/firestorm/garbage.go
View File

@ -188,7 +188,7 @@ func (gc *GarbageCollector) cleanup() {
wb.Delete(k)
}
err = wb.Execute()
err = writer.ExecuteBatch(wb)
if err != nil {
writer.Close()
logger.Printf("garbage collector fatal: %v", err)
@ -205,7 +205,13 @@ func (gc *GarbageCollector) cleanup() {
tfidrow := NewTermFreqRow(0, nil, docID, docNum, 0, 0, nil)
markerRowKey := tfidrow.Key()
writer.Delete(markerRowKey)
markerBatch := writer.NewBatch()
markerBatch.Delete(markerRowKey)
err = writer.ExecuteBatch(markerBatch)
if err != nil {
logger.Printf("garbage collector fatal: %v", err)
return
}
writer.Close()
}

54
index/firestorm/garbage_test.go
View File

@ -13,26 +13,22 @@ import (
"testing"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store/inmem"
"github.com/blevesearch/bleve/index/store/gtreap"
)
func TestGarbageCleanup(t *testing.T) {
kv, err := inmem.New()
if err != nil {
t.Fatal(err)
}
aq := index.NewAnalysisQueue(1)
f := NewFirestorm(kv, aq)
err = kv.Open()
f, err := NewFirestorm(gtreap.Name, nil, aq)
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.store.Writer()
err = f.Open()
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.(*Firestorm).store.Writer()
if err != nil {
t.Fatal(err)
}
@ -57,28 +53,40 @@ func TestGarbageCleanup(t *testing.T) {
}
for _, row := range rows {
err = kvwriter.Set(row.row.Key(), row.row.Value())
wb := kvwriter.NewBatch()
wb.Set(row.row.Key(), row.row.Value())
err = kvwriter.ExecuteBatch(wb)
if err != nil {
t.Fatal(err)
}
}
// warmup ensures that deletedDocNums is seeded correctly
err = f.warmup(kvwriter)
if err != nil {
t.Fatal(err)
}
err = kvwriter.Close()
if err != nil {
t.Fatal(err)
}
kvreader, err := f.(*Firestorm).store.Reader()
if err != nil {
t.Fatal(err)
}
// warmup ensures that deletedDocNums is seeded correctly
err = f.(*Firestorm).warmup(kvreader)
if err != nil {
t.Fatal(err)
}
err = kvreader.Close()
if err != nil {
t.Fatal(err)
}
// now invoke garbage collector cleanup manually
f.garbageCollector.cleanup()
f.(*Firestorm).garbageCollector.cleanup()
// assert that garbage rows are gone
reader, err := f.store.Reader()
reader, err := f.(*Firestorm).store.Reader()
if err != nil {
t.Fatal(err)
}
@ -102,8 +110,8 @@ func TestGarbageCleanup(t *testing.T) {
}
// assert that deletedDocsNumbers size is 0
if f.compensator.GarbageCount() != 0 {
t.Errorf("expected deletedDocsNumbers size to be 0, got %d", f.compensator.GarbageCount())
if f.(*Firestorm).compensator.GarbageCount() != 0 {
t.Errorf("expected deletedDocsNumbers size to be 0, got %d", f.(*Firestorm).compensator.GarbageCount())
}
}

24
index/firestorm/internal.go
View File

@ -31,11 +31,29 @@ func NewInternalRowKV(key, value []byte) (*InternalRow, error) {
return &rv, nil
}
func (ir *InternalRow) Key() []byte {
buf := make([]byte, len(ir.key)+1)
func (ir *InternalRow) KeySize() int {
return 1 + len(ir.key)
}
func (ir *InternalRow) KeyTo(buf []byte) (int, error) {
buf[0] = 'i'
copy(buf[1:], ir.key)
return buf
return 1 + len(ir.key), nil
}
func (ir *InternalRow) Key() []byte {
buf := make([]byte, ir.KeySize())
n, _ := ir.KeyTo(buf)
return buf[:n]
}
func (ir *InternalRow) ValueSize() int {
return len(ir.val)
}
func (ir *InternalRow) ValueTo(buf []byte) (int, error) {
copy(buf, ir.val)
return len(ir.val), nil
}
func (ir *InternalRow) Value() []byte {

32
index/firestorm/lookup_test.go
View File

@ -13,26 +13,22 @@ import (
"testing"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store/inmem"
"github.com/blevesearch/bleve/index/store/gtreap"
)
func TestLookups(t *testing.T) {
kv, err := inmem.New()
if err != nil {
t.Fatal(err)
}
aq := index.NewAnalysisQueue(1)
f := NewFirestorm(kv, aq)
err = kv.Open()
f, err := NewFirestorm(gtreap.Name, nil, aq)
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.store.Writer()
err = f.Open()
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.(*Firestorm).store.Writer()
if err != nil {
t.Fatal(err)
}
@ -52,21 +48,23 @@ func TestLookups(t *testing.T) {
}
for _, row := range rows {
err = kvwriter.Set(row.row.Key(), row.row.Value())
wb := kvwriter.NewBatch()
wb.Set(row.row.Key(), row.row.Value())
err = kvwriter.ExecuteBatch(wb)
if err != nil {
t.Fatal(err)
}
// also see the compensator
if tfr, ok := row.row.(*TermFreqRow); ok {
f.compensator.Mutate(tfr.DocID(), tfr.DocNum())
f.(*Firestorm).compensator.Mutate(tfr.DocID(), tfr.DocNum())
// expect this mutation to be in the in-flight list
val := f.compensator.inFlight.Get(&InFlightItem{docID: tfr.DocID()})
val := f.(*Firestorm).compensator.inFlight.Get(&InFlightItem{docID: tfr.DocID()})
if val == nil {
t.Errorf("expected key: % x to be in the inflight list", tfr.DocID())
}
f.lookuper.lookup(&lookupTask{docID: tfr.DocID(), docNum: tfr.DocNum()})
f.(*Firestorm).lookuper.lookup(&lookupTask{docID: tfr.DocID(), docNum: tfr.DocNum()})
// now expect this mutation to NOT be in the in-flight list
val = f.compensator.inFlight.Get(&InFlightItem{docID: tfr.DocID()})
val = f.(*Firestorm).compensator.inFlight.Get(&InFlightItem{docID: tfr.DocID()})
if val != nil {
t.Errorf("expected key: % x to NOT be in the inflight list, got %v", tfr.DocID(), val)
}

7
index/firestorm/reader.go
View File

@ -83,11 +83,8 @@ func (r *firestormReader) Document(id string) (*document.Document, error) {
rv := document.NewDocument(id)
prefix := StoredPrefixDocIDNum(docID, docNum)
err = visitPrefix(r.r, prefix, func(key, val []byte) (bool, error) {
safeVal := val
if !r.r.BytesSafeAfterClose() {
safeVal = make([]byte, len(val))
copy(safeVal, val)
}
safeVal := make([]byte, len(val))
copy(safeVal, val)
row, err := NewStoredRowKV(key, safeVal)
if err != nil {
return false, err

10
index/firestorm/reader_dict.go
View File

@ -10,7 +10,6 @@
package firestorm
import (
"bytes"
"fmt"
"github.com/blevesearch/bleve/index"
@ -21,7 +20,6 @@ type firestormDictionaryReader struct {
r *firestormReader
field uint16
start []byte
end []byte
i store.KVIterator
}
@ -33,12 +31,11 @@ func newFirestormDictionaryReader(r *firestormReader, field uint16, start, end [
}
endKey := DictionaryRowKey(field, end)
logger.Printf("end key '%s' - % x", endKey, endKey)
i := r.r.Iterator(startKey)
i := r.r.RangeIterator(startKey, endKey)
rv := firestormDictionaryReader{
r: r,
field: field,
start: startKey,
end: endKey,
i: i,
}
return &rv, nil
@ -52,11 +49,6 @@ func (r *firestormDictionaryReader) Next() (*index.DictEntry, error) {
logger.Printf("see key '%s' - % x", key, key)
// past end term
if bytes.Compare(key, r.end) > 0 {
return nil, nil
}
currRow, err := NewDictionaryRowKV(key, val)
if err != nil {
return nil, fmt.Errorf("unexpected error parsing dictionary row kv: %v", err)

49
index/firestorm/reader_dict_test.go
View File

@ -17,7 +17,7 @@ import (
"github.com/blevesearch/bleve/analysis"
"github.com/blevesearch/bleve/analysis/tokenizers/regexp_tokenizer"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store/inmem"
"github.com/blevesearch/bleve/index/store/gtreap"
)
var testAnalyzer = &analysis.Analyzer{
@ -25,21 +25,18 @@ var testAnalyzer = &analysis.Analyzer{
}
func TestDictionaryReader(t *testing.T) {
kv, err := inmem.New()
if err != nil {
t.Fatal(err)
}
aq := index.NewAnalysisQueue(1)
f := NewFirestorm(kv, aq)
err = kv.Open()
f, err := NewFirestorm(gtreap.Name, nil, aq)
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.store.Writer()
err = f.Open()
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.(*Firestorm).store.Writer()
if err != nil {
t.Fatal(err)
}
@ -52,13 +49,35 @@ func TestDictionaryReader(t *testing.T) {
}
for _, row := range rows {
err = kvwriter.Set(row.Key(), row.Value())
wb := kvwriter.NewBatch()
wb.Set(row.Key(), row.Value())
err = kvwriter.ExecuteBatch(wb)
if err != nil {
t.Fatal(err)
}
}
f.warmup(kvwriter)
err = kvwriter.Close()
if err != nil {
t.Fatal(err)
}
kvreader, err := f.(*Firestorm).store.Reader()
if err != nil {
t.Fatal(err)
}
f.(*Firestorm).warmup(kvreader)
err = kvreader.Close()
if err != nil {
t.Fatal(err)
}
kvwriter, err = f.(*Firestorm).store.Writer()
if err != nil {
t.Fatal(err)
}
rows = []index.IndexRow{
@ -79,7 +98,9 @@ func TestDictionaryReader(t *testing.T) {
}
for _, row := range rows {
err = kvwriter.Set(row.Key(), row.Value())
wb := kvwriter.NewBatch()
wb.Set(row.Key(), row.Value())
err = kvwriter.ExecuteBatch(wb)
if err != nil {
t.Fatal(err)
}

6
index/firestorm/reader_docs.go
View File

@ -18,7 +18,6 @@ import (
type firestormDocIDReader struct {
r *firestormReader
start []byte
end []byte
i store.KVIterator
}
@ -35,12 +34,11 @@ func newFirestormDocIDReader(r *firestormReader, start, end string) (*firestormD
logger.Printf("end key '%s' - % x", endKey, endKey)
i := r.r.Iterator(startKey)
i := r.r.RangeIterator(startKey, endKey)
rv := firestormDocIDReader{
r: r,
start: startKey,
end: endKey,
i: i,
}
@ -50,7 +48,7 @@ func newFirestormDocIDReader(r *firestormReader, start, end string) (*firestormD
func (r *firestormDocIDReader) Next() (string, error) {
if r.i != nil {
key, val, valid := r.i.Current()
for valid && bytes.Compare(key, r.end) <= 0 {
for valid {
logger.Printf("see key: '%s' - % x", key, key)
tfrsByDocNum := make(map[uint64]*TermFreqRow)
tfr, err := NewTermFreqRowKV(key, val)

44
index/firestorm/reader_docs_test.go
View File

@ -15,25 +15,22 @@ import (
"testing"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store/inmem"
"github.com/blevesearch/bleve/index/store/gtreap"
)
func TestDocIDReaderSomeGarbage(t *testing.T) {
kv, err := inmem.New()
if err != nil {
t.Fatal(err)
}
aq := index.NewAnalysisQueue(1)
f := NewFirestorm(kv, aq)
err = kv.Open()
f, err := NewFirestorm(gtreap.Name, nil, aq)
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.store.Writer()
err = f.Open()
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.(*Firestorm).store.Writer()
if err != nil {
t.Fatal(err)
}
@ -64,17 +61,34 @@ func TestDocIDReaderSomeGarbage(t *testing.T) {
}
for _, row := range rows {
err = kvwriter.Set(row.Key(), row.Value())
wb := kvwriter.NewBatch()
wb.Set(row.Key(), row.Value())
err = kvwriter.ExecuteBatch(wb)
if err != nil {
t.Fatal(err)
}
}
f.compensator.inFlight = f.compensator.inFlight.Upsert(&InFlightItem{docID: []byte("c"), docNum: 0}, rand.Int())
f.compensator.deletedDocNumbers.Set(4)
f.(*Firestorm).compensator.inFlight = f.(*Firestorm).compensator.inFlight.Upsert(&InFlightItem{docID: []byte("c"), docNum: 0}, rand.Int())
f.(*Firestorm).compensator.deletedDocNumbers.Set(4)
err = kvwriter.Close()
if err != nil {
t.Fatal(err)
}
kvreader, err := f.(*Firestorm).store.Reader()
if err != nil {
t.Fatal(err)
}
// warmup to load field cache and set maxRead correctly
f.warmup(kvwriter)
f.(*Firestorm).warmup(kvreader)
err = kvreader.Close()
if err != nil {
t.Fatal(err)
}
r, err := f.Reader()
if err != nil {

4
index/firestorm/reader_terms.go
View File

@ -34,7 +34,7 @@ func newFirestormTermFieldReader(r *firestormReader, field uint16, term []byte)
prefix := TermFreqIteratorStart(field, term)
logger.Printf("starting term freq iterator at: '%s' - % x", prefix, prefix)
i := r.r.Iterator(prefix)
i := r.r.PrefixIterator(prefix)
rv := firestormTermFieldReader{
r: r,
field: field,
@ -61,7 +61,7 @@ func newFirestormTermFieldReader(r *firestormReader, field uint16, term []byte)
func (r *firestormTermFieldReader) Next() (*index.TermFieldDoc, error) {
if r.i != nil {
key, val, valid := r.i.Current()
for valid && bytes.HasPrefix(key, r.prefix) {
for valid {
logger.Printf("see key: '%s' - % x", key, key)
tfrsByDocNum := make(map[uint64]*TermFreqRow)
tfr, err := NewTermFreqRowKV(key, val)

82
index/firestorm/reader_terms_test.go
View File

@ -15,25 +15,22 @@ import (
"testing"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store/inmem"
"github.com/blevesearch/bleve/index/store/gtreap"
)
func TestTermReaderNoGarbage(t *testing.T) {
kv, err := inmem.New()
if err != nil {
t.Fatal(err)
}
aq := index.NewAnalysisQueue(1)
f := NewFirestorm(kv, aq)
err = kv.Open()
f, err := NewFirestorm(gtreap.Name, nil, aq)
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.store.Writer()
err = f.Open()
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.(*Firestorm).store.Writer()
if err != nil {
t.Fatal(err)
}
@ -49,14 +46,31 @@ func TestTermReaderNoGarbage(t *testing.T) {
}
for _, row := range rows {
err = kvwriter.Set(row.Key(), row.Value())
wb := kvwriter.NewBatch()
wb.Set(row.Key(), row.Value())
err = kvwriter.ExecuteBatch(wb)
if err != nil {
t.Fatal(err)
}
}
err = kvwriter.Close()
if err != nil {
t.Fatal(err)
}
kvreader, err := f.(*Firestorm).store.Reader()
if err != nil {
t.Fatal(err)
}
// warmup to load field cache and set maxRead correctly
f.warmup(kvwriter)
f.(*Firestorm).warmup(kvreader)
err = kvreader.Close()
if err != nil {
t.Fatal(err)
}
r, err := f.Reader()
if err != nil {
@ -95,21 +109,18 @@ func TestTermReaderNoGarbage(t *testing.T) {
}
func TestTermReaderSomeGarbage(t *testing.T) {
kv, err := inmem.New()
if err != nil {
t.Fatal(err)
}
aq := index.NewAnalysisQueue(1)
f := NewFirestorm(kv, aq)
err = kv.Open()
f, err := NewFirestorm(gtreap.Name, nil, aq)
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.store.Writer()
err = f.Open()
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.(*Firestorm).store.Writer()
if err != nil {
t.Fatal(err)
}
@ -140,17 +151,34 @@ func TestTermReaderSomeGarbage(t *testing.T) {
}
for _, row := range rows {
err = kvwriter.Set(row.Key(), row.Value())
wb := kvwriter.NewBatch()
wb.Set(row.Key(), row.Value())
err = kvwriter.ExecuteBatch(wb)
if err != nil {
t.Fatal(err)
}
}
f.compensator.inFlight = f.compensator.inFlight.Upsert(&InFlightItem{docID: []byte("c"), docNum: 0}, rand.Int())
f.compensator.deletedDocNumbers.Set(4)
f.(*Firestorm).compensator.inFlight = f.(*Firestorm).compensator.inFlight.Upsert(&InFlightItem{docID: []byte("c"), docNum: 0}, rand.Int())
f.(*Firestorm).compensator.deletedDocNumbers.Set(4)
err = kvwriter.Close()
if err != nil {
t.Fatal(err)
}
kvreader, err := f.(*Firestorm).store.Reader()
if err != nil {
t.Fatal(err)
}
// warmup to load field cache and set maxRead correctly
f.warmup(kvwriter)
f.(*Firestorm).warmup(kvreader)
err = kvreader.Close()
if err != nil {
t.Fatal(err)
}
r, err := f.Reader()
if err != nil {

34
index/firestorm/stored.go
View File

@ -81,13 +81,15 @@ func NewStoredRowKV(key, value []byte) (*StoredRow, error) {
return &rv, nil
}
func (sr *StoredRow) Key() []byte {
docLen := len(sr.docID)
buf := make([]byte, 1+docLen+1+binary.MaxVarintLen64+2+(binary.MaxVarintLen64*len(sr.arrayPositions)))
func (sr *StoredRow) KeySize() int {
return 1 + len(sr.docID) + 1 + binary.MaxVarintLen64 + 2 + (binary.MaxVarintLen64 * len(sr.arrayPositions))
}
func (sr *StoredRow) KeyTo(buf []byte) (int, error) {
buf[0] = 's'
copy(buf[1:], sr.docID)
buf[1+docLen] = ByteSeparator
bytesUsed := 1 + docLen + 1
buf[1+len(sr.docID)] = ByteSeparator
bytesUsed := 1 + len(sr.docID) + 1
bytesUsed += binary.PutUvarint(buf[bytesUsed:], sr.docNum)
binary.LittleEndian.PutUint16(buf[bytesUsed:], sr.field)
bytesUsed += 2
@ -95,12 +97,28 @@ func (sr *StoredRow) Key() []byte {
varbytes := binary.PutUvarint(buf[bytesUsed:], arrayPosition)
bytesUsed += varbytes
}
return buf[0:bytesUsed]
return bytesUsed, nil
}
func (sr *StoredRow) Key() []byte {
buf := make([]byte, sr.KeySize())
n, _ := sr.KeyTo(buf)
return buf[:n]
}
func (sr *StoredRow) ValueSize() int {
return sr.value.Size()
}
func (sr *StoredRow) ValueTo(buf []byte) (int, error) {
return sr.value.MarshalTo(buf)
}
func (sr *StoredRow) Value() []byte {
rv, _ := sr.value.Marshal()
return rv
buf := make([]byte, sr.ValueSize())
n, _ := sr.ValueTo(buf)
return buf[:n]
}
func (sr *StoredRow) DocID() []byte {

37
index/firestorm/termfreq.go
View File

@ -90,8 +90,11 @@ func NewTermFreqRowKV(key, value []byte) (*TermFreqRow, error) {
return &rv, nil
}
func (tfr *TermFreqRow) Key() []byte {
buf := make([]byte, 3+len(tfr.term)+1+len(tfr.docID)+1+binary.MaxVarintLen64)
func (tfr *TermFreqRow) KeySize() int {
return 3 + len(tfr.term) + 1 + len(tfr.docID) + 1 + binary.MaxVarintLen64
}
func (tfr *TermFreqRow) KeyTo(buf []byte) (int, error) {
buf[0] = 't'
binary.LittleEndian.PutUint16(buf[1:3], tfr.field)
termLen := copy(buf[3:], tfr.term)
@ -99,12 +102,27 @@ func (tfr *TermFreqRow) Key() []byte {
docLen := copy(buf[3+termLen+1:], tfr.docID)
buf[3+termLen+1+docLen] = ByteSeparator
used := binary.PutUvarint(buf[3+termLen+1+docLen+1:], tfr.docNum)
return buf[:3+termLen+1+docLen+1+used]
return 3 + termLen + 1 + docLen + 1 + used, nil
}
func (tfr *TermFreqRow) Key() []byte {
buf := make([]byte, tfr.KeySize())
n, _ := tfr.KeyTo(buf)
return buf[:n]
}
func (tfr *TermFreqRow) ValueSize() int {
return tfr.value.Size()
}
func (tfr *TermFreqRow) ValueTo(buf []byte) (int, error) {
return tfr.value.MarshalTo(buf)
}
func (tfr *TermFreqRow) Value() []byte {
rv, _ := tfr.value.Marshal()
return rv
buf := make([]byte, tfr.ValueSize())
n, _ := tfr.ValueTo(buf)
return buf[:n]
}
func (tfr *TermFreqRow) String() string {
@ -149,6 +167,15 @@ func (tfr *TermFreqRow) Vectors() []*TermVector {
return tfr.value.GetVectors()
}
func (tfr *TermFreqRow) DictionaryRowKeySize() int {
return 3 + len(tfr.term)
}
func (tfr *TermFreqRow) DictionaryRowKeyTo(buf []byte) (int, error) {
dr := NewDictionaryRow(tfr.field, tfr.term, 0)
return dr.KeyTo(buf)
}
func (tfr *TermFreqRow) DictionaryRowKey() []byte {
dr := NewDictionaryRow(tfr.field, tfr.term, 0)
return dr.Key()

9
index/firestorm/util.go
View File

@ -10,7 +10,6 @@
package firestorm
import (
"bytes"
"io/ioutil"
"log"
@ -24,14 +23,14 @@ func visitPrefix(reader store.KVReader, prefix []byte, visitor KVVisitor) (err e
if start == nil {
start = []byte{0}
}
it := reader.Iterator(start)
it := reader.PrefixIterator(prefix)
defer func() {
if cerr := it.Close(); err == nil && cerr != nil {
err = cerr
}
}()
k, v, valid := it.Current()
for valid && bytes.HasPrefix(k, prefix) {
for valid {
var cont bool
cont, err = visitor(k, v)
if err != nil {
@ -49,14 +48,14 @@ func visitPrefix(reader store.KVReader, prefix []byte, visitor KVVisitor) (err e
}
func visitRange(reader store.KVReader, start, end []byte, visitor KVVisitor) (err error) {
it := reader.Iterator(start)
it := reader.RangeIterator(start, end)
defer func() {
if cerr := it.Close(); err == nil && cerr != nil {
err = cerr
}
}()
k, v, valid := it.Current()
for valid && bytes.Compare(k, end) < 0 {
for valid {
var cont bool
cont, err = visitor(k, v)
if err != nil {

26
index/firestorm/version.go
View File

@ -42,13 +42,31 @@ func NewVersionRowV(val []byte) (*VersionRow, error) {
return &rv, nil
}
func (vr *VersionRow) KeySize() int {
return 1
}
func (vr *VersionRow) KeyTo(buf []byte) (int, error) {
buf[0] = VersionKey[0]
return 1, nil
}
func (vr *VersionRow) Key() []byte {
return VersionKey
}
func (vr *VersionRow) ValueSize() int {
return vr.value.Size()
}
func (vr *VersionRow) ValueTo(buf []byte) (int, error) {
return vr.value.MarshalTo(buf)
}
func (vr *VersionRow) Value() []byte {
rv, _ := vr.value.Marshal()
return rv
buf := make([]byte, vr.ValueSize())
n, _ := vr.value.MarshalTo(buf)
return buf[:n]
}
func (vr *VersionRow) Version() uint64 {
@ -83,6 +101,8 @@ func (f *Firestorm) checkVersion(reader store.KVReader) (newIndex bool, err erro
func (f *Firestorm) storeVersion(writer store.KVWriter) error {
vr := NewVersionRow(Version)
err := writer.Set(vr.Key(), vr.Value())
wb := writer.NewBatch()
wb.Set(vr.Key(), vr.Value())
err := writer.ExecuteBatch(wb)
return err
}

34
index/firestorm/warmup.go
View File

@ -20,25 +20,39 @@ import (
const IDFieldName = "_id"
func (f *Firestorm) bootstrap(writer store.KVWriter) error {
// record version
err := f.storeVersion(writer)
func (f *Firestorm) bootstrap() (err error) {
kvwriter, err := f.store.Writer()
if err != nil {
return err
return
}
defer func() {
if cerr := kvwriter.Close(); err == nil && cerr != nil {
err = cerr
}
}()
// record version
err = f.storeVersion(kvwriter)
if err != nil {
return
}
// define _id field
_, idFieldRow := f.fieldIndexOrNewRow(IDFieldName)
err = writer.Set(idFieldRow.Key(), idFieldRow.Value())
wb := kvwriter.NewBatch()
wb.Set(idFieldRow.Key(), idFieldRow.Value())
err = kvwriter.ExecuteBatch(wb)
if err != nil {
return err
return
}
return nil
return
}
func (f *Firestorm) warmup(writer store.KVWriter) error {
func (f *Firestorm) warmup(reader store.KVReader) error {
// load all the existing fields
err := f.loadFields(writer)
err := f.loadFields(reader)
if err != nil {
return err
}
@ -55,7 +69,7 @@ func (f *Firestorm) warmup(writer store.KVWriter) error {
var lastDocId []byte
lastDocNumbers := make(DocNumberList, 1)
err = visitPrefix(writer, tfkPrefix, func(key, val []byte) (bool, error) {
err = visitPrefix(reader, tfkPrefix, func(key, val []byte) (bool, error) {
tfk, err := NewTermFreqRowKV(key, val)
if err != nil {
return false, err

126
index/firestorm/warmup_test.go
View File

@ -13,37 +13,23 @@ import (
"testing"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store/inmem"
"github.com/blevesearch/bleve/index/store/gtreap"
)
func TestBootstrap(t *testing.T) {
kv, err := inmem.New()
if err != nil {
t.Fatal(err)
}
aq := index.NewAnalysisQueue(1)
f := NewFirestorm(kv, aq)
err = kv.Open()
f, err := NewFirestorm(gtreap.Name, nil, aq)
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.store.Writer()
if err != nil {
t.Fatal(err)
}
err = f.bootstrap(kvwriter)
err = f.Open() // open calls bootstrap
if err != nil {
t.Fatal(err)
}
// assert that version is set
reader, err := kv.Reader()
reader, err := f.(*Firestorm).store.Reader()
if err != nil {
t.Fatal(err)
}
@ -60,7 +46,7 @@ func TestBootstrap(t *testing.T) {
}
// assert that field cache has _id
id, existed := f.fieldCache.FieldNamed(IDFieldName, false)
id, existed := f.(*Firestorm).fieldCache.FieldNamed(IDFieldName, false)
if !existed {
t.Errorf("expect '%s' in field cache", IDFieldName)
}
@ -84,29 +70,25 @@ func TestBootstrap(t *testing.T) {
}
// assert that highDocNumber is 0
if f.highDocNumber != 0 {
t.Errorf("expected highDocNumber to be 0, got %d", f.highDocNumber)
if f.(*Firestorm).highDocNumber != 0 {
t.Errorf("expected highDocNumber to be 0, got %d", f.(*Firestorm).highDocNumber)
}
}
func TestWarmupNoGarbage(t *testing.T) {
kv, err := inmem.New()
if err != nil {
t.Fatal(err)
}
aq := index.NewAnalysisQueue(1)
f := NewFirestorm(kv, aq)
err = kv.Open()
f, err := NewFirestorm(gtreap.Name, nil, aq)
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.store.Writer()
err = f.Open()
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.(*Firestorm).store.Writer()
if err != nil {
t.Fatal(err)
}
@ -121,13 +103,30 @@ func TestWarmupNoGarbage(t *testing.T) {
expectedGarbage := uint64(0)
for _, row := range rows {
err = kvwriter.Set(row.Key(), row.Value())
wb := kvwriter.NewBatch()
wb.Set(row.Key(), row.Value())
err = kvwriter.ExecuteBatch(wb)
if err != nil {
t.Fatal(err)
}
}
err = f.warmup(kvwriter)
err = kvwriter.Close()
if err != nil {
t.Fatal(err)
}
kvreader, err := f.(*Firestorm).store.Reader()
if err != nil {
t.Fatal(err)
}
err = f.(*Firestorm).warmup(kvreader)
if err != nil {
t.Fatal(err)
}
err = kvreader.Close()
if err != nil {
t.Fatal(err)
}
@ -142,33 +141,29 @@ func TestWarmupNoGarbage(t *testing.T) {
}
// assert that deleted doc numbers size is 0
if f.compensator.GarbageCount() != expectedGarbage {
t.Errorf("expected 0 deleted doc numbers, got %d", f.compensator.GarbageCount())
if f.(*Firestorm).compensator.GarbageCount() != expectedGarbage {
t.Errorf("expected 0 deleted doc numbers, got %d", f.(*Firestorm).compensator.GarbageCount())
}
// assert that highDocNumber is 3
if f.highDocNumber != 3 {
t.Errorf("expected highDocNumber to be 3, got %d", f.highDocNumber)
if f.(*Firestorm).highDocNumber != 3 {
t.Errorf("expected highDocNumber to be 3, got %d", f.(*Firestorm).highDocNumber)
}
}
func TestWarmupSomeGarbage(t *testing.T) {
kv, err := inmem.New()
if err != nil {
t.Fatal(err)
}
aq := index.NewAnalysisQueue(1)
f := NewFirestorm(kv, aq)
err = kv.Open()
f, err := NewFirestorm(gtreap.Name, nil, aq)
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.store.Writer()
err = f.Open()
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.(*Firestorm).store.Writer()
if err != nil {
t.Fatal(err)
}
@ -185,13 +180,30 @@ func TestWarmupSomeGarbage(t *testing.T) {
expectedGarbage := uint64(2)
for _, row := range rows {
err = kvwriter.Set(row.Key(), row.Value())
wb := kvwriter.NewBatch()
wb.Set(row.Key(), row.Value())
err = kvwriter.ExecuteBatch(wb)
if err != nil {
t.Fatal(err)
}
}
err = f.warmup(kvwriter)
err = kvwriter.Close()
if err != nil {
t.Fatal(err)
}
kvreader, err := f.(*Firestorm).store.Reader()
if err != nil {
t.Fatal(err)
}
err = f.(*Firestorm).warmup(kvreader)
if err != nil {
t.Fatal(err)
}
err = kvreader.Close()
if err != nil {
t.Fatal(err)
}
@ -206,20 +218,20 @@ func TestWarmupSomeGarbage(t *testing.T) {
}
// assert that deleted doc numbers size is 0
if f.compensator.GarbageCount() != expectedGarbage {
t.Errorf("expected %d deleted doc numbers, got %d", expectedGarbage, f.compensator.GarbageCount())
if f.(*Firestorm).compensator.GarbageCount() != expectedGarbage {
t.Errorf("expected %d deleted doc numbers, got %d", expectedGarbage, f.(*Firestorm).compensator.GarbageCount())
}
// assert that doc numbers 1 and 4 are on the deleted list
if !f.compensator.deletedDocNumbers.Test(1) {
if !f.(*Firestorm).compensator.deletedDocNumbers.Test(1) {
t.Errorf("expected doc number 1 to be deleted")
}
if !f.compensator.deletedDocNumbers.Test(4) {
if !f.(*Firestorm).compensator.deletedDocNumbers.Test(4) {
t.Errorf("expected doc number 4 to be deleted")
}
// assert that highDocNumber is 5
if f.highDocNumber != 5 {
t.Errorf("expected highDocNumber to be 5, got %d", f.highDocNumber)
if f.(*Firestorm).highDocNumber != 5 {
t.Errorf("expected highDocNumber to be 5, got %d", f.(*Firestorm).highDocNumber)
}
}

19
index/index.go
View File

@ -16,6 +16,8 @@ import (
"github.com/blevesearch/bleve/document"
)
var ErrorUnknownStorageType = fmt.Errorf("unknown storage type")
type Index interface {
Open() error
Close() error
@ -33,6 +35,8 @@ type Index interface {
DumpDoc(id string) chan interface{}
DumpFields() chan interface{}
// Reader returns a low-level accessor on the index data. Close it to
// release associated resources.
Reader() (IndexReader, error)
Stats() json.Marshaler
@ -42,9 +46,15 @@ type Index interface {
type IndexReader interface {
TermFieldReader(term []byte, field string) (TermFieldReader, error)
// DocIDReader returns an iterator over documents which identifiers are
// greater than or equal to start and smaller than end. The caller must
// close returned instance to release associated resources.
DocIDReader(start, end string) (DocIDReader, error)
FieldDict(field string) (FieldDict, error)
// FieldDictRange is currently defined to include the start and end terms
FieldDictRange(field string, startTerm []byte, endTerm []byte) (FieldDict, error)
FieldDictPrefix(field string, termPrefix []byte) (FieldDict, error)
@ -95,8 +105,17 @@ type FieldDict interface {
Close() error
}
// DocIDReader is the interface exposing enumeration of documents identifiers.
// Close the reader to release associated resources.
type DocIDReader interface {
// Next returns the next document identifier in ascending lexicographic
// byte order, or io.EOF when the end of the sequence is reached.
Next() (string, error)
// Advance resets the iteration to the first identifier greater than or
// equal to ID. If ID is smaller than the start of the range, the iteration
// will start there instead. If ID is greater than or equal to the end of
// the range, Next() call will return io.EOF.
Advance(ID string) (string, error)
Close() error
}

56
index/store/batch.go
View File

@ -15,55 +15,39 @@ type op struct {
}
type EmulatedBatch struct {
w KVWriter
ops []*op
merge *EmulatedMerge
Ops []*op
Merger *EmulatedMerge
}
func NewEmulatedBatch(w KVWriter, mo MergeOperator) *EmulatedBatch {
func NewEmulatedBatch(mo MergeOperator) *EmulatedBatch {
return &EmulatedBatch{
w: w,
ops: make([]*op, 0, 1000),
merge: NewEmulatedMerge(mo),
Ops: make([]*op, 0, 1000),
Merger: NewEmulatedMerge(mo),
}
}
func (b *EmulatedBatch) Set(key, val []byte) {
b.ops = append(b.ops, &op{key, val})
ck := make([]byte, len(key))
copy(ck, key)
cv := make([]byte, len(val))
copy(cv, val)
b.Ops = append(b.Ops, &op{ck, cv})
}
func (b *EmulatedBatch) Delete(key []byte) {
b.ops = append(b.ops, &op{key, nil})
ck := make([]byte, len(key))
copy(ck, key)
b.Ops = append(b.Ops, &op{ck, nil})
}
func (b *EmulatedBatch) Merge(key, val []byte) {
b.merge.Merge(key, val)
ck := make([]byte, len(key))
copy(ck, key)
cv := make([]byte, len(val))
copy(cv, val)
b.Merger.Merge(key, val)
}
func (b *EmulatedBatch) Execute() error {
// first process merges
err := b.merge.Execute(b.w)
if err != nil {
return err
}
// now apply all the ops
for _, op := range b.ops {
if op.V != nil {
err := b.w.Set(op.K, op.V)
if err != nil {
return err
}
} else {
err := b.w.Delete(op.K)
if err != nil {
return err
}
}
}
return nil
}
func (b *EmulatedBatch) Close() error {
return nil
func (b *EmulatedBatch) Reset() {
b.Ops = b.Ops[:0]
}

30
index/store/boltdb/iterator.go
View File

@ -10,6 +10,8 @@
package boltdb
import (
"bytes"
"github.com/boltdb/bolt"
)
@ -17,24 +19,44 @@ type Iterator struct {
store *Store
tx *bolt.Tx
cursor *bolt.Cursor
prefix []byte
start []byte
end []byte
valid bool
key []byte
val []byte
}
func (i *Iterator) SeekFirst() {
i.key, i.val = i.cursor.First()
func (i *Iterator) updateValid() {
i.valid = (i.key != nil)
if i.valid {
if i.prefix != nil {
i.valid = bytes.HasPrefix(i.key, i.prefix)
} else if i.end != nil {
i.valid = bytes.Compare(i.key, i.end) < 0
}
}
}
func (i *Iterator) Seek(k []byte) {
if i.start != nil && bytes.Compare(k, i.start) < 0 {
k = i.start
}
if i.prefix != nil && !bytes.HasPrefix(k, i.prefix) {
if bytes.Compare(k, i.prefix) < 0 {
k = i.prefix
} else {
i.valid = false
return
}
}
i.key, i.val = i.cursor.Seek(k)
i.valid = (i.key != nil)
i.updateValid()
}
func (i *Iterator) Next() {
i.key, i.val = i.cursor.Next()
i.valid = (i.key != nil)
i.updateValid()
}
func (i *Iterator) Current() ([]byte, []byte, bool) {

32
index/store/boltdb/reader.go
View File

@ -19,16 +19,17 @@ type Reader struct {
tx *bolt.Tx
}
func (r *Reader) BytesSafeAfterClose() bool {
return false
}
func (r *Reader) Get(key []byte) ([]byte, error) {
rv := r.tx.Bucket([]byte(r.store.bucket)).Get(key)
var rv []byte
v := r.tx.Bucket([]byte(r.store.bucket)).Get(key)
if v != nil {
rv = make([]byte, len(v))
copy(rv, v)
}
return rv, nil
}
func (r *Reader) Iterator(key []byte) store.KVIterator {
func (r *Reader) PrefixIterator(prefix []byte) store.KVIterator {
b := r.tx.Bucket([]byte(r.store.bucket))
cursor := b.Cursor()
@ -36,9 +37,26 @@ func (r *Reader) Iterator(key []byte) store.KVIterator {
store: r.store,
tx: r.tx,
cursor: cursor,
prefix: prefix,
}
rv.Seek(key)
rv.Seek(prefix)
return rv
}
func (r *Reader) RangeIterator(start, end []byte) store.KVIterator {
b := r.tx.Bucket([]byte(r.store.bucket))
cursor := b.Cursor()
rv := &Iterator{
store: r.store,
tx: r.tx,
cursor: cursor,
start: start,
end: end,
}
rv.Seek(start)
return rv
}

83
index/store/boltdb/store.go
View File

@ -7,11 +7,18 @@
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// Package boltdb implements a store.KVStore on top of BoltDB. It supports the
// following options:
//
// "bucket" (string): the name of BoltDB bucket to use, defaults to "bleve".
//
// "nosync" (bool): if true, set boltdb.DB.NoSync to true. It speeds up index
// operations in exchange of losing integrity guarantees if indexation aborts
// without closing the index. Use it when rebuilding indexes from zero.
package boltdb
import (
"fmt"
"sync"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
@ -24,40 +31,46 @@ type Store struct {
path string
bucket string
db *bolt.DB
writer sync.Mutex
noSync bool
mo store.MergeOperator
}
func New(path string, bucket string) *Store {
rv := Store{
path: path,
bucket: bucket,
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
path, ok := config["path"].(string)
if !ok {
return nil, fmt.Errorf("must specify path")
}
return &rv
}
func (bs *Store) Open() error {
bucket, ok := config["bucket"].(string)
if !ok {
bucket = "bleve"
}
var err error
bs.db, err = bolt.Open(bs.path, 0600, nil)
noSync, _ := config["nosync"].(bool)
db, err := bolt.Open(path, 0600, nil)
if err != nil {
return err
return nil, err
}
db.NoSync = noSync
err = bs.db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(bs.bucket))
err = db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(bucket))
return err
})
if err != nil {
return err
return nil, err
}
return nil
}
func (bs *Store) SetMergeOperator(mo store.MergeOperator) {
bs.mo = mo
rv := Store{
path: path,
bucket: bucket,
db: db,
mo: mo,
noSync: noSync,
}
return &rv, nil
}
func (bs *Store) Close() error {
@ -76,37 +89,11 @@ func (bs *Store) Reader() (store.KVReader, error) {
}
func (bs *Store) Writer() (store.KVWriter, error) {
bs.writer.Lock()
tx, err := bs.db.Begin(true)
if err != nil {
bs.writer.Unlock()
return nil, err
}
reader := &Reader{
store: bs,
tx: tx,
}
return &Writer{
store: bs,
tx: tx,
reader: reader,
store: bs,
}, nil
}
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
path, ok := config["path"].(string)
if !ok {
return nil, fmt.Errorf("must specify path")
}
bucket, ok := config["bucket"].(string)
if !ok {
bucket = "bleve"
}
return New(path, bucket), nil
}
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
registry.RegisterKVStore(Name, New)
}

325
index/store/boltdb/store_test.go
View File

@ -10,289 +10,82 @@
package boltdb
import (
"fmt"
"os"
"reflect"
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/index/store/test"
)
func TestStore(t *testing.T) {
s := New("test", "bleve")
err := s.Open()
func open(t *testing.T, mo store.MergeOperator) store.KVStore {
rv, err := New(mo, map[string]interface{}{"path": "test"})
if err != nil {
t.Fatal(err)
}
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
CommonTestKVStore(t, s)
return rv
}
func TestReaderIsolation(t *testing.T) {
s := New("test", "bleve")
err := s.Open()
func cleanup(t *testing.T, s store.KVStore) {
err := s.Close()
if err != nil {
t.Fatal(err)
}
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
CommonTestReaderIsolation(t, s)
}
func CommonTestKVStore(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Set([]byte("z"), []byte("val-z"))
if err != nil {
t.Fatal(err)
}
err = writer.Delete([]byte("z"))
if err != nil {
t.Fatal(err)
}
batch := writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
batch.Set([]byte("c"), []byte("val-c"))
batch.Set([]byte("d"), []byte("val-d"))
batch.Set([]byte("e"), []byte("val-e"))
batch.Set([]byte("f"), []byte("val-f"))
batch.Set([]byte("g"), []byte("val-g"))
batch.Set([]byte("h"), []byte("val-h"))
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = batch.Execute()
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("b"))
key, val, valid := it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "b" {
t.Fatalf("expected key b, got %s", key)
}
if string(val) != "val-b" {
t.Fatalf("expected value val-b, got %s", val)
}
it.Next()
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "c" {
t.Fatalf("expected key c, got %s", key)
}
if string(val) != "val-c" {
t.Fatalf("expected value val-c, got %s", val)
}
it.Seek([]byte("i"))
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "i" {
t.Fatalf("expected key i, got %s", key)
}
if string(val) != "val-i" {
t.Fatalf("expected value val-i, got %s", val)
}
err = it.Close()
err = os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}
func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
// insert a kv pair
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
// **************************************************
// this is a hack to try to pre-emptively overflow
// boltdb writes *MAY* block a long reader
// in particular, if the write requires additional
// allocation, it must acquire the same lock as
// the reader, thus cannot continue until that
// reader is closed.
// in general this is not a problem for bleve
// (though it may affect performance in some cases)
// but it is a problem for this test which attemps
// to easily verify that readers are isolated
// this hack writes enough initial data such that
// the subsequent writes do not require additional
// space
hackSize := 1000
for i := 0; i < hackSize; i++ {
k := fmt.Sprintf("x%d", i)
err = writer.Set([]byte(k), []byte("filler"))
if err != nil {
t.Fatal(err)
}
}
// **************************************************
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// create an isolated reader
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
// verify that we see the value already inserted
val, err := reader.Get([]byte("a"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-a")) {
t.Errorf("expected val-a, got nil")
}
// verify that an iterator sees it
count := 0
it := reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != hackSize+1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
// add something after the reader was created
writer, err = s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("b"), []byte("val-b"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// ensure that a newer reader sees it
newReader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := newReader.Close()
if err != nil {
t.Fatal(err)
}
}()
val, err = newReader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-b")) {
t.Errorf("expected val-b, got nil")
}
// ensure that the director iterator sees it
count = 0
it2 := newReader.Iterator([]byte{0})
defer func() {
err := it2.Close()
if err != nil {
t.Fatal(err)
}
}()
for it2.Valid() {
it2.Next()
count++
}
if count != hackSize+2 {
t.Errorf("expected iterator to see 2, saw %d", count)
}
// but that the isolated reader does not
val, err = reader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got %v", val)
}
// and ensure that the iterator on the isolated reader also does not
count = 0
it3 := reader.Iterator([]byte{0})
defer func() {
err := it3.Close()
if err != nil {
t.Fatal(err)
}
}()
for it3.Valid() {
it3.Next()
count++
}
if count != hackSize+1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
func TestBoltDBKVCrud(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestKVCrud(t, s)
}
func TestBoltDBReaderIsolation(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestReaderIsolation(t, s)
}
func TestBoltDBReaderOwnsGetBytes(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestReaderOwnsGetBytes(t, s)
}
func TestBoltDBWriterOwnsBytes(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestWriterOwnsBytes(t, s)
}
func TestBoltDBPrefixIterator(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestPrefixIterator(t, s)
}
func TestBoltDBPrefixIteratorSeek(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestPrefixIteratorSeek(t, s)
}
func TestBoltDBRangeIterator(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestRangeIterator(t, s)
}
func TestBoltDBRangeIteratorSeek(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestRangeIteratorSeek(t, s)
}
func TestBoltDBMerge(t *testing.T) {
s := open(t, &test.TestMergeCounter{})
defer cleanup(t, s)
test.CommonTestMerge(t, s)
}

74
index/store/boltdb/writer.go
View File

@ -10,41 +10,61 @@
package boltdb
import (
"fmt"
"github.com/blevesearch/bleve/index/store"
"github.com/boltdb/bolt"
)
type Writer struct {
store *Store
tx *bolt.Tx
reader *Reader
}
func (w *Writer) Set(key, val []byte) error {
return w.tx.Bucket([]byte(w.store.bucket)).Put(key, val)
}
func (w *Writer) Delete(key []byte) error {
return w.tx.Bucket([]byte(w.store.bucket)).Delete(key)
store *Store
}
func (w *Writer) NewBatch() store.KVBatch {
return store.NewEmulatedBatch(w, w.store.mo)
return store.NewEmulatedBatch(w.store.mo)
}
func (w *Writer) ExecuteBatch(batch store.KVBatch) error {
emulatedBatch, ok := batch.(*store.EmulatedBatch)
if !ok {
return fmt.Errorf("wrong type of batch")
}
tx, err := w.store.db.Begin(true)
if err != nil {
return err
}
for k, mergeOps := range emulatedBatch.Merger.Merges {
kb := []byte(k)
existingVal := tx.Bucket([]byte(w.store.bucket)).Get(kb)
mergedVal, fullMergeOk := w.store.mo.FullMerge(kb, existingVal, mergeOps)
if !fullMergeOk {
return fmt.Errorf("merge operator returned failure")
}
err = tx.Bucket([]byte(w.store.bucket)).Put(kb, mergedVal)
if err != nil {
return err
}
}
for _, op := range emulatedBatch.Ops {
if op.V != nil {
err := tx.Bucket([]byte(w.store.bucket)).Put(op.K, op.V)
if err != nil {
return err
}
} else {
err := tx.Bucket([]byte(w.store.bucket)).Delete(op.K)
if err != nil {
return err
}
}
}
return tx.Commit()
}
func (w *Writer) Close() error {
defer w.store.writer.Unlock()
return w.tx.Commit()
}
func (w *Writer) BytesSafeAfterClose() bool {
return w.reader.BytesSafeAfterClose()
}
func (w *Writer) Get(key []byte) ([]byte, error) {
return w.reader.Get(key)
}
func (w *Writer) Iterator(key []byte) store.KVIterator {
return w.reader.Iterator(key)
return nil
}

23
index/store/goleveldb/batch.go
View File

@ -15,7 +15,7 @@ import (
)
type Batch struct {
w *Writer
store *Store
merge *store.EmulatedMerge
batch *leveldb.Batch
}
@ -32,22 +32,7 @@ func (b *Batch) Merge(key, val []byte) {
b.merge.Merge(key, val)
}
func (b *Batch) Execute() error {
// first process merges
ops, err := b.merge.ExecuteDeferred(b.w)
if err != nil {
return err
}
for _, op := range ops {
b.batch.Put(op.K, op.V)
}
wopts := defaultWriteOptions()
err = b.w.store.db.Write(b.batch, wopts)
return err
}
func (b *Batch) Close() error {
return nil
func (b *Batch) Reset() {
b.batch.Reset()
b.merge = store.NewEmulatedMerge(b.store.mo)
}

52
index/store/goleveldb/config.go Normal file
View File

@ -0,0 +1,52 @@
package goleveldb
import (
"github.com/syndtr/goleveldb/leveldb/filter"
"github.com/syndtr/goleveldb/leveldb/opt"
)
func applyConfig(o *opt.Options, config map[string]interface{}) (*opt.Options, error) {
ro, ok := config["read_only"].(bool)
if ok {
o.ReadOnly = ro
}
cim, ok := config["create_if_missing"].(bool)
if ok {
o.ErrorIfMissing = !cim
}
eie, ok := config["error_if_exists"].(bool)
if ok {
o.ErrorIfExist = eie
}
wbs, ok := config["write_buffer_size"].(float64)
if ok {
o.WriteBuffer = int(wbs)
}
bs, ok := config["block_size"].(float64)
if ok {
o.BlockSize = int(bs)
}
bri, ok := config["block_restart_interval"].(float64)
if ok {
o.BlockRestartInterval = int(bri)
}
lcc, ok := config["lru_cache_capacity"].(float64)
if ok {
o.BlockCacheCapacity = int(lcc)
}
bfbpk, ok := config["bloom_filter_bits_per_key"].(float64)
if ok {
bf := filter.NewBloomFilter(int(bfbpk))
o.Filter = bf
}
return o, nil
}

54
index/store/goleveldb/iterator.go
View File

@ -9,53 +9,18 @@
package goleveldb
import (
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/iterator"
)
import "github.com/syndtr/goleveldb/leveldb/iterator"
type Iterator struct {
store *Store
iterator iterator.Iterator
copyk []byte
copyv []byte
}
func newIterator(store *Store) *Iterator {
ropts := defaultReadOptions()
iter := store.db.NewIterator(nil, ropts)
rv := Iterator{
store: store,
iterator: iter,
}
return &rv
}
func newIteratorWithSnapshot(store *Store, snapshot *leveldb.Snapshot) *Iterator {
options := defaultReadOptions()
iter := snapshot.NewIterator(nil, options)
rv := Iterator{
store: store,
iterator: iter,
}
return &rv
}
func (ldi *Iterator) SeekFirst() {
ldi.copyk = nil
ldi.copyv = nil
ldi.iterator.First()
}
func (ldi *Iterator) Seek(key []byte) {
ldi.copyk = nil
ldi.copyv = nil
ldi.iterator.Seek(key)
}
func (ldi *Iterator) Next() {
ldi.copyk = nil
ldi.copyv = nil
ldi.iterator.Next()
}
@ -67,21 +32,11 @@ func (ldi *Iterator) Current() ([]byte, []byte, bool) {
}
func (ldi *Iterator) Key() []byte {
k := ldi.iterator.Key()
if ldi.copyk == nil {
ldi.copyk = make([]byte, len(k))
copy(ldi.copyk, k)
}
return ldi.copyk
return ldi.iterator.Key()
}
func (ldi *Iterator) Value() []byte {
v := ldi.iterator.Value()
if ldi.copyv == nil {
ldi.copyv = make([]byte, len(v))
copy(ldi.copyv, v)
}
return ldi.copyv
return ldi.iterator.Value()
}
func (ldi *Iterator) Valid() bool {
@ -89,7 +44,6 @@ func (ldi *Iterator) Valid() bool {
}
func (ldi *Iterator) Close() error {
ldi.copyk = nil
ldi.copyv = nil
ldi.iterator.Release()
return nil
}

46
index/store/goleveldb/reader.go
View File

@ -12,6 +12,7 @@ package goleveldb
import (
"github.com/blevesearch/bleve/index/store"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/util"
)
type Reader struct {
@ -19,26 +20,37 @@ type Reader struct {
snapshot *leveldb.Snapshot
}
func newReader(store *Store) (*Reader, error) {
snapshot, _ := store.db.GetSnapshot()
return &Reader{
store: store,
snapshot: snapshot,
}, nil
}
func (r *Reader) BytesSafeAfterClose() bool {
return false
}
func (r *Reader) Get(key []byte) ([]byte, error) {
return r.store.getWithSnapshot(key, r.snapshot)
b, err := r.snapshot.Get(key, r.store.defaultReadOptions)
if err == leveldb.ErrNotFound {
return nil, nil
}
return b, err
}
func (r *Reader) Iterator(key []byte) store.KVIterator {
rv := newIteratorWithSnapshot(r.store, r.snapshot)
rv.Seek(key)
return rv
func (r *Reader) PrefixIterator(prefix []byte) store.KVIterator {
byteRange := util.BytesPrefix(prefix)
iter := r.snapshot.NewIterator(byteRange, r.store.defaultReadOptions)
iter.First()
rv := Iterator{
store: r.store,
iterator: iter,
}
return &rv
}
func (r *Reader) RangeIterator(start, end []byte) store.KVIterator {
byteRange := &util.Range{
Start: start,
Limit: end,
}
iter := r.snapshot.NewIterator(byteRange, r.store.defaultReadOptions)
iter.First()
rv := Iterator{
store: r.store,
iterator: iter,
}
return &rv
}
func (r *Reader) Close() error {

161
index/store/goleveldb/store.go
View File

@ -11,167 +11,72 @@ package goleveldb
import (
"fmt"
"sync"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/filter"
"github.com/syndtr/goleveldb/leveldb/opt"
)
const Name = "goleveldb"
type Store struct {
path string
opts *opt.Options
db *leveldb.DB
writer sync.Mutex
mo store.MergeOperator
path string
opts *opt.Options
db *leveldb.DB
mo store.MergeOperator
defaultWriteOptions *opt.WriteOptions
defaultReadOptions *opt.ReadOptions
}
func New(path string, config map[string]interface{}) (*Store, error) {
rv := Store{
path: path,
opts: &opt.Options{},
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
path, ok := config["path"].(string)
if !ok {
return nil, fmt.Errorf("must specify path")
}
_, err := applyConfig(rv.opts, config)
opts, err := applyConfig(&opt.Options{}, config)
if err != nil {
return nil, err
}
return &rv, nil
}
func (ldbs *Store) Open() error {
var err error
ldbs.db, err = leveldb.OpenFile(ldbs.path, ldbs.opts)
db, err := leveldb.OpenFile(path, opts)
if err != nil {
return err
return nil, err
}
return nil
}
func (ldbs *Store) SetMergeOperator(mo store.MergeOperator) {
ldbs.mo = mo
}
func (ldbs *Store) get(key []byte) ([]byte, error) {
options := defaultReadOptions()
b, err := ldbs.db.Get(key, options)
if err == leveldb.ErrNotFound {
return nil, nil
rv := Store{
path: path,
opts: opts,
db: db,
mo: mo,
defaultReadOptions: &opt.ReadOptions{},
defaultWriteOptions: &opt.WriteOptions{},
}
return b, err
}
func (ldbs *Store) getWithSnapshot(key []byte, snapshot *leveldb.Snapshot) ([]byte, error) {
options := defaultReadOptions()
b, err := snapshot.Get(key, options)
if err == leveldb.ErrNotFound {
return nil, nil
}
return b, err
}
func (ldbs *Store) set(key, val []byte) error {
ldbs.writer.Lock()
defer ldbs.writer.Unlock()
return ldbs.setlocked(key, val)
}
func (ldbs *Store) setlocked(key, val []byte) error {
options := defaultWriteOptions()
err := ldbs.db.Put(key, val, options)
return err
}
func (ldbs *Store) delete(key []byte) error {
ldbs.writer.Lock()
defer ldbs.writer.Unlock()
return ldbs.deletelocked(key)
}
func (ldbs *Store) deletelocked(key []byte) error {
options := defaultWriteOptions()
err := ldbs.db.Delete(key, options)
return err
rv.defaultWriteOptions.Sync = true
return &rv, nil
}
func (ldbs *Store) Close() error {
return ldbs.db.Close()
}
func (ldbs *Store) iterator(key []byte) store.KVIterator {
rv := newIterator(ldbs)
rv.Seek(key)
return rv
}
func (ldbs *Store) Reader() (store.KVReader, error) {
return newReader(ldbs)
snapshot, _ := ldbs.db.GetSnapshot()
return &Reader{
store: ldbs,
snapshot: snapshot,
}, nil
}
func (ldbs *Store) Writer() (store.KVWriter, error) {
return newWriter(ldbs)
}
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
path, ok := config["path"].(string)
if !ok {
return nil, fmt.Errorf("must specify path")
}
return New(path, config)
return &Writer{
store: ldbs,
}, nil
}
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
}
func applyConfig(o *opt.Options, config map[string]interface{}) (
*opt.Options, error) {
ro, ok := config["read_only"].(bool)
if ok {
o.ReadOnly = ro
}
cim, ok := config["create_if_missing"].(bool)
if ok {
o.ErrorIfMissing = !cim
}
eie, ok := config["error_if_exists"].(bool)
if ok {
o.ErrorIfExist = eie
}
wbs, ok := config["write_buffer_size"].(float64)
if ok {
o.WriteBuffer = int(wbs)
}
bs, ok := config["block_size"].(float64)
if ok {
o.BlockSize = int(bs)
}
bri, ok := config["block_restart_interval"].(float64)
if ok {
o.BlockRestartInterval = int(bri)
}
lcc, ok := config["lru_cache_capacity"].(float64)
if ok {
o.BlockCacheCapacity = int(lcc)
}
bfbpk, ok := config["bloom_filter_bits_per_key"].(float64)
if ok {
bf := filter.NewBloomFilter(int(bfbpk))
o.Filter = bf
}
return o, nil
registry.RegisterKVStore(Name, New)
}

420
index/store/goleveldb/store_test.go
View File

@ -11,378 +11,84 @@ package goleveldb
import (
"os"
"reflect"
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/index/store/test"
)
var leveldbTestOptions = map[string]interface{}{
"create_if_missing": true,
func open(t *testing.T, mo store.MergeOperator) store.KVStore {
rv, err := New(mo, map[string]interface{}{
"path": "test",
"create_if_missing": true,
})
if err != nil {
t.Fatal(err)
}
return rv
}
func TestLevelDBStore(t *testing.T) {
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
s, err := New("test", leveldbTestOptions)
func cleanup(t *testing.T, s store.KVStore) {
err := s.Close()
if err != nil {
t.Fatal(err)
}
err = s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}()
CommonTestKVStore(t, s)
}
func TestLevelDBStoreIterator(t *testing.T) {
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
s, err := New("test", leveldbTestOptions)
if err != nil {
t.Fatal(err)
}
err = s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}()
CommonTestKVStoreIterator(t, s)
}
func TestReaderIsolation(t *testing.T) {
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
s, err := New("test", leveldbTestOptions)
if err != nil {
t.Fatal(err)
}
err = s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}()
CommonTestReaderIsolation(t, s)
}
func CommonTestKVStore(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Set([]byte("z"), []byte("val-z"))
if err != nil {
t.Fatal(err)
}
err = writer.Delete([]byte("z"))
if err != nil {
t.Fatal(err)
}
batch := writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
batch.Set([]byte("c"), []byte("val-c"))
batch.Set([]byte("d"), []byte("val-d"))
batch.Set([]byte("e"), []byte("val-e"))
batch.Set([]byte("f"), []byte("val-f"))
batch.Set([]byte("g"), []byte("val-g"))
batch.Set([]byte("h"), []byte("val-h"))
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = batch.Execute()
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("b"))
key, val, valid := it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "b" {
t.Fatalf("expected key b, got %s", key)
}
if string(val) != "val-b" {
t.Fatalf("expected value val-b, got %s", val)
}
it.Next()
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "c" {
t.Fatalf("expected key c, got %s", key)
}
if string(val) != "val-c" {
t.Fatalf("expected value val-c, got %s", val)
}
it.Seek([]byte("i"))
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "i" {
t.Fatalf("expected key i, got %s", key)
}
if string(val) != "val-i" {
t.Fatalf("expected value val-i, got %s", val)
}
err = it.Close()
err = os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}
func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
// insert a kv pair
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// create an isolated reader
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
// verify that we see the value already inserted
val, err := reader.Get([]byte("a"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-a")) {
t.Errorf("expected val-a, got nil")
}
// verify that an iterator sees it
count := 0
it := reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
// add something after the reader was created
writer, err = s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("b"), []byte("val-b"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// ensure that a newer reader sees it
newReader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := newReader.Close()
if err != nil {
t.Fatal(err)
}
}()
val, err = newReader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-b")) {
t.Errorf("expected val-b, got nil")
}
// ensure that the director iterator sees it
count = 0
it = newReader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 2 {
t.Errorf("expected iterator to see 2, saw %d", count)
}
// but that the isolated reader does not
val, err = reader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got %v", val)
}
// and ensure that the iterator on the isolated reader also does not
count = 0
it = reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
func TestGoLevelDBKVCrud(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestKVCrud(t, s)
}
func CommonTestKVStoreIterator(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
data := []struct {
k []byte
v []byte
}{
{[]byte("t\x09\x00paint\xff/sponsor/gold/thumbtack/"), []byte("a")},
{[]byte("t\x09\x00party\xff/sponsor/gold/thumbtack/"), []byte("a")},
{[]byte("t\x09\x00personal\xff/sponsor/gold/thumbtack/"), []byte("a")},
{[]byte("t\x09\x00plan\xff/sponsor/gold/thumbtack/"), []byte("a")},
}
batch := writer.NewBatch()
for _, d := range data {
batch.Set(d.k, d.v)
}
err = batch.Execute()
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("a"))
keys := make([][]byte, 0, len(data))
key, _, valid := it.Current()
for valid {
keys = append(keys, key)
it.Next()
key, _, valid = it.Current()
}
if len(keys) != len(data) {
t.Errorf("expected same number of keys, got %d != %d", len(keys), len(data))
}
for i, dk := range data {
if !reflect.DeepEqual(dk.k, keys[i]) {
t.Errorf("expected key %s got %s", dk.k, keys[i])
}
}
err = it.Close()
if err != nil {
t.Fatal(err)
}
func TestGoLevelDBReaderIsolation(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestReaderIsolation(t, s)
}
func TestGoLevelDBReaderOwnsGetBytes(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestReaderOwnsGetBytes(t, s)
}
func TestGoLevelDBWriterOwnsBytes(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestWriterOwnsBytes(t, s)
}
func TestGoLevelDBPrefixIterator(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestPrefixIterator(t, s)
}
func TestGoLevelDBPrefixIteratorSeek(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestPrefixIteratorSeek(t, s)
}
func TestGoLevelDBRangeIterator(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestRangeIterator(t, s)
}
func TestGoLevelDBRangeIteratorSeek(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestRangeIteratorSeek(t, s)
}
func TestGoLevelDBMerge(t *testing.T) {
s := open(t, &test.TestMergeCounter{})
defer cleanup(t, s)
test.CommonTestMerge(t, s)
}

26
index/store/goleveldb/util.go
View File

@ -1,26 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package goleveldb
import (
"github.com/syndtr/goleveldb/leveldb/opt"
)
func defaultWriteOptions() *opt.WriteOptions {
wo := &opt.WriteOptions{}
// request fsync on write for safety
wo.Sync = true
return wo
}
func defaultReadOptions() *opt.ReadOptions {
ro := &opt.ReadOptions{}
return ro
}

60
index/store/goleveldb/writer.go
View File

@ -10,6 +10,8 @@
package goleveldb
import (
"fmt"
"github.com/blevesearch/bleve/index/store"
"github.com/syndtr/goleveldb/leveldb"
)
@ -18,46 +20,40 @@ type Writer struct {
store *Store
}
func newWriter(store *Store) (*Writer, error) {
store.writer.Lock()
return &Writer{
store: store,
}, nil
}
func (w *Writer) BytesSafeAfterClose() bool {
return false
}
func (w *Writer) Set(key, val []byte) error {
return w.store.setlocked(key, val)
}
func (w *Writer) Delete(key []byte) error {
return w.store.deletelocked(key)
}
func (w *Writer) NewBatch() store.KVBatch {
rv := Batch{
w: w,
store: w.store,
merge: store.NewEmulatedMerge(w.store.mo),
batch: new(leveldb.Batch),
}
return &rv
}
func (w *Writer) ExecuteBatch(b store.KVBatch) error {
batch, ok := b.(*Batch)
if !ok {
return fmt.Errorf("wrong type of batch")
}
// first process merges
for k, mergeOps := range batch.merge.Merges {
kb := []byte(k)
existingVal, err := w.store.db.Get(kb, w.store.defaultReadOptions)
if err != nil && err != leveldb.ErrNotFound {
return err
}
mergedVal, fullMergeOk := w.store.mo.FullMerge(kb, existingVal, mergeOps)
if !fullMergeOk {
return fmt.Errorf("merge operator returned failure")
}
// add the final merge to this batch
batch.batch.Put(kb, mergedVal)
}
// now execute the batch
return w.store.db.Write(batch.batch, w.store.defaultWriteOptions)
}
func (w *Writer) Close() error {
w.store.writer.Unlock()
return nil
}
// these two methods can safely read using the regular
// methods without a read transaction, because we know
// that no one else is writing but us
func (w *Writer) Get(key []byte) ([]byte, error) {
return w.store.get(key)
}
func (w *Writer) Iterator(key []byte) store.KVIterator {
return w.store.iterator(key)
}

259
index/store/gtreap/gtreap_test.go
View File

@ -1,259 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the
// License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an "AS
// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language
// governing permissions and limitations under the License.
package gtreap
import (
"reflect"
"testing"
"github.com/blevesearch/bleve/index/store"
)
func TestGTreapStore(t *testing.T) {
s, err := StoreConstructor(nil)
if err != nil {
t.Fatal(err)
}
CommonTestKVStore(t, s)
}
func TestReaderIsolation(t *testing.T) {
s, err := StoreConstructor(nil)
if err != nil {
t.Fatal(err)
}
CommonTestReaderIsolation(t, s)
}
func CommonTestKVStore(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Set([]byte("z"), []byte("val-z"))
if err != nil {
t.Fatal(err)
}
err = writer.Delete([]byte("z"))
if err != nil {
t.Fatal(err)
}
batch := writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
batch.Set([]byte("c"), []byte("val-c"))
batch.Set([]byte("d"), []byte("val-d"))
batch.Set([]byte("e"), []byte("val-e"))
batch.Set([]byte("f"), []byte("val-f"))
batch.Set([]byte("g"), []byte("val-g"))
batch.Set([]byte("h"), []byte("val-h"))
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = batch.Execute()
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("b"))
key, val, valid := it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "b" {
t.Fatalf("expected key b, got %s", key)
}
if string(val) != "val-b" {
t.Fatalf("expected value val-b, got %s", val)
}
it.Next()
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "c" {
t.Fatalf("expected key c, got %s", key)
}
if string(val) != "val-c" {
t.Fatalf("expected value val-c, got %s", val)
}
it.Seek([]byte("i"))
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "i" {
t.Fatalf("expected key i, got %s", key)
}
if string(val) != "val-i" {
t.Fatalf("expected value val-i, got %s", val)
}
err = it.Close()
if err != nil {
t.Fatal(err)
}
}
func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
// insert a kv pair
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// create an isolated reader
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
// verify that we see the value already inserted
val, err := reader.Get([]byte("a"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-a")) {
t.Errorf("expected val-a, got nil")
}
// verify that an iterator sees it
count := 0
it := reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
// add something after the reader was created
writer, err = s.Writer()
if err != nil {
t.Error(err)
}
valB := []byte("val-b")
err = writer.Set([]byte("b"), valB)
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// ensure that a newer reader sees it
newReader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := newReader.Close()
if err != nil {
t.Fatal(err)
}
}()
val, err = newReader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-b")) {
t.Errorf("expected val-b, got %s", val)
}
// ensure that the director iterator sees it
count = 0
it2 := newReader.Iterator([]byte{0})
defer func() {
err := it2.Close()
if err != nil {
t.Fatal(err)
}
}()
for it2.Valid() {
it2.Next()
count++
}
if count != 2 {
t.Errorf("expected iterator to see 2, saw %d", count)
}
// but that the isolated reader does not
val, err = reader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got %v", val)
}
// and ensure that the iterator on the isolated reader also does not
count = 0
it3 := reader.Iterator([]byte{0})
defer func() {
err := it3.Close()
if err != nil {
t.Fatal(err)
}
}()
for it3.Valid() {
it3.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
}

41
index/store/gtreap/iterator.go
View File

@ -15,6 +15,7 @@
package gtreap
import (
"bytes"
"sync"
"github.com/steveyen/gtreap"
@ -28,22 +29,33 @@ type Iterator struct {
nextCh chan *Item
curr *Item
currOk bool
}
func newIterator(t *gtreap.Treap) *Iterator {
return &Iterator{t: t}
}
func (w *Iterator) SeekFirst() {
min := w.t.Min()
if min != nil {
w.restart(min.(*Item))
} else {
w.restart(nil)
}
prefix []byte
start []byte
end []byte
}
func (w *Iterator) Seek(k []byte) {
if w.start != nil && bytes.Compare(k, w.start) < 0 {
k = w.start
}
if w.prefix != nil && !bytes.HasPrefix(k, w.prefix) {
if bytes.Compare(k, w.prefix) < 0 {
k = w.prefix
} else {
var end []byte
for i := len(w.prefix) - 1; i >= 0; i-- {
c := w.prefix[i]
if c < 0xff {
end = make([]byte, i+1)
copy(end, w.prefix)
end[i] = c + 1
break
}
}
k = end
}
}
w.restart(&Item{k: k})
}
@ -93,6 +105,11 @@ func (w *Iterator) Current() ([]byte, []byte, bool) {
if !w.currOk || w.curr == nil {
return nil, nil, false
}
if w.prefix != nil && !bytes.HasPrefix(w.curr.k, w.prefix) {
return nil, nil, false
} else if w.end != nil && bytes.Compare(w.curr.k, w.end) >= 0 {
return nil, nil, false
}
return w.curr.k, w.curr.v, w.currOk
}

28
index/store/gtreap/reader.go
View File

@ -24,20 +24,34 @@ type Reader struct {
t *gtreap.Treap
}
func (w *Reader) BytesSafeAfterClose() bool {
return false
}
func (w *Reader) Get(k []byte) (v []byte, err error) {
var rv []byte
itm := w.t.Get(&Item{k: k})
if itm != nil {
return itm.(*Item).v, nil
rv = make([]byte, len(itm.(*Item).v))
copy(rv, itm.(*Item).v)
return rv, nil
}
return nil, nil
}
func (w *Reader) Iterator(k []byte) store.KVIterator {
return newIterator(w.t).restart(&Item{k: k})
func (w *Reader) PrefixIterator(k []byte) store.KVIterator {
rv := Iterator{
t: w.t,
prefix: k,
}
rv.restart(&Item{k: k})
return &rv
}
func (w *Reader) RangeIterator(start, end []byte) store.KVIterator {
rv := Iterator{
t: w.t,
start: start,
end: end,
}
rv.restart(&Item{k: start})
return &rv
}
func (w *Reader) Close() error {

57
index/store/gtreap/gtreap.go → index/store/gtreap/store.go
View File

@ -12,36 +12,24 @@
// Package gtreap provides an in-memory implementation of the
// KVStore interfaces using the gtreap balanced-binary treap,
// copy-on-write data structure.
package gtreap
import (
"bytes"
"fmt"
"sync"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
"github.com/steveyen/gtreap"
)
const Name = "gtreap"
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
}
const MAX_CONCURRENT_WRITERS = 1
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
s := &Store{
availableWriters: make(chan bool, MAX_CONCURRENT_WRITERS),
t: gtreap.NewTreap(itemCompare),
}
for i := 0; i < MAX_CONCURRENT_WRITERS; i++ {
s.availableWriters <- true
}
return s, nil
type Store struct {
m sync.Mutex
t *gtreap.Treap
mo store.MergeOperator
}
type Item struct {
@ -53,29 +41,15 @@ func itemCompare(a, b interface{}) int {
return bytes.Compare(a.(*Item).k, b.(*Item).k)
}
type Store struct {
availableWriters chan bool
m sync.Mutex
t *gtreap.Treap
mo store.MergeOperator
}
type Writer struct {
s *Store
}
func (s *Store) Open() error {
return nil
}
func (s *Store) SetMergeOperator(mo store.MergeOperator) {
s.mo = mo
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
rv := Store{
t: gtreap.NewTreap(itemCompare),
mo: mo,
}
return &rv, nil
}
func (s *Store) Close() error {
close(s.availableWriters)
return nil
}
@ -87,10 +61,9 @@ func (s *Store) Reader() (store.KVReader, error) {
}
func (s *Store) Writer() (store.KVWriter, error) {
available, ok := <-s.availableWriters
if !ok || !available {
return nil, fmt.Errorf("no available writers")
}
return &Writer{s: s}, nil
}
func init() {
registry.RegisterKVStore(Name, New)
}

89
index/store/gtreap/store_test.go Normal file
View File

@ -0,0 +1,89 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the
// License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an "AS
// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language
// governing permissions and limitations under the License.
package gtreap
import (
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/index/store/test"
)
func open(t *testing.T, mo store.MergeOperator) store.KVStore {
rv, err := New(mo, nil)
if err != nil {
t.Fatal(err)
}
return rv
}
func cleanup(t *testing.T, s store.KVStore) {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}
func TestGTreapKVCrud(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestKVCrud(t, s)
}
func TestGTreapReaderIsolation(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestReaderIsolation(t, s)
}
func TestGTreapReaderOwnsGetBytes(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestReaderOwnsGetBytes(t, s)
}
func TestGTreapWriterOwnsBytes(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestWriterOwnsBytes(t, s)
}
func TestGTreapPrefixIterator(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestPrefixIterator(t, s)
}
func TestGTreapPrefixIteratorSeek(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestPrefixIteratorSeek(t, s)
}
func TestGTreapRangeIterator(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestRangeIterator(t, s)
}
func TestGTreapRangeIteratorSeek(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestRangeIteratorSeek(t, s)
}
func TestGTreapMerge(t *testing.T) {
s := open(t, &test.TestMergeCounter{})
defer cleanup(t, s)
test.CommonTestMerge(t, s)
}

89
index/store/gtreap/writer.go
View File

@ -15,58 +15,55 @@
package gtreap
import (
"fmt"
"math/rand"
"github.com/blevesearch/bleve/index/store"
)
func (w *Writer) BytesSafeAfterClose() bool {
return false
}
func (w *Writer) Get(k []byte) (v []byte, err error) {
w.s.m.Lock()
t := w.s.t
w.s.m.Unlock()
itm := t.Get(&Item{k: k})
if itm != nil {
return itm.(*Item).v, nil
}
return nil, nil
}
func (w *Writer) Iterator(k []byte) store.KVIterator {
w.s.m.Lock()
t := w.s.t
w.s.m.Unlock()
return newIterator(t).restart(&Item{k: k})
}
func (w *Writer) Close() error {
w.s.availableWriters <- true
w.s = nil
return nil
}
func (w *Writer) Set(k, v []byte) (err error) {
w.s.m.Lock()
w.s.t = w.s.t.Upsert(&Item{k: k, v: v}, rand.Int())
w.s.m.Unlock()
return nil
}
func (w *Writer) Delete(k []byte) (err error) {
w.s.m.Lock()
w.s.t = w.s.t.Delete(&Item{k: k})
w.s.m.Unlock()
return nil
type Writer struct {
s *Store
}
func (w *Writer) NewBatch() store.KVBatch {
return store.NewEmulatedBatch(w, w.s.mo)
return store.NewEmulatedBatch(w.s.mo)
}
func (w *Writer) ExecuteBatch(batch store.KVBatch) error {
emulatedBatch, ok := batch.(*store.EmulatedBatch)
if !ok {
return fmt.Errorf("wrong type of batch")
}
w.s.m.Lock()
for k, mergeOps := range emulatedBatch.Merger.Merges {
kb := []byte(k)
var existingVal []byte
existingItem := w.s.t.Get(&Item{k: kb})
if existingItem != nil {
existingVal = w.s.t.Get(&Item{k: kb}).(*Item).v
}
mergedVal, fullMergeOk := w.s.mo.FullMerge(kb, existingVal, mergeOps)
if !fullMergeOk {
return fmt.Errorf("merge operator returned failure")
}
w.s.t = w.s.t.Upsert(&Item{k: kb, v: mergedVal}, rand.Int())
}
for _, op := range emulatedBatch.Ops {
if op.V != nil {
w.s.t = w.s.t.Upsert(&Item{k: op.K, v: op.V}, rand.Int())
} else {
w.s.t = w.s.t.Delete(&Item{k: op.K})
}
}
w.s.m.Unlock()
return nil
}
func (w *Writer) Close() error {
w.s = nil
return nil
}

70
index/store/inmem/iterator.go
View File

@ -1,70 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package inmem
import (
"github.com/ryszard/goskiplist/skiplist"
)
type Iterator struct {
store *Store
iterator skiplist.Iterator
valid bool
}
func newIterator(store *Store) *Iterator {
rv := Iterator{
store: store,
iterator: store.list.Iterator(),
}
return &rv
}
func (i *Iterator) SeekFirst() {
i.Seek([]byte{0})
}
func (i *Iterator) Seek(k []byte) {
i.valid = i.iterator.Seek(string(k))
}
func (i *Iterator) Next() {
i.valid = i.iterator.Next()
}
func (i *Iterator) Current() ([]byte, []byte, bool) {
if i.valid {
return []byte(i.Key()), []byte(i.Value()), true
}
return nil, nil, false
}
func (i *Iterator) Key() []byte {
if i.valid {
return []byte(i.iterator.Key().(string))
}
return nil
}
func (i *Iterator) Value() []byte {
if i.valid {
return []byte(i.iterator.Value().(string))
}
return nil
}
func (i *Iterator) Valid() bool {
return i.valid
}
func (i *Iterator) Close() error {
i.iterator.Close()
return nil
}

40
index/store/inmem/reader.go
View File

@ -1,40 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package inmem
import (
"github.com/blevesearch/bleve/index/store"
)
type Reader struct {
store *Store
}
func newReader(store *Store) (*Reader, error) {
return &Reader{
store: store,
}, nil
}
func (r *Reader) BytesSafeAfterClose() bool {
return false
}
func (r *Reader) Get(key []byte) ([]byte, error) {
return r.store.get(key)
}
func (r *Reader) Iterator(key []byte) store.KVIterator {
return r.store.iterator(key)
}
func (r *Reader) Close() error {
return nil
}

106
index/store/inmem/store.go
View File

@ -1,106 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package inmem
import (
"sync"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
"github.com/ryszard/goskiplist/skiplist"
)
const Name = "mem"
type Store struct {
list *skiplist.SkipList
writer sync.Mutex
mo store.MergeOperator
}
func New() (*Store, error) {
rv := Store{
list: skiplist.NewStringMap(),
}
return &rv, nil
}
func MustOpen() *Store {
rv := Store{
list: skiplist.NewStringMap(),
}
return &rv
}
func (i *Store) Open() error {
return nil
}
func (i *Store) SetMergeOperator(mo store.MergeOperator) {
i.mo = mo
}
func (i *Store) get(key []byte) ([]byte, error) {
val, ok := i.list.Get(string(key))
if ok {
return []byte(val.(string)), nil
}
return nil, nil
}
func (i *Store) set(key, val []byte) error {
i.writer.Lock()
defer i.writer.Unlock()
return i.setlocked(key, val)
}
func (i *Store) setlocked(key, val []byte) error {
i.list.Set(string(key), string(val))
return nil
}
func (i *Store) delete(key []byte) error {
i.writer.Lock()
defer i.writer.Unlock()
return i.deletelocked(key)
}
func (i *Store) deletelocked(key []byte) error {
i.list.Delete(string(key))
return nil
}
func (i *Store) Close() error {
return nil
}
func (i *Store) iterator(key []byte) store.KVIterator {
rv := newIterator(i)
rv.Seek(key)
return rv
}
func (i *Store) Reader() (store.KVReader, error) {
return newReader(i)
}
func (i *Store) Writer() (store.KVWriter, error) {
return newWriter(i)
}
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
return New()
}
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
}

254
index/store/inmem/store_test.go
View File

@ -1,254 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package inmem
import (
"reflect"
"testing"
"github.com/blevesearch/bleve/index/store"
)
func TestStore(t *testing.T) {
s, err := New()
if err != nil {
t.Fatal(err)
}
defer func() {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}()
CommonTestKVStore(t, s)
}
func CommonTestKVStore(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Set([]byte("z"), []byte("val-z"))
if err != nil {
t.Fatal(err)
}
err = writer.Delete([]byte("z"))
if err != nil {
t.Fatal(err)
}
batch := writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
batch.Set([]byte("c"), []byte("val-c"))
batch.Set([]byte("d"), []byte("val-d"))
batch.Set([]byte("e"), []byte("val-e"))
batch.Set([]byte("f"), []byte("val-f"))
batch.Set([]byte("g"), []byte("val-g"))
batch.Set([]byte("h"), []byte("val-h"))
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = batch.Execute()
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("b"))
key, val, valid := it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "b" {
t.Fatalf("expected key b, got %s", key)
}
if string(val) != "val-b" {
t.Fatalf("expected value val-b, got %s", val)
}
it.Next()
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "c" {
t.Fatalf("expected key c, got %s", key)
}
if string(val) != "val-c" {
t.Fatalf("expected value val-c, got %s", val)
}
it.Seek([]byte("i"))
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "i" {
t.Fatalf("expected key i, got %s", key)
}
if string(val) != "val-i" {
t.Fatalf("expected value val-i, got %s", val)
}
err = it.Close()
if err != nil {
t.Fatal(err)
}
}
func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
// insert a kv pair
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// create an isolated reader
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
// verify that we see the value already inserted
val, err := reader.Get([]byte("a"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-a")) {
t.Errorf("expected val-a, got nil")
}
// verify that an iterator sees it
count := 0
it := reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
// add something after the reader was created
writer, err = s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("b"), []byte("val-b"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// ensure that a newer reader sees it
newReader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := newReader.Close()
if err != nil {
t.Fatal(err)
}
}()
val, err = newReader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-b")) {
t.Errorf("expected val-b, got nil")
}
// ensure that the director iterator sees it
count = 0
it = newReader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 2 {
t.Errorf("expected iterator to see 2, saw %d", count)
}
// but that the isolated reader does not
val, err = reader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got %v", val)
}
// and ensure that the iterator on the isolated reader also does not
count = 0
it = reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
}

57
index/store/inmem/writer.go
View File

@ -1,57 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package inmem
import (
"github.com/blevesearch/bleve/index/store"
)
type Writer struct {
store *Store
}
func newWriter(store *Store) (*Writer, error) {
store.writer.Lock()
return &Writer{
store: store,
}, nil
}
func (w *Writer) BytesSafeAfterClose() bool {
return false
}
func (w *Writer) Set(key, val []byte) error {
return w.store.setlocked(key, val)
}
func (w *Writer) Delete(key []byte) error {
return w.store.deletelocked(key)
}
func (w *Writer) NewBatch() store.KVBatch {
return store.NewEmulatedBatch(w, w.store.mo)
}
func (w *Writer) Close() error {
w.store.writer.Unlock()
return nil
}
// these two methods can safely read using the regular
// methods without a read transaction, because we know
// that no one else is writing but us
func (w *Writer) Get(key []byte) ([]byte, error) {
return w.store.get(key)
}
func (w *Writer) Iterator(key []byte) store.KVIterator {
return w.store.iterator(key)
}

119
index/store/kvstore.go
View File

@ -9,50 +9,113 @@
package store
type KVBatch interface {
Set(key, val []byte)
Delete(key []byte)
Merge(key, val []byte)
Execute() error
// KVStore is an abstraction for working with KV stores
type KVStore interface {
// Writer returns a KVWriter which can be used to
// make changes to the KVStore. If a writer cannot
// be obtained a non-nil error is returned.
Writer() (KVWriter, error)
// Reader returns a KVReader which can be used to
// read data from the KVStore. If a reader cannot
// be obtained a non-nil error is returned.
Reader() (KVReader, error)
// Close closes the KVStore
Close() error
}
// KVReader is an abstraction of an **ISOLATED** reader
// In this context isolated is defined to mean that
// writes/deletes made after the KVReader is opened
// are not observed.
// Because there is usually a cost associated with
// keeping isolated readers active, users should
// close them as soon as they are no longer needed.
type KVReader interface {
// Get returns the value associated with the key
// If the key does not exist, nil is returned.
// The caller owns the bytes returned.
Get(key []byte) ([]byte, error)
// PrefixIterator returns a KVIterator that will
// visit all K/V pairs with the provided prefix
PrefixIterator(prefix []byte) KVIterator
// RangeIterator returns a KVIterator that will
// visit all K/V pairs >= start AND < end
RangeIterator(start, end []byte) KVIterator
// Close closes the iterator
Close() error
}
// KVIterator is an abstraction around key iteration
type KVIterator interface {
SeekFirst()
Seek([]byte)
// Seek will advance the iterator to the specified key
Seek(key []byte)
// Next will advance the iterator to the next key
Next()
Current() ([]byte, []byte, bool)
// Key returns the key pointed to by the iterator
// The bytes returned are **ONLY** valid until the next call to Seek/Next/Close
// Continued use after that requires that they be copied.
Key() []byte
// Value returns the value pointed to by the iterator
// The bytes returned are **ONLY** valid until the next call to Seek/Next/Close
// Continued use after that requires that they be copied.
Value() []byte
// Valid returns whether or not the iterator is in a valid state
Valid() bool
// Current returns Key(),Value(),Valid() in a single operation
Current() ([]byte, []byte, bool)
// Close closes the iterator
Close() error
}
type KVStore interface {
Open() error
SetMergeOperator(MergeOperator)
Writer() (KVWriter, error)
Reader() (KVReader, error)
Close() error
}
// KVWriter is an abstraction for mutating the KVStore
// KVWriter does **NOT** enforce restrictions of a single writer
// if the underlying KVStore allows concurrent writes, the
// KVWriter interface should also do so, it is up to the caller
// to do this in a way that is safe and makes sense
type KVWriter interface {
KVReader
Set(key, val []byte) error
Delete(key []byte) error
NewBatch() KVBatch
}
type KVReader interface {
BytesSafeAfterClose() bool
Get(key []byte) ([]byte, error)
Iterator(key []byte) KVIterator
// NewBatch returns a KVBatch for performaing batch operations on this kvstore
NewBatch() KVBatch
// ExecuteBatch will execute the KVBatch, the provided KVBatch **MUST** have
// been created by the same KVStore (though not necessarily the same KVWriter)
// Batch execution is atomic, either all the operations or none will be performed
ExecuteBatch(batch KVBatch) error
// Close closes the writer
Close() error
}
type RangeIterable interface {
// iterates keys >= start and < end
RangeIterator(start, end []byte) KVIterator
// KVBatch is an abstraction for making multiple KV mutations at once
type KVBatch interface {
// Set updates the key with the specified value
// both key and value []byte may be reused as soon as this call returns
Set(key, val []byte)
// Delete removes the specified key
// the key []byte may be reused as soon as this call returns
Delete(key []byte)
// Merge merges old value with the new value at the specified key
// as prescribed by the KVStores merge operator
// both key and value []byte may be reused as soon as this call returns
Merge(key, val []byte)
// Reset frees resources for this batch and allows reuse
Reset()
}

69
index/store/merge.go
View File

@ -9,10 +9,6 @@
package store
import (
"fmt"
)
// At the moment this happens to be the same interface as described by
// RocksDB, but this may not always be the case.
@ -32,41 +28,20 @@ type MergeOperator interface {
Name() string
}
// EmulatedMergeSingle removes some duplicated code across
// KV stores which do not support merge operations
// on their own. It is up to the caller to ensure
// that an appropriate lock has been acquired in
// order for this behavior to be valid
func EmulatedMergeSingle(writer KVWriter, mo MergeOperator, key []byte, operand []byte) error {
existingValue, err := writer.Get(key)
if err != nil {
return err
}
newValue, ok := mo.FullMerge(key, existingValue, [][]byte{operand})
if !ok {
return fmt.Errorf("merge operator returned failure")
}
err = writer.Set(key, newValue)
if err != nil {
return err
}
return nil
}
type EmulatedMerge struct {
merges map[string][][]byte
Merges map[string][][]byte
mo MergeOperator
}
func NewEmulatedMerge(mo MergeOperator) *EmulatedMerge {
return &EmulatedMerge{
merges: make(map[string][][]byte),
Merges: make(map[string][][]byte),
mo: mo,
}
}
func (m *EmulatedMerge) Merge(key, val []byte) {
ops, ok := m.merges[string(key)]
ops, ok := m.Merges[string(key)]
if ok && len(ops) > 0 {
last := ops[len(ops)-1]
mergedVal, partialMergeOk := m.mo.PartialMerge(key, last, val)
@ -80,41 +55,5 @@ func (m *EmulatedMerge) Merge(key, val []byte) {
} else {
ops = [][]byte{val}
}
m.merges[string(key)] = ops
}
func (m *EmulatedMerge) Execute(w KVWriter) error {
for k, mergeOps := range m.merges {
kb := []byte(k)
existingVal, err := w.Get(kb)
if err != nil {
return err
}
mergedVal, fullMergeOk := m.mo.FullMerge(kb, existingVal, mergeOps)
if !fullMergeOk {
return fmt.Errorf("merge operator returned failure")
}
err = w.Set(kb, mergedVal)
if err != nil {
return err
}
}
return nil
}
func (m *EmulatedMerge) ExecuteDeferred(w KVWriter) ([]*op, error) {
rv := make([]*op, 0, 1000)
for k, mergeOps := range m.merges {
kb := []byte(k)
existingVal, err := w.Get(kb)
if err != nil {
return nil, err
}
mergedVal, fullMergeOk := m.mo.FullMerge(kb, existingVal, mergeOps)
if !fullMergeOk {
return nil, fmt.Errorf("merge operator returned failure")
}
rv = append(rv, &op{kb, mergedVal})
}
return rv, nil
m.Merges[string(key)] = ops
}

26
index/store/metrics/batch.go Normal file
View File

@ -0,0 +1,26 @@
package metrics
import "github.com/blevesearch/bleve/index/store"
type Batch struct {
s *Store
o store.KVBatch
}
func (b *Batch) Set(key, val []byte) {
b.o.Set(key, val)
}
func (b *Batch) Delete(key []byte) {
b.o.Delete(key)
}
func (b *Batch) Merge(key, val []byte) {
b.s.TimerBatchMerge.Time(func() {
b.o.Merge(key, val)
})
}
func (b *Batch) Reset() {
b.o.Reset()
}

44
index/store/metrics/iterator.go Normal file
View File

@ -0,0 +1,44 @@
package metrics
import "github.com/blevesearch/bleve/index/store"
type Iterator struct {
s *Store
o store.KVIterator
}
func (i *Iterator) Seek(x []byte) {
i.s.TimerIteratorSeek.Time(func() {
i.o.Seek(x)
})
}
func (i *Iterator) Next() {
i.s.TimerIteratorNext.Time(func() {
i.o.Next()
})
}
func (i *Iterator) Current() ([]byte, []byte, bool) {
return i.o.Current()
}
func (i *Iterator) Key() []byte {
return i.o.Key()
}
func (i *Iterator) Value() []byte {
return i.o.Value()
}
func (i *Iterator) Valid() bool {
return i.o.Valid()
}
func (i *Iterator) Close() error {
err := i.o.Close()
if err != nil {
i.s.AddError("Iterator.Close", err, nil)
}
return err
}

482
index/store/metrics/metrics.go
View File

@ -1,482 +0,0 @@
// Copyright (c) 2015 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the
// License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an "AS
// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language
// governing permissions and limitations under the License.
// Package metrics provides a bleve.store.KVStore implementation that
// wraps another, real KVStore implementation, and uses go-metrics to
// track runtime performance metrics.
package metrics
import (
"container/list"
"encoding/json"
"fmt"
"io"
"sync"
"time"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
"github.com/rcrowley/go-metrics"
)
const Name = "metrics"
const MaxErrors = 100
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
}
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
name, ok := config["kvStoreName_actual"].(string)
if !ok || name == "" {
return nil, fmt.Errorf("metrics: missing kvStoreName_actual,"+
" config: %#v", config)
}
if name == Name {
return nil, fmt.Errorf("metrics: circular kvStoreName_actual")
}
ctr := registry.KVStoreConstructorByName(name)
if ctr == nil {
return nil, fmt.Errorf("metrics: no kv store constructor,"+
" kvStoreName_actual: %s", name)
}
kvs, err := ctr(config)
if err != nil {
return nil, err
}
return NewBleveMetricsStore(kvs), nil
}
func NewBleveMetricsStore(o store.KVStore) *Store {
return &Store{
o: o,
TimerReaderGet: metrics.NewTimer(),
TimerReaderIterator: metrics.NewTimer(),
TimerWriterGet: metrics.NewTimer(),
TimerWriterIterator: metrics.NewTimer(),
TimerWriterSet: metrics.NewTimer(),
TimerWriterDelete: metrics.NewTimer(),
TimerIteratorSeekFirst: metrics.NewTimer(),
TimerIteratorSeek: metrics.NewTimer(),
TimerIteratorNext: metrics.NewTimer(),
TimerBatchMerge: metrics.NewTimer(),
TimerBatchExecute: metrics.NewTimer(),
errors: list.New(),
}
}
// The following structs are wrappers around "real" bleve kvstore
// implementations.
type Store struct {
o store.KVStore
TimerReaderGet metrics.Timer
TimerReaderIterator metrics.Timer
TimerWriterGet metrics.Timer
TimerWriterIterator metrics.Timer
TimerWriterSet metrics.Timer
TimerWriterDelete metrics.Timer
TimerIteratorSeekFirst metrics.Timer
TimerIteratorSeek metrics.Timer
TimerIteratorNext metrics.Timer
TimerBatchMerge metrics.Timer
TimerBatchExecute metrics.Timer
m sync.Mutex // Protects the fields that follow.
errors *list.List // Capped list of StoreError's.
}
type StoreError struct {
Time string
Op string
Err string
Key string
}
type Reader struct {
s *Store
o store.KVReader
}
type Writer struct {
s *Store
o store.KVWriter
}
type Iterator struct {
s *Store
o store.KVIterator
}
type Batch struct {
s *Store
o store.KVBatch
}
func (s *Store) Open() error {
return s.o.Open()
}
func (s *Store) Close() error {
return s.o.Close()
}
func (s *Store) SetMergeOperator(mo store.MergeOperator) {
s.o.SetMergeOperator(mo)
}
func (s *Store) Reader() (store.KVReader, error) {
o, err := s.o.Reader()
if err != nil {
s.AddError("Reader", err, nil)
return nil, err
}
return &Reader{s: s, o: o}, nil
}
func (s *Store) Writer() (store.KVWriter, error) {
o, err := s.o.Writer()
if err != nil {
s.AddError("Writer", err, nil)
return nil, err
}
return &Writer{s: s, o: o}, nil
}
func (s *Store) Actual() store.KVStore {
return s.o
}
func (w *Reader) BytesSafeAfterClose() bool {
return w.o.BytesSafeAfterClose()
}
func (w *Reader) Get(key []byte) (v []byte, err error) {
w.s.TimerReaderGet.Time(func() {
v, err = w.o.Get(key)
if err != nil {
w.s.AddError("Reader.Get", err, key)
}
})
return
}
func (w *Reader) Iterator(key []byte) (i store.KVIterator) {
w.s.TimerReaderIterator.Time(func() {
i = &Iterator{s: w.s, o: w.o.Iterator(key)}
})
return
}
func (w *Reader) Close() error {
err := w.o.Close()
if err != nil {
w.s.AddError("Reader.Close", err, nil)
}
return err
}
func (w *Writer) BytesSafeAfterClose() bool {
return w.o.BytesSafeAfterClose()
}
func (w *Writer) Get(key []byte) (v []byte, err error) {
w.s.TimerWriterGet.Time(func() {
v, err = w.o.Get(key)
if err != nil {
w.s.AddError("Writer.Get", err, key)
}
})
return
}
func (w *Writer) Iterator(key []byte) (i store.KVIterator) {
w.s.TimerWriterIterator.Time(func() {
i = &Iterator{s: w.s, o: w.o.Iterator(key)}
})
return
}
func (w *Writer) Close() error {
err := w.o.Close()
if err != nil {
w.s.AddError("Writer.Close", err, nil)
}
return err
}
func (w *Writer) Set(key, val []byte) (err error) {
w.s.TimerWriterSet.Time(func() {
err = w.o.Set(key, val)
if err != nil {
w.s.AddError("Writer.Set", err, key)
}
})
return
}
func (w *Writer) Delete(key []byte) (err error) {
w.s.TimerWriterDelete.Time(func() {
err = w.o.Delete(key)
if err != nil {
w.s.AddError("Writer.Delete", err, key)
}
})
return
}
func (w *Writer) NewBatch() store.KVBatch {
return &Batch{s: w.s, o: w.o.NewBatch()}
}
func (w *Iterator) SeekFirst() {
w.s.TimerIteratorSeekFirst.Time(func() {
w.o.SeekFirst()
})
}
func (w *Iterator) Seek(x []byte) {
w.s.TimerIteratorSeek.Time(func() {
w.o.Seek(x)
})
}
func (w *Iterator) Next() {
w.s.TimerIteratorNext.Time(func() {
w.o.Next()
})
}
func (w *Iterator) Current() ([]byte, []byte, bool) {
return w.o.Current()
}
func (w *Iterator) Key() []byte {
return w.o.Key()
}
func (w *Iterator) Value() []byte {
return w.o.Value()
}
func (w *Iterator) Valid() bool {
return w.o.Valid()
}
func (w *Iterator) Close() error {
err := w.o.Close()
if err != nil {
w.s.AddError("Iterator.Close", err, nil)
}
return err
}
func (w *Batch) Set(key, val []byte) {
w.o.Set(key, val)
}
func (w *Batch) Delete(key []byte) {
w.o.Delete(key)
}
func (w *Batch) Merge(key, val []byte) {
w.s.TimerBatchMerge.Time(func() {
w.o.Merge(key, val)
})
}
func (w *Batch) Execute() (err error) {
w.s.TimerBatchExecute.Time(func() {
err = w.o.Execute()
if err != nil {
w.s.AddError("Batch.Execute", err, nil)
}
})
return
}
func (w *Batch) Close() error {
err := w.o.Close()
if err != nil {
w.s.AddError("Batch.Close", err, nil)
}
return err
}
// --------------------------------------------------------
func (s *Store) AddError(op string, err error, key []byte) {
e := &StoreError{
Time: time.Now().Format(time.RFC3339Nano),
Op: op,
Err: fmt.Sprintf("%v", err),
Key: string(key),
}
s.m.Lock()
for s.errors.Len() >= MaxErrors {
s.errors.Remove(s.errors.Front())
}
s.errors.PushBack(e)
s.m.Unlock()
}
// --------------------------------------------------------
func (s *Store) WriteJSON(w io.Writer) {
w.Write([]byte(`{"TimerReaderGet":`))
WriteTimerJSON(w, s.TimerReaderGet)
w.Write([]byte(`,"TimerReaderIterator":`))
WriteTimerJSON(w, s.TimerReaderIterator)
w.Write([]byte(`,"TimerWriterGet":`))
WriteTimerJSON(w, s.TimerWriterGet)
w.Write([]byte(`,"TimerWriterIterator":`))
WriteTimerJSON(w, s.TimerWriterIterator)
w.Write([]byte(`,"TimerWriterSet":`))
WriteTimerJSON(w, s.TimerWriterSet)
w.Write([]byte(`,"TimerWriterDelete":`))
WriteTimerJSON(w, s.TimerWriterDelete)
w.Write([]byte(`,"TimerIteratorSeekFirst":`))
WriteTimerJSON(w, s.TimerIteratorSeekFirst)
w.Write([]byte(`,"TimerIteratorSeek":`))
WriteTimerJSON(w, s.TimerIteratorSeek)
w.Write([]byte(`,"TimerIteratorNext":`))
WriteTimerJSON(w, s.TimerIteratorNext)
w.Write([]byte(`,"TimerBatchMerge":`))
WriteTimerJSON(w, s.TimerBatchMerge)
w.Write([]byte(`,"TimerBatchExecute":`))
WriteTimerJSON(w, s.TimerBatchExecute)
w.Write([]byte(`,"Errors":[`))
s.m.Lock()
e := s.errors.Front()
i := 0
for e != nil {
se, ok := e.Value.(*StoreError)
if ok && se != nil {
if i > 0 {
w.Write([]byte(","))
}
buf, err := json.Marshal(se)
if err == nil {
w.Write(buf)
}
}
e = e.Next()
i = i + 1
}
s.m.Unlock()
w.Write([]byte(`]`))
w.Write([]byte(`}`))
}
func (s *Store) WriteCSVHeader(w io.Writer) {
WriteTimerCSVHeader(w, "TimerReaderGet")
WriteTimerCSVHeader(w, "TimerReaderIterator")
WriteTimerCSVHeader(w, "TimerWriterGet")
WriteTimerCSVHeader(w, "TimerWriterIterator")
WriteTimerCSVHeader(w, "TimerWriterSet")
WriteTimerCSVHeader(w, "TimerWriterDelete")
WriteTimerCSVHeader(w, "TimerIteratorSeekFirst")
WriteTimerCSVHeader(w, "TimerIteratorSeek")
WriteTimerCSVHeader(w, "TimerIteratorNext")
WriteTimerCSVHeader(w, "TimerBatchMerge")
WriteTimerCSVHeader(w, "TimerBatchExecute")
}
func (s *Store) WriteCSV(w io.Writer) {
WriteTimerCSV(w, s.TimerReaderGet)
WriteTimerCSV(w, s.TimerReaderIterator)
WriteTimerCSV(w, s.TimerWriterGet)
WriteTimerCSV(w, s.TimerWriterIterator)
WriteTimerCSV(w, s.TimerWriterSet)
WriteTimerCSV(w, s.TimerWriterDelete)
WriteTimerCSV(w, s.TimerIteratorSeekFirst)
WriteTimerCSV(w, s.TimerIteratorSeek)
WriteTimerCSV(w, s.TimerIteratorNext)
WriteTimerCSV(w, s.TimerBatchMerge)
WriteTimerCSV(w, s.TimerBatchExecute)
}
// --------------------------------------------------------
// NOTE: This is copy & pasted from cbft as otherwise there
// would be an import cycle.
var timerPercentiles = []float64{0.5, 0.75, 0.95, 0.99, 0.999}
func WriteTimerJSON(w io.Writer, timer metrics.Timer) {
t := timer.Snapshot()
p := t.Percentiles(timerPercentiles)
fmt.Fprintf(w, `{"count":%9d,`, t.Count())
fmt.Fprintf(w, `"min":%9d,`, t.Min())
fmt.Fprintf(w, `"max":%9d,`, t.Max())
fmt.Fprintf(w, `"mean":%12.2f,`, t.Mean())
fmt.Fprintf(w, `"stddev":%12.2f,`, t.StdDev())
fmt.Fprintf(w, `"percentiles":{`)
fmt.Fprintf(w, `"median":%12.2f,`, p[0])
fmt.Fprintf(w, `"75%%":%12.2f,`, p[1])
fmt.Fprintf(w, `"95%%":%12.2f,`, p[2])
fmt.Fprintf(w, `"99%%":%12.2f,`, p[3])
fmt.Fprintf(w, `"99.9%%":%12.2f},`, p[4])
fmt.Fprintf(w, `"rates":{`)
fmt.Fprintf(w, `"1-min":%12.2f,`, t.Rate1())
fmt.Fprintf(w, `"5-min":%12.2f,`, t.Rate5())
fmt.Fprintf(w, `"15-min":%12.2f,`, t.Rate15())
fmt.Fprintf(w, `"mean":%12.2f}}`, t.RateMean())
}
func WriteTimerCSVHeader(w io.Writer, prefix string) {
fmt.Fprintf(w, "%s-count,", prefix)
fmt.Fprintf(w, "%s-min,", prefix)
fmt.Fprintf(w, "%s-max,", prefix)
fmt.Fprintf(w, "%s-mean,", prefix)
fmt.Fprintf(w, "%s-stddev,", prefix)
fmt.Fprintf(w, "%s-percentile-50%%,", prefix)
fmt.Fprintf(w, "%s-percentile-75%%,", prefix)
fmt.Fprintf(w, "%s-percentile-95%%,", prefix)
fmt.Fprintf(w, "%s-percentile-99%%,", prefix)
fmt.Fprintf(w, "%s-percentile-99.9%%,", prefix)
fmt.Fprintf(w, "%s-rate-1-min,", prefix)
fmt.Fprintf(w, "%s-rate-5-min,", prefix)
fmt.Fprintf(w, "%s-rate-15-min,", prefix)
fmt.Fprintf(w, "%s-rate-mean", prefix)
}
func WriteTimerCSV(w io.Writer, timer metrics.Timer) {
t := timer.Snapshot()
p := t.Percentiles(timerPercentiles)
fmt.Fprintf(w, `%d,`, t.Count())
fmt.Fprintf(w, `%d,`, t.Min())
fmt.Fprintf(w, `%d,`, t.Max())
fmt.Fprintf(w, `%f,`, t.Mean())
fmt.Fprintf(w, `%f,`, t.StdDev())
fmt.Fprintf(w, `%f,`, p[0])
fmt.Fprintf(w, `%f,`, p[1])
fmt.Fprintf(w, `%f,`, p[2])
fmt.Fprintf(w, `%f,`, p[3])
fmt.Fprintf(w, `%f,`, p[4])
fmt.Fprintf(w, `%f,`, t.Rate1())
fmt.Fprintf(w, `%f,`, t.Rate5())
fmt.Fprintf(w, `%f,`, t.Rate15())
fmt.Fprintf(w, `%f`, t.RateMean())
}

249
index/store/metrics/metrics_test.go
View File

@ -16,35 +16,31 @@ import (
"bytes"
"encoding/json"
"fmt"
"reflect"
"testing"
"github.com/blevesearch/bleve/index/store"
_ "github.com/blevesearch/bleve/index/store/gtreap"
"github.com/blevesearch/bleve/index/store/gtreap"
)
func TestMetricsStore(t *testing.T) {
s, err := StoreConstructor(map[string]interface{}{})
s, err := New(nil, map[string]interface{}{})
if err == nil {
t.Errorf("expected err when bad config")
}
s, err = StoreConstructor(map[string]interface{}{
s, err = New(nil, map[string]interface{}{
"kvStoreName_actual": "some-invalid-kvstore-name",
})
if err == nil {
t.Errorf("expected err when unknown kvStoreName_actual")
}
s, err = StoreConstructor(map[string]interface{}{
"kvStoreName_actual": "gtreap",
s, err = New(nil, map[string]interface{}{
"kvStoreName_actual": gtreap.Name,
})
if err != nil {
t.Fatal(err)
}
CommonTestKVStore(t, s)
b := bytes.NewBuffer(nil)
s.(*Store).WriteJSON(b)
if b.Len() <= 0 {
@ -72,240 +68,9 @@ func TestMetricsStore(t *testing.T) {
}
}
func TestReaderIsolation(t *testing.T) {
s, err := StoreConstructor(map[string]interface{}{
"kvStoreName_actual": "gtreap",
})
if err != nil {
t.Fatal(err)
}
CommonTestReaderIsolation(t, s)
}
func CommonTestKVStore(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Set([]byte("z"), []byte("val-z"))
if err != nil {
t.Fatal(err)
}
err = writer.Delete([]byte("z"))
if err != nil {
t.Fatal(err)
}
batch := writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
batch.Set([]byte("c"), []byte("val-c"))
batch.Set([]byte("d"), []byte("val-d"))
batch.Set([]byte("e"), []byte("val-e"))
batch.Set([]byte("f"), []byte("val-f"))
batch.Set([]byte("g"), []byte("val-g"))
batch.Set([]byte("h"), []byte("val-h"))
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = batch.Execute()
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("b"))
key, val, valid := it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "b" {
t.Fatalf("expected key b, got %s", key)
}
if string(val) != "val-b" {
t.Fatalf("expected value val-b, got %s", val)
}
it.Next()
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "c" {
t.Fatalf("expected key c, got %s", key)
}
if string(val) != "val-c" {
t.Fatalf("expected value val-c, got %s", val)
}
it.Seek([]byte("i"))
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "i" {
t.Fatalf("expected key i, got %s", key)
}
if string(val) != "val-i" {
t.Fatalf("expected value val-i, got %s", val)
}
err = it.Close()
if err != nil {
t.Fatal(err)
}
}
func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
// insert a kv pair
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// create an isolated reader
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
// verify that we see the value already inserted
val, err := reader.Get([]byte("a"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-a")) {
t.Errorf("expected val-a, got nil")
}
// verify that an iterator sees it
count := 0
it := reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
// add something after the reader was created
writer, err = s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("b"), []byte("val-b"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// ensure that a newer reader sees it
newReader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := newReader.Close()
if err != nil {
t.Fatal(err)
}
}()
val, err = newReader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-b")) {
t.Errorf("expected val-b, got nil")
}
// ensure that the director iterator sees it
count = 0
it2 := newReader.Iterator([]byte{0})
defer func() {
err := it2.Close()
if err != nil {
t.Fatal(err)
}
}()
for it2.Valid() {
it2.Next()
count++
}
if count != 2 {
t.Errorf("expected iterator to see 2, saw %d", count)
}
// but that the isolated reader does not
val, err = reader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got %v", val)
}
// and ensure that the iterator on the isolated reader also does not
count = 0
it3 := reader.Iterator([]byte{0})
defer func() {
err := it3.Close()
if err != nil {
t.Fatal(err)
}
}()
for it3.Valid() {
it3.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
}
func TestErrors(t *testing.T) {
s, err := StoreConstructor(map[string]interface{}{
"kvStoreName_actual": "gtreap",
s, err := New(nil, map[string]interface{}{
"kvStoreName_actual": gtreap.Name,
})
if err != nil {
t.Fatal(err)

40
index/store/metrics/reader.go Normal file
View File

@ -0,0 +1,40 @@
package metrics
import "github.com/blevesearch/bleve/index/store"
type Reader struct {
s *Store
o store.KVReader
}
func (r *Reader) Get(key []byte) (v []byte, err error) {
r.s.TimerReaderGet.Time(func() {
v, err = r.o.Get(key)
if err != nil {
r.s.AddError("Reader.Get", err, key)
}
})
return
}
func (r *Reader) PrefixIterator(prefix []byte) (i store.KVIterator) {
r.s.TimerReaderPrefixIterator.Time(func() {
i = &Iterator{s: r.s, o: r.o.PrefixIterator(prefix)}
})
return
}
func (r *Reader) RangeIterator(start, end []byte) (i store.KVIterator) {
r.s.TimerReaderRangeIterator.Time(func() {
i = &Iterator{s: r.s, o: r.o.RangeIterator(start, end)}
})
return
}
func (r *Reader) Close() error {
err := r.o.Close()
if err != nil {
r.s.AddError("Reader.Close", err, nil)
}
return err
}

196
index/store/metrics/store.go Normal file
View File

@ -0,0 +1,196 @@
// Copyright (c) 2015 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the
// License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an "AS
// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language
// governing permissions and limitations under the License.
// Package metrics provides a bleve.store.KVStore implementation that
// wraps another, real KVStore implementation, and uses go-metrics to
// track runtime performance metrics.
package metrics
import (
"container/list"
"encoding/json"
"fmt"
"io"
"sync"
"time"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
"github.com/rcrowley/go-metrics"
)
const Name = "metrics"
type Store struct {
o store.KVStore
TimerReaderGet metrics.Timer
TimerReaderPrefixIterator metrics.Timer
TimerReaderRangeIterator metrics.Timer
TimerWriterExecuteBatch metrics.Timer
TimerIteratorSeek metrics.Timer
TimerIteratorNext metrics.Timer
TimerBatchMerge metrics.Timer
m sync.Mutex // Protects the fields that follow.
errors *list.List // Capped list of StoreError's.
}
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
name, ok := config["kvStoreName_actual"].(string)
if !ok || name == "" {
return nil, fmt.Errorf("metrics: missing kvStoreName_actual,"+
" config: %#v", config)
}
if name == Name {
return nil, fmt.Errorf("metrics: circular kvStoreName_actual")
}
ctr := registry.KVStoreConstructorByName(name)
if ctr == nil {
return nil, fmt.Errorf("metrics: no kv store constructor,"+
" kvStoreName_actual: %s", name)
}
kvs, err := ctr(mo, config)
if err != nil {
return nil, err
}
return &Store{
o: kvs,
TimerReaderGet: metrics.NewTimer(),
TimerReaderPrefixIterator: metrics.NewTimer(),
TimerReaderRangeIterator: metrics.NewTimer(),
TimerWriterExecuteBatch: metrics.NewTimer(),
TimerIteratorSeek: metrics.NewTimer(),
TimerIteratorNext: metrics.NewTimer(),
TimerBatchMerge: metrics.NewTimer(),
errors: list.New(),
}, nil
}
func init() {
registry.RegisterKVStore(Name, New)
}
func (s *Store) Close() error {
return s.o.Close()
}
func (s *Store) Reader() (store.KVReader, error) {
o, err := s.o.Reader()
if err != nil {
s.AddError("Reader", err, nil)
return nil, err
}
return &Reader{s: s, o: o}, nil
}
func (s *Store) Writer() (store.KVWriter, error) {
o, err := s.o.Writer()
if err != nil {
s.AddError("Writer", err, nil)
return nil, err
}
return &Writer{s: s, o: o}, nil
}
// Metric specific code below:
const MaxErrors = 100
type StoreError struct {
Time string
Op string
Err string
Key string
}
func (s *Store) AddError(op string, err error, key []byte) {
e := &StoreError{
Time: time.Now().Format(time.RFC3339Nano),
Op: op,
Err: fmt.Sprintf("%v", err),
Key: string(key),
}
s.m.Lock()
for s.errors.Len() >= MaxErrors {
s.errors.Remove(s.errors.Front())
}
s.errors.PushBack(e)
s.m.Unlock()
}
func (s *Store) WriteJSON(w io.Writer) {
w.Write([]byte(`{"TimerReaderGet":`))
WriteTimerJSON(w, s.TimerReaderGet)
w.Write([]byte(`,"TimerReaderPrefixIterator":`))
WriteTimerJSON(w, s.TimerReaderPrefixIterator)
w.Write([]byte(`,"TimerReaderRangeIterator":`))
WriteTimerJSON(w, s.TimerReaderRangeIterator)
w.Write([]byte(`,"TimerWriterExecuteBatch":`))
WriteTimerJSON(w, s.TimerWriterExecuteBatch)
w.Write([]byte(`,"TimerIteratorSeek":`))
WriteTimerJSON(w, s.TimerIteratorSeek)
w.Write([]byte(`,"TimerIteratorNext":`))
WriteTimerJSON(w, s.TimerIteratorNext)
w.Write([]byte(`,"TimerBatchMerge":`))
WriteTimerJSON(w, s.TimerBatchMerge)
w.Write([]byte(`,"Errors":[`))
s.m.Lock()
e := s.errors.Front()
i := 0
for e != nil {
se, ok := e.Value.(*StoreError)
if ok && se != nil {
if i > 0 {
w.Write([]byte(","))
}
buf, err := json.Marshal(se)
if err == nil {
w.Write(buf)
}
}
e = e.Next()
i = i + 1
}
s.m.Unlock()
w.Write([]byte(`]`))
w.Write([]byte(`}`))
}
func (s *Store) WriteCSVHeader(w io.Writer) {
WriteTimerCSVHeader(w, "TimerReaderGet")
WriteTimerCSVHeader(w, "TimerReaderPrefixIterator")
WriteTimerCSVHeader(w, "TimerReaderRangeIterator")
WriteTimerCSVHeader(w, "TimerWtierExecuteBatch")
WriteTimerCSVHeader(w, "TimerIteratorSeek")
WriteTimerCSVHeader(w, "TimerIteratorNext")
WriteTimerCSVHeader(w, "TimerBatchMerge")
}
func (s *Store) WriteCSV(w io.Writer) {
WriteTimerCSV(w, s.TimerReaderGet)
WriteTimerCSV(w, s.TimerReaderPrefixIterator)
WriteTimerCSV(w, s.TimerReaderRangeIterator)
WriteTimerCSV(w, s.TimerWriterExecuteBatch)
WriteTimerCSV(w, s.TimerIteratorSeek)
WriteTimerCSV(w, s.TimerIteratorNext)
WriteTimerCSV(w, s.TimerBatchMerge)
}

78
index/store/metrics/store_test.go Normal file
View File

@ -0,0 +1,78 @@
package metrics
import (
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/index/store/gtreap"
"github.com/blevesearch/bleve/index/store/test"
)
func open(t *testing.T, mo store.MergeOperator) store.KVStore {
rv, err := New(mo, map[string]interface{}{"kvStoreName_actual": gtreap.Name})
if err != nil {
t.Fatal(err)
}
return rv
}
func cleanup(t *testing.T, s store.KVStore) {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}
func TestMetricsKVCrud(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestKVCrud(t, s)
}
func TestMetricsReaderIsolation(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestReaderIsolation(t, s)
}
func TestMetricsReaderOwnsGetBytes(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestReaderOwnsGetBytes(t, s)
}
func TestMetricsWriterOwnsBytes(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestWriterOwnsBytes(t, s)
}
func TestMetricsPrefixIterator(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestPrefixIterator(t, s)
}
func TestMetricsPrefixIteratorSeek(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestPrefixIteratorSeek(t, s)
}
func TestMetricsRangeIterator(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestRangeIterator(t, s)
}
func TestMetricsRangeIteratorSeek(t *testing.T) {
s := open(t, nil)
defer cleanup(t, s)
test.CommonTestRangeIteratorSeek(t, s)
}
func TestMetricsMerge(t *testing.T) {
s := open(t, &test.TestMergeCounter{})
defer cleanup(t, s)
test.CommonTestMerge(t, s)
}

72
index/store/metrics/util.go Normal file
View File

@ -0,0 +1,72 @@
package metrics
import (
"fmt"
"io"
"github.com/rcrowley/go-metrics"
)
// NOTE: This is copy & pasted from cbft as otherwise there
// would be an import cycle.
var timerPercentiles = []float64{0.5, 0.75, 0.95, 0.99, 0.999}
func WriteTimerJSON(w io.Writer, timer metrics.Timer) {
t := timer.Snapshot()
p := t.Percentiles(timerPercentiles)
fmt.Fprintf(w, `{"count":%9d,`, t.Count())
fmt.Fprintf(w, `"min":%9d,`, t.Min())
fmt.Fprintf(w, `"max":%9d,`, t.Max())
fmt.Fprintf(w, `"mean":%12.2f,`, t.Mean())
fmt.Fprintf(w, `"stddev":%12.2f,`, t.StdDev())
fmt.Fprintf(w, `"percentiles":{`)
fmt.Fprintf(w, `"median":%12.2f,`, p[0])
fmt.Fprintf(w, `"75%%":%12.2f,`, p[1])
fmt.Fprintf(w, `"95%%":%12.2f,`, p[2])
fmt.Fprintf(w, `"99%%":%12.2f,`, p[3])
fmt.Fprintf(w, `"99.9%%":%12.2f},`, p[4])
fmt.Fprintf(w, `"rates":{`)
fmt.Fprintf(w, `"1-min":%12.2f,`, t.Rate1())
fmt.Fprintf(w, `"5-min":%12.2f,`, t.Rate5())
fmt.Fprintf(w, `"15-min":%12.2f,`, t.Rate15())
fmt.Fprintf(w, `"mean":%12.2f}}`, t.RateMean())
}
func WriteTimerCSVHeader(w io.Writer, prefix string) {
fmt.Fprintf(w, "%s-count,", prefix)
fmt.Fprintf(w, "%s-min,", prefix)
fmt.Fprintf(w, "%s-max,", prefix)
fmt.Fprintf(w, "%s-mean,", prefix)
fmt.Fprintf(w, "%s-stddev,", prefix)
fmt.Fprintf(w, "%s-percentile-50%%,", prefix)
fmt.Fprintf(w, "%s-percentile-75%%,", prefix)
fmt.Fprintf(w, "%s-percentile-95%%,", prefix)
fmt.Fprintf(w, "%s-percentile-99%%,", prefix)
fmt.Fprintf(w, "%s-percentile-99.9%%,", prefix)
fmt.Fprintf(w, "%s-rate-1-min,", prefix)
fmt.Fprintf(w, "%s-rate-5-min,", prefix)
fmt.Fprintf(w, "%s-rate-15-min,", prefix)
fmt.Fprintf(w, "%s-rate-mean", prefix)
}
func WriteTimerCSV(w io.Writer, timer metrics.Timer) {
t := timer.Snapshot()
p := t.Percentiles(timerPercentiles)
fmt.Fprintf(w, `%d,`, t.Count())
fmt.Fprintf(w, `%d,`, t.Min())
fmt.Fprintf(w, `%d,`, t.Max())
fmt.Fprintf(w, `%f,`, t.Mean())
fmt.Fprintf(w, `%f,`, t.StdDev())
fmt.Fprintf(w, `%f,`, p[0])
fmt.Fprintf(w, `%f,`, p[1])
fmt.Fprintf(w, `%f,`, p[2])
fmt.Fprintf(w, `%f,`, p[3])
fmt.Fprintf(w, `%f,`, p[4])
fmt.Fprintf(w, `%f,`, t.Rate1())
fmt.Fprintf(w, `%f,`, t.Rate5())
fmt.Fprintf(w, `%f,`, t.Rate15())
fmt.Fprintf(w, `%f`, t.RateMean())
}

38
index/store/metrics/writer.go Normal file
View File

@ -0,0 +1,38 @@
package metrics
import (
"fmt"
"github.com/blevesearch/bleve/index/store"
)
type Writer struct {
s *Store
o store.KVWriter
}
func (w *Writer) Close() error {
err := w.o.Close()
if err != nil {
w.s.AddError("Writer.Close", err, nil)
}
return err
}
func (w *Writer) NewBatch() store.KVBatch {
return &Batch{s: w.s, o: w.o.NewBatch()}
}
func (w *Writer) ExecuteBatch(b store.KVBatch) (err error) {
batch, ok := b.(*Batch)
if !ok {
return fmt.Errorf("wrong type of batch")
}
w.s.TimerWriterExecuteBatch.Time(func() {
err = w.o.ExecuteBatch(batch.o)
if err != nil {
w.s.AddError("Writer.ExecuteBatch", err, nil)
}
})
return
}

150
index/store/null/null.go
View File

@ -18,167 +18,87 @@ const Name = "null"
type Store struct{}
func New() (*Store, error) {
rv := Store{}
return &rv, nil
}
func (i *Store) Open() error {
return nil
}
func (i *Store) SetMergeOperator(mo store.MergeOperator) {
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
return &Store{}, nil
}
func (i *Store) Close() error {
return nil
}
func (i *Store) iterator(key []byte) store.KVIterator {
rv := newIterator(i)
return rv
}
func (i *Store) Reader() (store.KVReader, error) {
return newReader(i)
return &reader{}, nil
}
func (i *Store) Writer() (store.KVWriter, error) {
return newWriter(i)
return &writer{}, nil
}
func (i *Store) newBatch() store.KVBatch {
return newBatch(i)
}
type reader struct{}
type Reader struct {
store *Store
}
func newReader(store *Store) (*Reader, error) {
return &Reader{
store: store,
}, nil
}
func (r *Reader) BytesSafeAfterClose() bool {
return true
}
func (r *Reader) Get(key []byte) ([]byte, error) {
func (r *reader) Get(key []byte) ([]byte, error) {
return nil, nil
}
func (r *Reader) Iterator(key []byte) store.KVIterator {
return r.store.iterator(key)
func (r *reader) PrefixIterator(prefix []byte) store.KVIterator {
return &iterator{}
}
func (r *Reader) Close() error {
func (r *reader) RangeIterator(start, end []byte) store.KVIterator {
return &iterator{}
}
func (r *reader) Close() error {
return nil
}
type Iterator struct{}
type iterator struct{}
func newIterator(store *Store) *Iterator {
return &Iterator{}
}
func (i *iterator) SeekFirst() {}
func (i *iterator) Seek(k []byte) {}
func (i *iterator) Next() {}
func (i *Iterator) SeekFirst() {}
func (i *Iterator) Seek(k []byte) {}
func (i *Iterator) Next() {}
func (i *Iterator) Current() ([]byte, []byte, bool) {
func (i *iterator) Current() ([]byte, []byte, bool) {
return nil, nil, false
}
func (i *Iterator) Key() []byte {
func (i *iterator) Key() []byte {
return nil
}
func (i *Iterator) Value() []byte {
func (i *iterator) Value() []byte {
return nil
}
func (i *Iterator) Valid() bool {
func (i *iterator) Valid() bool {
return false
}
func (i *Iterator) Close() error {
func (i *iterator) Close() error {
return nil
}
type Batch struct{}
type batch struct{}
func newBatch(s *Store) *Batch {
rv := Batch{}
return &rv
func (i *batch) Set(key, val []byte) {}
func (i *batch) Delete(key []byte) {}
func (i *batch) Merge(key, val []byte) {}
func (i *batch) Reset() {}
type writer struct{}
func (w *writer) NewBatch() store.KVBatch {
return &batch{}
}
func (i *Batch) Set(key, val []byte) {
}
func (i *Batch) Delete(key []byte) {
}
func (i *Batch) Merge(key, val []byte) {
}
func (i *Batch) Execute() error {
func (w *writer) ExecuteBatch(store.KVBatch) error {
return nil
}
func (i *Batch) Close() error {
func (w *writer) Close() error {
return nil
}
type Writer struct {
store *Store
}
func newWriter(store *Store) (*Writer, error) {
return &Writer{
store: store,
}, nil
}
func (w *Writer) BytesSafeAfterClose() bool {
return true
}
func (w *Writer) Set(key, val []byte) error {
return nil
}
func (w *Writer) Delete(key []byte) error {
return nil
}
func (w *Writer) NewBatch() store.KVBatch {
return newBatch(w.store)
}
func (w *Writer) Close() error {
return nil
}
// these two methods can safely read using the regular
// methods without a read transaction, because we know
// that no one else is writing but us
func (w *Writer) Get(key []byte) ([]byte, error) {
return nil, nil
}
func (w *Writer) Iterator(key []byte) store.KVIterator {
return w.store.iterator(key)
}
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
return New()
}
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
registry.RegisterKVStore(Name, New)
}

24
index/store/null/null_test.go
View File

@ -7,32 +7,22 @@ import (
)
func TestStore(t *testing.T) {
s, err := New()
s, err := New(nil, nil)
if err != nil {
t.Fatal(err)
}
CommonTestKVStore(t, s)
NullTestKVStore(t, s)
}
func CommonTestKVStore(t *testing.T, s store.KVStore) {
// NullTestKVStore has very different expectations
// compared to CommonTestKVStore
func NullTestKVStore(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Set([]byte("z"), []byte("val-z"))
if err != nil {
t.Fatal(err)
}
err = writer.Delete([]byte("z"))
if err != nil {
t.Fatal(err)
}
batch := writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
@ -45,7 +35,7 @@ func CommonTestKVStore(t *testing.T, s store.KVStore) {
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = batch.Execute()
err = writer.ExecuteBatch(batch)
if err != nil {
t.Fatal(err)
}
@ -64,7 +54,7 @@ func CommonTestKVStore(t *testing.T, s store.KVStore) {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("b"))
it := reader.RangeIterator([]byte("b"), nil)
key, val, valid := it.Current()
if valid {
t.Fatalf("valid true, expected false")

11
index/store/test/README.md Normal file
View File

@ -0,0 +1,11 @@
# Generic KVStore implementation tests
These are a set of common tests that should pass on any correct KVStore implementation.
Each test function in this package has the form:
func CommonTest<name>(t *testing.T, s store.KVStore) {...}
A KVStore implementation test should use the same name, including its own KVStore name in the test function. It should instantiate an instance of the store, and pass the testing.T and store to the common function.
The common test functions should *NOT* close the KVStore. The KVStore test implementation should close the store and cleanup any state.

262
index/store/test/bytes.go Normal file
View File

@ -0,0 +1,262 @@
package test
import (
"bytes"
"reflect"
"testing"
"github.com/blevesearch/bleve/index/store"
)
// tests which focus on the byte ownership
// CommonTestReaderOwnsGetBytes attempts to mutate the returned bytes
// first, while the reader is still open, second after that reader is
// closed, then the original key is read again, to ensure these
// modifications did not cause panic, or mutate the stored value
func CommonTestReaderOwnsGetBytes(t *testing.T, s store.KVStore) {
originalKey := []byte("key")
originalVal := []byte("val")
// open a writer
writer, err := s.Writer()
if err != nil {
t.Fatal(err)
}
// write key/val
batch := writer.NewBatch()
batch.Set(originalKey, originalVal)
err = writer.ExecuteBatch(batch)
if err != nil {
t.Fatal(err)
}
// close the writer
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// open a reader
reader, err := s.Reader()
if err != nil {
t.Fatal(err)
}
// read key
returnedVal, err := reader.Get(originalKey)
if err != nil {
t.Fatal(err)
}
// check that it is the expected value
if !reflect.DeepEqual(returnedVal, originalVal) {
t.Fatalf("expected value: %v for '%s', got %v", originalVal, originalKey, returnedVal)
}
// mutate the returned value with reader still open
for i := range returnedVal {
returnedVal[i] = '1'
}
// read the key again
returnedVal2, err := reader.Get(originalKey)
if err != nil {
t.Fatal(err)
}
// check that it is the expected value
if !reflect.DeepEqual(returnedVal2, originalVal) {
t.Fatalf("expected value: %v for '%s', got %v", originalVal, originalKey, returnedVal2)
}
// close the reader
err = reader.Close()
if err != nil {
t.Fatal(err)
}
// mutate the original returned value again
for i := range returnedVal {
returnedVal[i] = '2'
}
// open another reader
reader, err = s.Reader()
if err != nil {
t.Fatal(err)
}
// read the key again
returnedVal3, err := reader.Get(originalKey)
if err != nil {
t.Fatal(err)
}
// check that it is the expected value
if !reflect.DeepEqual(returnedVal3, originalVal) {
t.Fatalf("expected value: %v for '%s', got %v", originalVal, originalKey, returnedVal3)
}
// close the reader
err = reader.Close()
if err != nil {
t.Fatal(err)
}
// finally check that the value we mutated still has what we set it to
for i := range returnedVal {
if returnedVal[i] != '2' {
t.Errorf("expected byte to be '2', got %v", returnedVal[i])
}
}
}
func CommonTestWriterOwnsBytes(t *testing.T, s store.KVStore) {
keyBuffer := make([]byte, 5)
valBuffer := make([]byte, 5)
// open a writer
writer, err := s.Writer()
if err != nil {
t.Fatal(err)
}
// write key/val pairs reusing same buffer
batch := writer.NewBatch()
for i := 0; i < 10; i++ {
keyBuffer[0] = 'k'
keyBuffer[1] = 'e'
keyBuffer[2] = 'y'
keyBuffer[3] = '-'
keyBuffer[4] = byte('0' + i)
valBuffer[0] = 'v'
valBuffer[1] = 'a'
valBuffer[2] = 'l'
valBuffer[3] = '-'
valBuffer[4] = byte('0' + i)
batch.Set(keyBuffer, valBuffer)
}
err = writer.ExecuteBatch(batch)
if err != nil {
t.Fatal(err)
}
// close the writer
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// open a reader
reader, err := s.Reader()
if err != nil {
t.Fatal(err)
}
// check that we can read back what we expect
allks := make([][]byte, 0)
allvs := make([][]byte, 0)
iter := reader.RangeIterator(nil, nil)
for iter.Valid() {
// if we want to keep bytes from iteration we must copy
k := iter.Key()
copyk := make([]byte, len(k))
copy(copyk, k)
allks = append(allks, copyk)
v := iter.Key()
copyv := make([]byte, len(v))
copy(copyv, v)
allvs = append(allvs, copyv)
iter.Next()
}
err = iter.Close()
if err != nil {
t.Fatal(err)
}
if len(allks) != 10 {
t.Fatalf("expected 10 k/v pairs, got %d", len(allks))
}
for i, key := range allks {
val := allvs[i]
if !bytes.HasSuffix(key, []byte{byte('0' + i)}) {
t.Errorf("expected key %v to end in %d", key, []byte{byte('0' + i)})
}
if !bytes.HasSuffix(val, []byte{byte('0' + i)}) {
t.Errorf("expected val %v to end in %d", val, []byte{byte('0' + i)})
}
}
// close the reader
err = reader.Close()
if err != nil {
t.Fatal(err)
}
// open a writer
writer, err = s.Writer()
if err != nil {
t.Fatal(err)
}
// now delete using same approach
batch = writer.NewBatch()
for i := 0; i < 10; i++ {
keyBuffer[0] = 'k'
keyBuffer[1] = 'e'
keyBuffer[2] = 'y'
keyBuffer[3] = '-'
keyBuffer[4] = byte('0' + i)
batch.Delete(keyBuffer)
}
err = writer.ExecuteBatch(batch)
if err != nil {
t.Fatal(err)
}
// close the writer
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// open a reader
reader, err = s.Reader()
if err != nil {
t.Fatal(err)
}
// check that we can read back what we expect
allks = make([][]byte, 0)
iter = reader.RangeIterator(nil, nil)
for iter.Valid() {
// if we want to keep bytes from iteration we must copy
k := iter.Key()
copyk := make([]byte, len(k))
copy(copyk, k)
allks = append(allks, copyk)
v := iter.Key()
copyv := make([]byte, len(v))
copy(copyv, v)
allvs = append(allvs, copyv)
iter.Next()
}
err = iter.Close()
if err != nil {
t.Fatal(err)
}
if len(allks) != 0 {
t.Fatalf("expected 0 k/v pairs remaining, got %d", len(allks))
}
// close the reader
err = reader.Close()
if err != nil {
t.Fatal(err)
}
}

98
index/store/test/crud.go Normal file
View File

@ -0,0 +1,98 @@
package test
import (
"testing"
"github.com/blevesearch/bleve/index/store"
)
// basic crud tests
func CommonTestKVCrud(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
batch := writer.NewBatch()
batch.Set([]byte("a"), []byte("val-a"))
batch.Set([]byte("z"), []byte("val-z"))
batch.Delete([]byte("z"))
err = writer.ExecuteBatch(batch)
if err != nil {
t.Fatal(err)
}
batch.Reset()
batch.Set([]byte("b"), []byte("val-b"))
batch.Set([]byte("c"), []byte("val-c"))
batch.Set([]byte("d"), []byte("val-d"))
batch.Set([]byte("e"), []byte("val-e"))
batch.Set([]byte("f"), []byte("val-f"))
batch.Set([]byte("g"), []byte("val-g"))
batch.Set([]byte("h"), []byte("val-h"))
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = writer.ExecuteBatch(batch)
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.RangeIterator([]byte("b"), nil)
key, val, valid := it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "b" {
t.Fatalf("expected key b, got %s", key)
}
if string(val) != "val-b" {
t.Fatalf("expected value val-b, got %s", val)
}
it.Next()
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "c" {
t.Fatalf("expected key c, got %s", key)
}
if string(val) != "val-c" {
t.Fatalf("expected value val-c, got %s", val)
}
it.Seek([]byte("i"))
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "i" {
t.Fatalf("expected key i, got %s", key)
}
if string(val) != "val-i" {
t.Fatalf("expected value val-i, got %s", val)
}
err = it.Close()
if err != nil {
t.Fatal(err)
}
}

177
index/store/test/isolation.go Normal file
View File

@ -0,0 +1,177 @@
package test
import (
"fmt"
"reflect"
"testing"
"github.com/blevesearch/bleve/index/store"
)
// tests focused on verfiying that readers are isolated from writers
func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
// insert a kv pair
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
// **************************************************
// this is a hack only required for BoltDB
// however its harmless so to keep the tests
// the same everywhere, we include it here
//
// this is a hack to try to pre-emptively overflow
// boltdb writes *MAY* block a long reader
// in particular, if the write requires additional
// allocation, it must acquire the same lock as
// the reader, thus cannot continue until that
// reader is closed.
// in general this is not a problem for bleve
// (though it may affect performance in some cases)
// but it is a problem for this test which attemps
// to easily verify that readers are isolated
// this hack writes enough initial data such that
// the subsequent writes do not require additional
// space
hackSize := 1000
batch := writer.NewBatch()
for i := 0; i < hackSize; i++ {
k := fmt.Sprintf("x%d", i)
batch.Set([]byte(k), []byte("filler"))
}
err = writer.ExecuteBatch(batch)
if err != nil {
t.Fatal(err)
}
// **************************************************
batch = writer.NewBatch()
batch.Set([]byte("a"), []byte("val-a"))
err = writer.ExecuteBatch(batch)
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// create an isolated reader
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
// verify that we see the value already inserted
val, err := reader.Get([]byte("a"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-a")) {
t.Errorf("expected val-a, got nil")
}
// verify that an iterator sees it
count := 0
it := reader.RangeIterator([]byte{0}, []byte{'x'})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
// add something after the reader was created
writer, err = s.Writer()
if err != nil {
t.Error(err)
}
batch = writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
err = writer.ExecuteBatch(batch)
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// ensure that a newer reader sees it
newReader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := newReader.Close()
if err != nil {
t.Fatal(err)
}
}()
val, err = newReader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-b")) {
t.Errorf("expected val-b, got nil")
}
// ensure that the direct iterator sees it
count = 0
it2 := newReader.RangeIterator([]byte{0}, []byte{'x'})
defer func() {
err := it2.Close()
if err != nil {
t.Fatal(err)
}
}()
for it2.Valid() {
it2.Next()
count++
}
if count != 2 {
t.Errorf("expected iterator to see 2, saw %d", count)
}
// but that the isolated reader does not
val, err = reader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got %v", val)
}
// and ensure that the iterator on the isolated reader also does not
count = 0
it3 := reader.RangeIterator([]byte{0}, []byte{'x'})
defer func() {
err := it3.Close()
if err != nil {
t.Fatal(err)
}
}()
for it3.Valid() {
it3.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
}

424
index/store/test/iterator.go Normal file
View File

@ -0,0 +1,424 @@
package test
import (
"bytes"
"reflect"
"strings"
"testing"
"github.com/blevesearch/bleve/index/store"
)
// tests around the correct behavior of iterators
type testRow struct {
key []byte
val []byte
}
func batchWriteRows(s store.KVStore, rows []testRow) error {
// open a writer
writer, err := s.Writer()
if err != nil {
return err
}
// write the data
batch := writer.NewBatch()
for _, row := range rows {
batch.Set(row.key, row.val)
}
err = writer.ExecuteBatch(batch)
if err != nil {
return err
}
// close the writer
err = writer.Close()
if err != nil {
return err
}
return nil
}
func CommonTestPrefixIterator(t *testing.T, s store.KVStore) {
data := []testRow{
{[]byte("apple"), []byte("val")},
{[]byte("cat1"), []byte("val")},
{[]byte("cat2"), []byte("val")},
{[]byte("cat3"), []byte("val")},
{[]byte("dog1"), []byte("val")},
{[]byte("dog2"), []byte("val")},
{[]byte("dog4"), []byte("val")},
{[]byte("elephant"), []byte("val")},
}
expectedCats := [][]byte{
[]byte("cat1"),
[]byte("cat2"),
[]byte("cat3"),
}
expectedDogs := [][]byte{
[]byte("dog1"),
// we seek to "dog3" and ensure it skips over "dog2"
// but still finds "dog4" even though there was no "dog3"
[]byte("dog4"),
}
err := batchWriteRows(s, data)
if err != nil {
t.Fatal(err)
}
// open a reader
reader, err := s.Reader()
if err != nil {
t.Fatal(err)
}
// get a prefix reader
cats := make([][]byte, 0)
iter := reader.PrefixIterator([]byte("cat"))
for iter.Valid() {
// if we want to keep bytes from iteration we must copy
k := iter.Key()
copyk := make([]byte, len(k))
copy(copyk, k)
cats = append(cats, copyk)
iter.Next()
}
err = iter.Close()
if err != nil {
t.Fatal(err)
}
// check that we found all the cats
if !reflect.DeepEqual(cats, expectedCats) {
t.Fatalf("expected cats %v, got %v", expectedCats, cats)
}
// get a prefix reader
dogs := make([][]byte, 0)
iter = reader.PrefixIterator([]byte("dog"))
for iter.Valid() {
// if we want to keep bytes from iteration we must copy
k := iter.Key()
copyk := make([]byte, len(k))
copy(copyk, k)
dogs = append(dogs, copyk)
if len(dogs) < 2 {
iter.Seek([]byte("dog3"))
} else {
iter.Next()
}
}
err = iter.Close()
if err != nil {
t.Fatal(err)
}
// check that we found the expected dogs
if !reflect.DeepEqual(dogs, expectedDogs) {
t.Fatalf("expected dogs %v, got %v", expectedDogs, dogs)
}
// close the reader
err = reader.Close()
if err != nil {
t.Fatal(err)
}
}
func CommonTestPrefixIteratorSeek(t *testing.T, s store.KVStore) {
data := []testRow{
{[]byte("a"), []byte("val")},
{[]byte("b1"), []byte("val")},
{[]byte("b2"), []byte("val")},
{[]byte("b3"), []byte("val")},
{[]byte("c"), []byte("val")},
}
err := batchWriteRows(s, data)
if err != nil {
t.Fatal(err)
}
// open a reader
reader, err := s.Reader()
if err != nil {
t.Fatal(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
// get an iterator on a central subset of the data
iter := reader.PrefixIterator([]byte("b"))
defer func() {
err := iter.Close()
if err != nil {
t.Fatal(err)
}
}()
// check that all keys have prefix
found := []string{}
for ; iter.Valid(); iter.Next() {
found = append(found, string(iter.Key()))
}
for _, f := range found {
if !strings.HasPrefix(f, "b") {
t.Errorf("got key '%s' that doesn't have correct prefix", f)
}
}
if len(found) != 3 {
t.Errorf("expected 3 keys with prefix, got %d", len(found))
}
// now try to seek before the prefix and repeat
found = []string{}
for iter.Seek([]byte("a")); iter.Valid(); iter.Next() {
found = append(found, string(iter.Key()))
}
for _, f := range found {
if !strings.HasPrefix(f, "b") {
t.Errorf("got key '%s' that doesn't have correct prefix", f)
}
}
if len(found) != 3 {
t.Errorf("expected 3 keys with prefix, got %d", len(found))
}
// now try to seek after the prefix and repeat
found = []string{}
for iter.Seek([]byte("c")); iter.Valid(); iter.Next() {
found = append(found, string(iter.Key()))
}
for _, f := range found {
if !strings.HasPrefix(f, "b") {
t.Errorf("got key '%s' that doesn't have correct prefix", f)
}
}
if len(found) != 0 {
t.Errorf("expected 0 keys with prefix, got %d", len(found))
}
}
func CommonTestRangeIterator(t *testing.T, s store.KVStore) {
data := []testRow{
{[]byte("a1"), []byte("val")},
{[]byte("b1"), []byte("val")},
{[]byte("b2"), []byte("val")},
{[]byte("b3"), []byte("val")},
{[]byte("c1"), []byte("val")},
{[]byte("c2"), []byte("val")},
{[]byte("c4"), []byte("val")},
{[]byte("d1"), []byte("val")},
}
expectedAll := make([][]byte, 0)
expectedBToC := make([][]byte, 0)
expectedCToDSeek3 := make([][]byte, 0)
expectedCToEnd := make([][]byte, 0)
for _, row := range data {
expectedAll = append(expectedAll, row.key)
if bytes.HasPrefix(row.key, []byte("b")) {
expectedBToC = append(expectedBToC, row.key)
}
if bytes.HasPrefix(row.key, []byte("c")) && !bytes.HasSuffix(row.key, []byte("2")) {
expectedCToDSeek3 = append(expectedCToDSeek3, row.key)
}
if bytes.Compare(row.key, []byte("c")) > 0 {
expectedCToEnd = append(expectedCToEnd, row.key)
}
}
err := batchWriteRows(s, data)
if err != nil {
t.Fatal(err)
}
// open a reader
reader, err := s.Reader()
if err != nil {
t.Fatal(err)
}
// get a range iterator (all)
all := make([][]byte, 0)
iter := reader.RangeIterator(nil, nil)
for iter.Valid() {
// if we want to keep bytes from iteration we must copy
k := iter.Key()
copyk := make([]byte, len(k))
copy(copyk, k)
all = append(all, copyk)
iter.Next()
}
err = iter.Close()
if err != nil {
t.Fatal(err)
}
// check that we found all
if !reflect.DeepEqual(all, expectedAll) {
t.Fatalf("expected all %v, got %v", expectedAll, all)
}
// get range iterator from b - c
bToC := make([][]byte, 0)
iter = reader.RangeIterator([]byte("b"), []byte("c"))
for iter.Valid() {
// if we want to keep bytes from iteration we must copy
k := iter.Key()
copyk := make([]byte, len(k))
copy(copyk, k)
bToC = append(bToC, copyk)
iter.Next()
}
err = iter.Close()
if err != nil {
t.Fatal(err)
}
// check that we found b-c
if !reflect.DeepEqual(bToC, expectedBToC) {
t.Fatalf("expected b-c %v, got %v", expectedBToC, bToC)
}
// get range iterator from c - d, but seek to 'c3'
cToDSeek3 := make([][]byte, 0)
iter = reader.RangeIterator([]byte("c"), []byte("d"))
for iter.Valid() {
// if we want to keep bytes from iteration we must copy
k := iter.Key()
copyk := make([]byte, len(k))
copy(copyk, k)
cToDSeek3 = append(cToDSeek3, copyk)
if len(cToDSeek3) < 2 {
iter.Seek([]byte("c3"))
} else {
iter.Next()
}
}
err = iter.Close()
if err != nil {
t.Fatal(err)
}
// check that we found c-d with seek to c3
if !reflect.DeepEqual(cToDSeek3, expectedCToDSeek3) {
t.Fatalf("expected b-c %v, got %v", expectedCToDSeek3, cToDSeek3)
}
// get range iterator from c to the end
cToEnd := make([][]byte, 0)
iter = reader.RangeIterator([]byte("c"), nil)
for iter.Valid() {
// if we want to keep bytes from iteration we must copy
k := iter.Key()
copyk := make([]byte, len(k))
copy(copyk, k)
cToEnd = append(cToEnd, copyk)
iter.Next()
}
err = iter.Close()
if err != nil {
t.Fatal(err)
}
// check that we found c to end
if !reflect.DeepEqual(cToEnd, expectedCToEnd) {
t.Fatalf("expected b-c %v, got %v", expectedCToEnd, cToEnd)
}
// close the reader
err = reader.Close()
if err != nil {
t.Fatal(err)
}
}
func CommonTestRangeIteratorSeek(t *testing.T, s store.KVStore) {
data := []testRow{
{[]byte("a1"), []byte("val")},
{[]byte("b1"), []byte("val")},
{[]byte("c1"), []byte("val")},
{[]byte("d1"), []byte("val")},
{[]byte("e1"), []byte("val")},
}
err := batchWriteRows(s, data)
if err != nil {
t.Fatal(err)
}
// open a reader
reader, err := s.Reader()
if err != nil {
t.Fatal(err)
}
// get an iterator on a central subset of the data
start := []byte("b1")
end := []byte("d1")
iter := reader.RangeIterator(start, end)
// seek before, at and after every possible key
targets := [][]byte{}
for _, row := range data {
prefix := string(row.key[:1])
targets = append(targets, []byte(prefix+"0"))
targets = append(targets, []byte(prefix+"1"))
targets = append(targets, []byte(prefix+"2"))
}
for _, target := range targets {
found := []string{}
for iter.Seek(target); iter.Valid(); iter.Next() {
found = append(found, string(iter.Key()))
if len(found) > len(data) {
t.Fatalf("enumerated more than data keys after seeking to %s",
string(target))
}
}
wanted := []string{}
for _, row := range data {
if bytes.Compare(row.key, start) < 0 ||
bytes.Compare(row.key, target) < 0 ||
bytes.Compare(row.key, end) >= 0 {
continue
}
wanted = append(wanted, string(row.key))
}
fs := strings.Join(found, ", ")
ws := strings.Join(wanted, ", ")
if fs != ws {
t.Fatalf("iterating from %s returned [%s] instead of [%s]",
string(target), fs, ws)
}
}
err = iter.Close()
if err != nil {
t.Fatal(err)
}
if err != nil {
t.Fatal(err)
}
// close the reader
err = reader.Close()
if err != nil {
t.Fatal(err)
}
}

108
index/store/test/merge.go Normal file
View File

@ -0,0 +1,108 @@
package test
import (
"encoding/binary"
"testing"
"github.com/blevesearch/bleve/index/store"
)
// test merge behavior
func encodeUint64(in uint64) []byte {
rv := make([]byte, 8)
binary.LittleEndian.PutUint64(rv, in)
return rv
}
func CommonTestMerge(t *testing.T, s store.KVStore) {
testKey := []byte("k1")
data := []struct {
key []byte
val []byte
}{
{testKey, encodeUint64(1)},
{testKey, encodeUint64(1)},
}
// open a writer
writer, err := s.Writer()
if err != nil {
t.Fatal(err)
}
// write the data
batch := writer.NewBatch()
for _, row := range data {
batch.Merge(row.key, row.val)
}
err = writer.ExecuteBatch(batch)
if err != nil {
t.Fatal(err)
}
// close the writer
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// open a reader
reader, err := s.Reader()
if err != nil {
t.Fatal(err)
}
// read key
returnedVal, err := reader.Get(testKey)
if err != nil {
t.Fatal(err)
}
// check the value
mergedval := binary.LittleEndian.Uint64(returnedVal)
if mergedval != 2 {
t.Errorf("expected 2, got %d", mergedval)
}
// close the reader
err = reader.Close()
if err != nil {
t.Fatal(err)
}
}
// a test merge operator which is just an incrementing counter of uint64
type TestMergeCounter struct{}
func (mc *TestMergeCounter) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) {
var newval uint64
if len(existingValue) > 0 {
newval = binary.LittleEndian.Uint64(existingValue)
}
// now process operands
for _, operand := range operands {
next := binary.LittleEndian.Uint64(operand)
newval += next
}
rv := make([]byte, 8)
binary.LittleEndian.PutUint64(rv, newval)
return rv, true
}
func (mc *TestMergeCounter) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) {
left := binary.LittleEndian.Uint64(leftOperand)
right := binary.LittleEndian.Uint64(rightOperand)
rv := make([]byte, 8)
binary.LittleEndian.PutUint64(rv, left+right)
return rv, true
}
func (mc *TestMergeCounter) Name() string {
return "test_merge_counter"
}

5
index/upside_down/analysis_test.go
View File

@ -18,12 +18,11 @@ func BenchmarkAnalyze(b *testing.B) {
b.Fatal(err)
}
s, err := null.New()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(null.Name, nil, analysisQueue)
if err != nil {
b.Fatal(err)
}
analysisQueue := index.NewAnalysisQueue(1)
idx := NewUpsideDownCouch(s, analysisQueue)
d := document.NewDocument("1")
f := document.NewTextFieldWithAnalyzer("desc", nil, bleveWikiArticle1K, analyzer)

35
index/upside_down/benchmark_boltdb_test.go
View File

@ -10,68 +10,61 @@
package upside_down
import (
"os"
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/index/store/boltdb"
)
func CreateBoltDB() (store.KVStore, error) {
s := boltdb.New("test", "bleve")
return s, nil
}
func DestroyBoltDB() error {
return os.RemoveAll("test")
var boltTestConfig = map[string]interface{}{
"path": "test",
}
func BenchmarkBoltDBIndexing1Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateBoltDB, DestroyBoltDB, 1)
CommonBenchmarkIndex(b, boltdb.Name, boltTestConfig, DestroyTest, 1)
}
func BenchmarkBoltDBIndexing2Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateBoltDB, DestroyBoltDB, 2)
CommonBenchmarkIndex(b, boltdb.Name, boltTestConfig, DestroyTest, 2)
}
func BenchmarkBoltDBIndexing4Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateBoltDB, DestroyBoltDB, 4)
CommonBenchmarkIndex(b, boltdb.Name, boltTestConfig, DestroyTest, 4)
}
// batches
func BenchmarkBoltDBIndexing1Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 1, 10)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 1, 10)
}
func BenchmarkBoltDBIndexing2Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 2, 10)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 2, 10)
}
func BenchmarkBoltDBIndexing4Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 4, 10)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 4, 10)
}
func BenchmarkBoltDBIndexing1Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 1, 100)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 1, 100)
}
func BenchmarkBoltDBIndexing2Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 2, 100)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 2, 100)
}
func BenchmarkBoltDBIndexing4Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 4, 100)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 4, 100)
}
func BenchmarkBoltBIndexing1Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 1, 1000)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 1, 1000)
}
func BenchmarkBoltBIndexing2Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 2, 1000)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 2, 1000)
}
func BenchmarkBoltBIndexing4Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 4, 1000)
CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 4, 1000)
}

21
index/upside_down/benchmark_common_test.go
View File

@ -10,13 +10,13 @@
package upside_down
import (
"os"
"strconv"
"testing"
_ "github.com/blevesearch/bleve/analysis/analyzers/standard_analyzer"
"github.com/blevesearch/bleve/document"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
)
@ -33,10 +33,13 @@ var benchmarkDocBodies = []string{
"The expansion ratio of a liquefied and cryogenic substance is the volume of a given amount of that substance in liquid form compared to the volume of the same amount of substance in gaseous form, at room temperature and normal atmospheric pressure.",
}
type KVStoreCreate func() (store.KVStore, error)
type KVStoreDestroy func() error
func CommonBenchmarkIndex(b *testing.B, create KVStoreCreate, destroy KVStoreDestroy, analysisWorkers int) {
func DestroyTest() error {
return os.RemoveAll("test")
}
func CommonBenchmarkIndex(b *testing.B, storeName string, storeConfig map[string]interface{}, destroy KVStoreDestroy, analysisWorkers int) {
cache := registry.NewCache()
analyzer, err := cache.AnalyzerNamed("standard")
@ -50,12 +53,11 @@ func CommonBenchmarkIndex(b *testing.B, create KVStoreCreate, destroy KVStoreDes
b.ResetTimer()
b.StopTimer()
for i := 0; i < b.N; i++ {
s, err := create()
analysisQueue := index.NewAnalysisQueue(analysisWorkers)
idx, err := NewUpsideDownCouch(storeName, storeConfig, analysisQueue)
if err != nil {
b.Fatal(err)
}
analysisQueue := index.NewAnalysisQueue(analysisWorkers)
idx := NewUpsideDownCouch(s, analysisQueue)
err = idx.Open()
if err != nil {
@ -81,7 +83,7 @@ func CommonBenchmarkIndex(b *testing.B, create KVStoreCreate, destroy KVStoreDes
}
}
func CommonBenchmarkIndexBatch(b *testing.B, create KVStoreCreate, destroy KVStoreDestroy, analysisWorkers, batchSize int) {
func CommonBenchmarkIndexBatch(b *testing.B, storeName string, storeConfig map[string]interface{}, destroy KVStoreDestroy, analysisWorkers, batchSize int) {
cache := registry.NewCache()
analyzer, err := cache.AnalyzerNamed("standard")
@ -93,12 +95,11 @@ func CommonBenchmarkIndexBatch(b *testing.B, create KVStoreCreate, destroy KVSto
b.StopTimer()
for i := 0; i < b.N; i++ {
s, err := create()
analysisQueue := index.NewAnalysisQueue(analysisWorkers)
idx, err := NewUpsideDownCouch(storeName, storeConfig, analysisQueue)
if err != nil {
b.Fatal(err)
}
analysisQueue := index.NewAnalysisQueue(analysisWorkers)
idx := NewUpsideDownCouch(s, analysisQueue)
err = idx.Open()
if err != nil {

170
index/upside_down/benchmark_forestdb_test.go
View File

@ -15,72 +15,194 @@ import (
"os"
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/blevex/forestdb"
)
func CreateForestDB() (store.KVStore, error) {
err := os.MkdirAll("testdir", 0700)
if err != nil {
return nil, err
}
s, err := forestdb.New("testdir/test", true, nil)
if err != nil {
return nil, err
}
return s, nil
var forestDBTestOption = map[string]interface{}{
"path": "testdir/test",
"create_if_missing": true,
}
// internally used to reset, so we also
// re-make the testdir
func DestroyForestDB() error {
return os.RemoveAll("testdir")
err := os.RemoveAll("testdir")
if err != nil {
return err
}
err = os.MkdirAll("testdir", 0700)
if err != nil {
return err
}
return nil
}
func BenchmarkForestDBIndexing1Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateForestDB, DestroyForestDB, 1)
err := os.MkdirAll("testdir", 0700)
if err != nil {
b.Fatal(err)
}
defer func() {
err := os.RemoveAll("testdir")
if err != nil {
b.Fatal(err)
}
}()
CommonBenchmarkIndex(b, forestdb.Name, forestDBTestOption, DestroyForestDB, 1)
}
func BenchmarkForestDBIndexing2Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateForestDB, DestroyForestDB, 2)
err := os.MkdirAll("testdir", 0700)
if err != nil {
b.Fatal(err)
}
defer func() {
err := os.RemoveAll("testdir")
if err != nil {
b.Fatal(err)
}
}()
CommonBenchmarkIndex(b, forestdb.Name, forestDBTestOption, DestroyForestDB, 2)
}
func BenchmarkForestDBIndexing4Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateForestDB, DestroyForestDB, 4)
err := os.MkdirAll("testdir", 0700)
if err != nil {
b.Fatal(err)
}
defer func() {
err := os.RemoveAll("testdir")
if err != nil {
b.Fatal(err)
}
}()
CommonBenchmarkIndex(b, forestdb.Name, forestDBTestOption, DestroyForestDB, 4)
}
// batches
func BenchmarkForestDBIndexing1Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 1, 10)
err := os.MkdirAll("testdir", 0700)
if err != nil {
b.Fatal(err)
}
defer func() {
err := os.RemoveAll("testdir")
if err != nil {
b.Fatal(err)
}
}()
CommonBenchmarkIndexBatch(b, forestdb.Name, forestDBTestOption, DestroyForestDB, 1, 10)
}
func BenchmarkForestDBIndexing2Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 2, 10)
err := os.MkdirAll("testdir", 0700)
if err != nil {
b.Fatal(err)
}
defer func() {
err := os.RemoveAll("testdir")
if err != nil {
b.Fatal(err)
}
}()
CommonBenchmarkIndexBatch(b, forestdb.Name, forestDBTestOption, DestroyForestDB, 2, 10)
}
func BenchmarkForestDBIndexing4Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 4, 10)
err := os.MkdirAll("testdir", 0700)
if err != nil {
b.Fatal(err)
}
defer func() {
err := os.RemoveAll("testdir")
if err != nil {
b.Fatal(err)
}
}()
CommonBenchmarkIndexBatch(b, forestdb.Name, forestDBTestOption, DestroyForestDB, 4, 10)
}
func BenchmarkForestDBIndexing1Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 1, 100)
err := os.MkdirAll("testdir", 0700)
if err != nil {
b.Fatal(err)
}
defer func() {
err := os.RemoveAll("testdir")
if err != nil {
b.Fatal(err)
}
}()
CommonBenchmarkIndexBatch(b, forestdb.Name, forestDBTestOption, DestroyForestDB, 1, 100)
}
func BenchmarkForestDBIndexing2Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 2, 100)
err := os.MkdirAll("testdir", 0700)
if err != nil {
b.Fatal(err)
}
defer func() {
err := os.RemoveAll("testdir")
if err != nil {
b.Fatal(err)
}
}()
CommonBenchmarkIndexBatch(b, forestdb.Name, forestDBTestOption, DestroyForestDB, 2, 100)
}
func BenchmarkForestDBIndexing4Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 4, 100)
err := os.MkdirAll("testdir", 0700)
if err != nil {
b.Fatal(err)
}
defer func() {
err := os.RemoveAll("testdir")
if err != nil {
b.Fatal(err)
}
}()
CommonBenchmarkIndexBatch(b, forestdb.Name, forestDBTestOption, DestroyForestDB, 4, 100)
}
func BenchmarkForestDBIndexing1Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 1, 1000)
err := os.MkdirAll("testdir", 0700)
if err != nil {
b.Fatal(err)
}
defer func() {
err := os.RemoveAll("testdir")
if err != nil {
b.Fatal(err)
}
}()
CommonBenchmarkIndexBatch(b, forestdb.Name, forestDBTestOption, DestroyForestDB, 1, 1000)
}
func BenchmarkForestDBIndexing2Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 2, 1000)
err := os.MkdirAll("testdir", 0700)
if err != nil {
b.Fatal(err)
}
defer func() {
err := os.RemoveAll("testdir")
if err != nil {
b.Fatal(err)
}
}()
CommonBenchmarkIndexBatch(b, forestdb.Name, forestDBTestOption, DestroyForestDB, 2, 1000)
}
func BenchmarkForestDBIndexing4Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 4, 1000)
err := os.MkdirAll("testdir", 0700)
if err != nil {
b.Fatal(err)
}
defer func() {
err := os.RemoveAll("testdir")
if err != nil {
b.Fatal(err)
}
}()
CommonBenchmarkIndexBatch(b, forestdb.Name, forestDBTestOption, DestroyForestDB, 4, 1000)
}

35
index/upside_down/benchmark_goleveldb_test.go
View File

@ -10,71 +10,62 @@
package upside_down
import (
"os"
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/index/store/goleveldb"
)
var goLevelDBTestOptions = map[string]interface{}{
"create_if_missing": true,
}
func CreateGoLevelDB() (store.KVStore, error) {
return goleveldb.New("test", goLevelDBTestOptions)
}
func DestroyGoLevelDB() error {
return os.RemoveAll("test")
"path": "test",
}
func BenchmarkGoLevelDBIndexing1Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateGoLevelDB, DestroyGoLevelDB, 1)
CommonBenchmarkIndex(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 1)
}
func BenchmarkGoLevelDBIndexing2Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateGoLevelDB, DestroyGoLevelDB, 2)
CommonBenchmarkIndex(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 2)
}
func BenchmarkGoLevelDBIndexing4Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateGoLevelDB, DestroyGoLevelDB, 4)
CommonBenchmarkIndex(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 4)
}
// batches
func BenchmarkGoLevelDBIndexing1Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 1, 10)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 1, 10)
}
func BenchmarkGoLevelDBIndexing2Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 2, 10)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 2, 10)
}
func BenchmarkGoLevelDBIndexing4Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 4, 10)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 4, 10)
}
func BenchmarkGoLevelDBIndexing1Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 1, 100)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 1, 100)
}
func BenchmarkGoLevelDBIndexing2Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 2, 100)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 2, 100)
}
func BenchmarkGoLevelDBIndexing4Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 4, 100)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 4, 100)
}
func BenchmarkGoLevelDBIndexing1Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 1, 1000)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 1, 1000)
}
func BenchmarkGoLevelDBIndexing2Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 2, 1000)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 2, 1000)
}
func BenchmarkGoLevelDBIndexing4Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 4, 1000)
CommonBenchmarkIndexBatch(b, goleveldb.Name, goLevelDBTestOptions, DestroyTest, 4, 1000)
}

35
index/upside_down/benchmark_gorocksdb_test.go
View File

@ -12,71 +12,62 @@
package upside_down
import (
"os"
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/blevex/rocksdb"
)
var rocksdbTestOptions = map[string]interface{}{
"path": "test",
"create_if_missing": true,
}
func CreateGoRocksDB() (store.KVStore, error) {
return rocksdb.New("test", rocksdbTestOptions)
}
func DestroyGoRocksDB() error {
return os.RemoveAll("test")
}
func BenchmarkRocksDBIndexing1Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateGoRocksDB, DestroyGoRocksDB, 1)
CommonBenchmarkIndex(b, rocksdb.Name, rocksdbTestOptions, DestroyTest, 1)
}
func BenchmarkRocksDBIndexing2Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateGoRocksDB, DestroyGoRocksDB, 2)
CommonBenchmarkIndex(b, rocksdb.Name, rocksdbTestOptions, DestroyTest, 2)
}
func BenchmarkRocksDBIndexing4Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateGoRocksDB, DestroyGoRocksDB, 4)
CommonBenchmarkIndex(b, rocksdb.Name, rocksdbTestOptions, DestroyTest, 4)
}
// batches
func BenchmarkRocksDBIndexing1Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 1, 10)
CommonBenchmarkIndexBatch(b, rocksdb.Name, rocksdbTestOptions, DestroyTest, 1, 10)
}
func BenchmarkRocksDBIndexing2Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 2, 10)
CommonBenchmarkIndexBatch(b, rocksdb.Name, rocksdbTestOptions, DestroyTest, 2, 10)
}
func BenchmarkRocksDBIndexing4Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 4, 10)
CommonBenchmarkIndexBatch(b, rocksdb.Name, rocksdbTestOptions, DestroyTest, 4, 10)
}
func BenchmarkRocksDBIndexing1Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 1, 100)
CommonBenchmarkIndexBatch(b, rocksdb.Name, rocksdbTestOptions, DestroyTest, 1, 100)
}
func BenchmarkRocksDBIndexing2Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 2, 100)
CommonBenchmarkIndexBatch(b, rocksdb.Name, rocksdbTestOptions, DestroyTest, 2, 100)
}
func BenchmarkRocksDBIndexing4Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 4, 100)
CommonBenchmarkIndexBatch(b, rocksdb.Name, rocksdbTestOptions, DestroyTest, 4, 100)
}
func BenchmarkRocksDBIndexing1Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 1, 1000)
CommonBenchmarkIndexBatch(b, rocksdb.Name, rocksdbTestOptions, DestroyTest, 1, 1000)
}
func BenchmarkRocksDBIndexing2Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 2, 1000)
CommonBenchmarkIndexBatch(b, rocksdb.Name, rocksdbTestOptions, DestroyTest, 2, 1000)
}
func BenchmarkRocksDBIndexing4Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 4, 1000)
CommonBenchmarkIndexBatch(b, rocksdb.Name, rocksdbTestOptions, DestroyTest, 4, 1000)
}

33
index/upside_down/benchmark_gtreap_test.go
View File

@ -12,64 +12,55 @@ package upside_down
import (
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/index/store/gtreap"
)
func CreateGTreap() (store.KVStore, error) {
return gtreap.StoreConstructor(nil)
}
func DestroyGTreap() error {
return nil
}
func BenchmarkGTreapIndexing1Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateGTreap, DestroyGTreap, 1)
CommonBenchmarkIndex(b, gtreap.Name, nil, DestroyTest, 1)
}
func BenchmarkGTreapIndexing2Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateGTreap, DestroyGTreap, 2)
CommonBenchmarkIndex(b, gtreap.Name, nil, DestroyTest, 2)
}
func BenchmarkGTreapIndexing4Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateGTreap, DestroyGTreap, 4)
CommonBenchmarkIndex(b, gtreap.Name, nil, DestroyTest, 4)
}
// batches
func BenchmarkGTreapIndexing1Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 1, 10)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 1, 10)
}
func BenchmarkGTreapIndexing2Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 2, 10)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 2, 10)
}
func BenchmarkGTreapIndexing4Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 4, 10)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 4, 10)
}
func BenchmarkGTreapIndexing1Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 1, 100)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 1, 100)
}
func BenchmarkGTreapIndexing2Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 2, 100)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 2, 100)
}
func BenchmarkGTreapIndexing4Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 4, 100)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 4, 100)
}
func BenchmarkGTreapIndexing1Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 1, 1000)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 1, 1000)
}
func BenchmarkGTreapIndexing2Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 2, 1000)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 2, 1000)
}
func BenchmarkGTreapIndexing4Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 4, 1000)
CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 4, 1000)
}

75
index/upside_down/benchmark_inmem_test.go
View File

@ -1,75 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package upside_down
import (
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/index/store/inmem"
)
func CreateInMem() (store.KVStore, error) {
return inmem.New()
}
func DestroyInMem() error {
return nil
}
func BenchmarkInMemIndexing1Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateInMem, DestroyInMem, 1)
}
func BenchmarkInMemIndexing2Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateInMem, DestroyInMem, 2)
}
func BenchmarkInMemIndexing4Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateInMem, DestroyInMem, 4)
}
// batches
func BenchmarkInMemIndexing1Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 1, 10)
}
func BenchmarkInMemIndexing2Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 2, 10)
}
func BenchmarkInMemIndexing4Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 4, 10)
}
func BenchmarkInMemIndexing1Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 1, 100)
}
func BenchmarkInMemIndexing2Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 2, 100)
}
func BenchmarkInMemIndexing4Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 4, 100)
}
func BenchmarkInMemIndexing1Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 1, 1000)
}
func BenchmarkInMemIndexing2Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 2, 1000)
}
func BenchmarkInMemIndexing4Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 4, 1000)
}

35
index/upside_down/benchmark_leveldb_test.go
View File

@ -12,71 +12,62 @@
package upside_down
import (
"os"
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/blevex/leveldb"
)
var leveldbTestOptions = map[string]interface{}{
"path": "test",
"create_if_missing": true,
}
func CreateLevelDB() (store.KVStore, error) {
return leveldb.New("test", leveldbTestOptions)
}
func DestroyLevelDB() error {
return os.RemoveAll("test")
}
func BenchmarkLevelDBIndexing1Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateLevelDB, DestroyLevelDB, 1)
CommonBenchmarkIndex(b, leveldb.Name, leveldbTestOptions, DestroyTest, 1)
}
func BenchmarkLevelDBIndexing2Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateLevelDB, DestroyLevelDB, 2)
CommonBenchmarkIndex(b, leveldb.Name, leveldbTestOptions, DestroyTest, 2)
}
func BenchmarkLevelDBIndexing4Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateLevelDB, DestroyLevelDB, 4)
CommonBenchmarkIndex(b, leveldb.Name, leveldbTestOptions, DestroyTest, 4)
}
// batches
func BenchmarkLevelDBIndexing1Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 1, 10)
CommonBenchmarkIndexBatch(b, leveldb.Name, leveldbTestOptions, DestroyTest, 1, 10)
}
func BenchmarkLevelDBIndexing2Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 2, 10)
CommonBenchmarkIndexBatch(b, leveldb.Name, leveldbTestOptions, DestroyTest, 2, 10)
}
func BenchmarkLevelDBIndexing4Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 4, 10)
CommonBenchmarkIndexBatch(b, leveldb.Name, leveldbTestOptions, DestroyTest, 4, 10)
}
func BenchmarkLevelDBIndexing1Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 1, 100)
CommonBenchmarkIndexBatch(b, leveldb.Name, leveldbTestOptions, DestroyTest, 1, 100)
}
func BenchmarkLevelDBIndexing2Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 2, 100)
CommonBenchmarkIndexBatch(b, leveldb.Name, leveldbTestOptions, DestroyTest, 2, 100)
}
func BenchmarkLevelDBIndexing4Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 4, 100)
CommonBenchmarkIndexBatch(b, leveldb.Name, leveldbTestOptions, DestroyTest, 4, 100)
}
func BenchmarkLevelDBIndexing1Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 1, 1000)
CommonBenchmarkIndexBatch(b, leveldb.Name, leveldbTestOptions, DestroyTest, 1, 1000)
}
func BenchmarkLevelDBIndexing2Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 2, 1000)
CommonBenchmarkIndexBatch(b, leveldb.Name, leveldbTestOptions, DestroyTest, 2, 1000)
}
func BenchmarkLevelDBIndexing4Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 4, 1000)
CommonBenchmarkIndexBatch(b, leveldb.Name, leveldbTestOptions, DestroyTest, 4, 1000)
}

33
index/upside_down/benchmark_null_test.go
View File

@ -12,64 +12,55 @@ package upside_down
import (
"testing"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/index/store/null"
)
func CreateNull() (store.KVStore, error) {
return null.New()
}
func DestroyNull() error {
return nil
}
func BenchmarkNullIndexing1Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateNull, DestroyNull, 1)
CommonBenchmarkIndex(b, null.Name, nil, DestroyTest, 1)
}
func BenchmarkNullIndexing2Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateNull, DestroyNull, 2)
CommonBenchmarkIndex(b, null.Name, nil, DestroyTest, 2)
}
func BenchmarkNullIndexing4Workers(b *testing.B) {
CommonBenchmarkIndex(b, CreateNull, DestroyNull, 4)
CommonBenchmarkIndex(b, null.Name, nil, DestroyTest, 4)
}
// batches
func BenchmarkNullIndexing1Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 1, 10)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 1, 10)
}
func BenchmarkNullIndexing2Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 2, 10)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 2, 10)
}
func BenchmarkNullIndexing4Workers10Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 4, 10)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 4, 10)
}
func BenchmarkNullIndexing1Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 1, 100)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 1, 100)
}
func BenchmarkNullIndexing2Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 2, 100)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 2, 100)
}
func BenchmarkNullIndexing4Workers100Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 4, 100)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 4, 100)
}
func BenchmarkNullIndexing1Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 1, 1000)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 1, 1000)
}
func BenchmarkNullIndexing2Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 2, 1000)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 2, 1000)
}
func BenchmarkNullIndexing4Workers1000Batch(b *testing.B) {
CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 4, 1000)
CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 4, 1000)
}

29
index/upside_down/dump.go
View File

@ -26,7 +26,7 @@ func (udc *UpsideDownCouch) dumpPrefix(kvreader store.KVReader, rv chan interfac
if start == nil {
start = []byte{0}
}
it := kvreader.Iterator(start)
it := kvreader.PrefixIterator(start)
defer func() {
cerr := it.Close()
if cerr != nil {
@ -36,9 +36,28 @@ func (udc *UpsideDownCouch) dumpPrefix(kvreader store.KVReader, rv chan interfac
key, val, valid := it.Current()
for valid {
if prefix != nil && !bytes.HasPrefix(key, prefix) {
break
row, err := ParseFromKeyValue(key, val)
if err != nil {
rv <- err
return
}
rv <- row
it.Next()
key, val, valid = it.Current()
}
}
func (udc *UpsideDownCouch) dumpRange(kvreader store.KVReader, rv chan interface{}, start, end []byte) {
it := kvreader.RangeIterator(start, end)
defer func() {
cerr := it.Close()
if cerr != nil {
rv <- cerr
}
}()
key, val, valid := it.Current()
for valid {
row, err := ParseFromKeyValue(key, val)
if err != nil {
@ -70,7 +89,7 @@ func (udc *UpsideDownCouch) DumpAll() chan interface{} {
}
}()
udc.dumpPrefix(kvreader, rv, nil)
udc.dumpRange(kvreader, rv, nil, nil)
}()
return rv
}
@ -149,7 +168,7 @@ func (udc *UpsideDownCouch) DumpDoc(id string) chan interface{} {
// now walk term keys in order and add them as well
if len(keys) > 0 {
it := kvreader.Iterator(keys[0])
it := kvreader.RangeIterator(keys[0], nil)
defer func() {
cerr := it.Close()
if cerr != nil {

12
index/upside_down/dump_test.go
View File

@ -10,7 +10,6 @@
package upside_down
import (
"os"
"testing"
"time"
@ -22,17 +21,18 @@ import (
func TestDump(t *testing.T) {
defer func() {
err := os.RemoveAll("test")
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
s := boltdb.New("test", "bleve")
s.SetMergeOperator(&mergeOperator)
analysisQueue := index.NewAnalysisQueue(1)
idx := NewUpsideDownCouch(s, analysisQueue)
err := idx.Open()
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}

Some files were not shown because too many files have changed in this diff Show More