2015-08-25 20:52:42 +02:00
|
|
|
// Copyright (c) 2015 Couchbase, Inc.
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
|
|
|
|
// except in compliance with the License. You may obtain a copy of the License at
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
// Unless required by applicable law or agreed to in writing, software distributed under the
|
|
|
|
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
|
|
|
// either express or implied. See the License for the specific language governing permissions
|
|
|
|
// and limitations under the License.
|
|
|
|
|
|
|
|
package firestorm
|
|
|
|
|
|
|
|
import (
|
|
|
|
"math/rand"
|
|
|
|
"reflect"
|
|
|
|
"testing"
|
|
|
|
|
|
|
|
"github.com/blevesearch/bleve/index"
|
2015-10-28 16:26:01 +01:00
|
|
|
"github.com/blevesearch/bleve/index/store/gtreap"
|
2015-08-25 20:52:42 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
func TestTermReaderNoGarbage(t *testing.T) {
|
2015-10-28 16:26:01 +01:00
|
|
|
aq := index.NewAnalysisQueue(1)
|
|
|
|
f, err := NewFirestorm(gtreap.Name, nil, aq)
|
2015-08-25 20:52:42 +02:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2015-10-28 16:26:01 +01:00
|
|
|
err = f.Open()
|
2015-08-25 20:52:42 +02:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2015-10-28 16:26:01 +01:00
|
|
|
kvwriter, err := f.(*Firestorm).store.Writer()
|
2015-08-25 20:52:42 +02:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
rows := []index.IndexRow{
|
|
|
|
NewFieldRow(0, IDFieldName),
|
|
|
|
NewFieldRow(1, "desc"),
|
|
|
|
NewTermFreqRow(0, nil, []byte("a"), 1, 0, 0.0, nil),
|
|
|
|
NewTermFreqRow(0, nil, []byte("b"), 2, 0, 0.0, nil),
|
|
|
|
NewTermFreqRow(0, nil, []byte("c"), 3, 0, 0.0, nil),
|
|
|
|
NewTermFreqRow(1, []byte("cat"), []byte("a"), 1, 3, 2.0, nil),
|
|
|
|
NewTermFreqRow(1, []byte("cat"), []byte("c"), 3, 1, 1.0, nil),
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, row := range rows {
|
2015-10-28 16:26:01 +01:00
|
|
|
wb := kvwriter.NewBatch()
|
|
|
|
wb.Set(row.Key(), row.Value())
|
|
|
|
err = kvwriter.ExecuteBatch(wb)
|
2015-08-25 20:52:42 +02:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-10-28 16:26:01 +01:00
|
|
|
err = kvwriter.Close()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
kvreader, err := f.(*Firestorm).store.Reader()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2015-08-25 20:52:42 +02:00
|
|
|
// warmup to load field cache and set maxRead correctly
|
2015-11-24 20:32:33 +01:00
|
|
|
err = f.(*Firestorm).warmup(kvreader)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2015-10-28 16:26:01 +01:00
|
|
|
|
|
|
|
err = kvreader.Close()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2015-08-25 20:52:42 +02:00
|
|
|
|
|
|
|
r, err := f.Reader()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
tfr, err := r.TermFieldReader([]byte("cat"), "desc")
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
expectedDocIds := []string{"a", "c"}
|
|
|
|
foundDocIds := make([]string, 0)
|
|
|
|
next, err := tfr.Next()
|
|
|
|
for next != nil && err == nil {
|
|
|
|
foundDocIds = append(foundDocIds, next.ID)
|
|
|
|
next, err = tfr.Next()
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if !reflect.DeepEqual(expectedDocIds, foundDocIds) {
|
|
|
|
t.Errorf("expected: %v, got %v", expectedDocIds, foundDocIds)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = tfr.Close()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = r.Close()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestTermReaderSomeGarbage(t *testing.T) {
|
2015-10-28 16:26:01 +01:00
|
|
|
aq := index.NewAnalysisQueue(1)
|
|
|
|
f, err := NewFirestorm(gtreap.Name, nil, aq)
|
2015-08-25 20:52:42 +02:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2015-10-28 16:26:01 +01:00
|
|
|
err = f.Open()
|
2015-08-25 20:52:42 +02:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2015-10-28 16:26:01 +01:00
|
|
|
kvwriter, err := f.(*Firestorm).store.Writer()
|
2015-08-25 20:52:42 +02:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
rows := []index.IndexRow{
|
|
|
|
NewFieldRow(0, IDFieldName),
|
|
|
|
NewFieldRow(1, "desc"),
|
|
|
|
NewTermFreqRow(0, nil, []byte("a"), 1, 0, 0.0, nil),
|
|
|
|
NewTermFreqRow(0, nil, []byte("b"), 2, 0, 0.0, nil),
|
|
|
|
NewTermFreqRow(0, nil, []byte("c"), 3, 0, 0.0, nil),
|
|
|
|
NewTermFreqRow(0, nil, []byte("d"), 4, 0, 0.0, nil),
|
|
|
|
NewTermFreqRow(0, nil, []byte("a"), 5, 0, 0.0, nil),
|
|
|
|
NewTermFreqRow(0, nil, []byte("b"), 6, 0, 0.0, nil),
|
|
|
|
NewTermFreqRow(0, nil, []byte("e"), 7, 0, 0.0, nil),
|
|
|
|
NewTermFreqRow(0, nil, []byte("g"), 8, 0, 0.0, nil),
|
|
|
|
// first version of all docs have cat
|
|
|
|
NewTermFreqRow(1, []byte("cat"), []byte("a"), 1, 1, 1.0, nil),
|
|
|
|
NewTermFreqRow(1, []byte("cat"), []byte("b"), 2, 1, 1.0, nil),
|
|
|
|
NewTermFreqRow(1, []byte("cat"), []byte("c"), 3, 1, 1.0, nil),
|
|
|
|
NewTermFreqRow(1, []byte("cat"), []byte("d"), 4, 1, 1.0, nil),
|
|
|
|
NewTermFreqRow(1, []byte("cat"), []byte("e"), 7, 1, 1.0, nil),
|
|
|
|
NewTermFreqRow(1, []byte("cat"), []byte("g"), 8, 1, 1.0, nil),
|
|
|
|
// updated version of a still has cat
|
|
|
|
NewTermFreqRow(1, []byte("cat"), []byte("a"), 5, 1, 1.0, nil),
|
|
|
|
// updated version of b does NOT have cat
|
|
|
|
// c has delete in-flight
|
|
|
|
// d has delete not-yet-garbage-collected
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, row := range rows {
|
2015-10-28 16:26:01 +01:00
|
|
|
wb := kvwriter.NewBatch()
|
|
|
|
wb.Set(row.Key(), row.Value())
|
|
|
|
err = kvwriter.ExecuteBatch(wb)
|
2015-08-25 20:52:42 +02:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-10-28 16:26:01 +01:00
|
|
|
f.(*Firestorm).compensator.inFlight = f.(*Firestorm).compensator.inFlight.Upsert(&InFlightItem{docID: []byte("c"), docNum: 0}, rand.Int())
|
|
|
|
f.(*Firestorm).compensator.deletedDocNumbers.Set(4)
|
|
|
|
|
|
|
|
err = kvwriter.Close()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
kvreader, err := f.(*Firestorm).store.Reader()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2015-08-25 20:52:42 +02:00
|
|
|
|
|
|
|
// warmup to load field cache and set maxRead correctly
|
2015-11-24 20:32:33 +01:00
|
|
|
err = f.(*Firestorm).warmup(kvreader)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2015-10-28 16:26:01 +01:00
|
|
|
|
|
|
|
err = kvreader.Close()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2015-08-25 20:52:42 +02:00
|
|
|
|
|
|
|
r, err := f.Reader()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
tfr, err := r.TermFieldReader([]byte("cat"), "desc")
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
expectedDocIds := []string{"a", "e", "g"}
|
|
|
|
foundDocIds := make([]string, 0)
|
|
|
|
next, err := tfr.Next()
|
|
|
|
for next != nil && err == nil {
|
|
|
|
foundDocIds = append(foundDocIds, next.ID)
|
|
|
|
next, err = tfr.Next()
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if !reflect.DeepEqual(expectedDocIds, foundDocIds) {
|
|
|
|
t.Errorf("expected: %v, got %v", expectedDocIds, foundDocIds)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = tfr.Close()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// now try again and Advance to skip over "e"
|
|
|
|
tfr, err = r.TermFieldReader([]byte("cat"), "desc")
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
expectedDocIds = []string{"a", "g"}
|
|
|
|
|
|
|
|
foundDocIds = make([]string, 0)
|
|
|
|
next, err = tfr.Next()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
} else {
|
|
|
|
foundDocIds = append(foundDocIds, next.ID)
|
|
|
|
}
|
|
|
|
next, err = tfr.Advance("f")
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
} else {
|
|
|
|
foundDocIds = append(foundDocIds, next.ID)
|
|
|
|
}
|
|
|
|
|
|
|
|
if !reflect.DeepEqual(expectedDocIds, foundDocIds) {
|
|
|
|
t.Errorf("expected: %v, got %v", expectedDocIds, foundDocIds)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = tfr.Close()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = r.Close()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|