gibheer/bleve
0
0
Fork 0
Code

Merge pull request #402 from mschoch/indexapiwork

Browse Source 
Index/Search API work
This commit is contained in:
Marty Schoch 2016-08-10 12:41:51 -04:00 committed by GitHub
commit 5f1454106d
45 changed files with 1144 additions and 600 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
index/index.go
View File

@ -10,6 +10,7 @@
package index
import (
"bytes"
"encoding/json"
"fmt"
"time"
@ -60,7 +61,7 @@ type AsyncIndex interface {
}
type IndexReader interface {
TermFieldReader(term []byte, field string) (TermFieldReader, error)
TermFieldReader(term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (TermFieldReader, error)
// DocIDReader returns an iterator over documents which identifiers are
// greater than or equal to start and smaller than end. Set start to the
@ -69,6 +70,8 @@ type IndexReader interface {
// The caller must close returned instance to release associated resources.
DocIDReader(start, end string) (DocIDReader, error)
DocIDReaderOnly(ids []string) (DocIDReader, error)
FieldDict(field string) (FieldDict, error)
// FieldDictRange is currently defined to include the start and end terms
@ -76,8 +79,8 @@ type IndexReader interface {
FieldDictPrefix(field string, termPrefix []byte) (FieldDict, error)
Document(id string) (*document.Document, error)
DocumentFieldTerms(id string) (FieldTerms, error)
DocumentFieldTermsForFields(id string, fields []string) (FieldTerms, error)
DocumentFieldTerms(id IndexInternalID) (FieldTerms, error)
DocumentFieldTermsForFields(id IndexInternalID, fields []string) (FieldTerms, error)
Fields() ([]string, error)
@ -85,6 +88,8 @@ type IndexReader interface {
DocCount() uint64
FinalizeDocID(id IndexInternalID) (string, error)
Close() error
}
@ -98,14 +103,36 @@ type TermFieldVector struct {
End uint64
}
// IndexInternalID is an opaque document identifier interal to the index impl
type IndexInternalID []byte
func (id IndexInternalID) Equals(other IndexInternalID) bool {
return id.Compare(other) == 0
}
func (id IndexInternalID) Compare(other IndexInternalID) int {
return bytes.Compare(id, other)
}
type TermFieldDoc struct {
Term string
ID string
ID IndexInternalID
Freq uint64
Norm float64
Vectors []*TermFieldVector
}
// Reset allows an already allocated TermFieldDoc to be reused
func (tfd *TermFieldDoc) Reset() *TermFieldDoc {
// remember the []byte used for the ID
id := tfd.ID
// idiom to copy over from empty TermFieldDoc (0 allocations)
*tfd = TermFieldDoc{}
// reuse the []byte already allocated (and reset len to 0)
tfd.ID = id[:0]
return tfd
}
// TermFieldReader is the interface exposing the enumeration of documents
// containing a given term in a given field. Documents are returned in byte
// lexicographic order over their identifiers.
@ -117,7 +144,7 @@ type TermFieldReader interface {
// Advance resets the enumeration at specified document or its immediate
// follower.
Advance(ID string, preAlloced *TermFieldDoc) (*TermFieldDoc, error)
Advance(ID IndexInternalID, preAlloced *TermFieldDoc) (*TermFieldDoc, error)
// Count returns the number of documents contains the term in this field.
Count() uint64
@ -137,15 +164,15 @@ type FieldDict interface {
// DocIDReader is the interface exposing enumeration of documents identifiers.
// Close the reader to release associated resources.
type DocIDReader interface {
// Next returns the next document identifier in ascending lexicographic
// byte order, or io.EOF when the end of the sequence is reached.
Next() (string, error)
// Next returns the next document internal identifier in the natural
// index order, or io.EOF when the end of the sequence is reached.
Next() (IndexInternalID, error)
// Advance resets the iteration to the first identifier greater than or
// equal to ID. If ID is smaller than the start of the range, the iteration
// Advance resets the iteration to the first internal identifier greater than
// or equal to ID. If ID is smaller than the start of the range, the iteration
// will start there instead. If ID is greater than or equal to the end of
// the range, Next() call will return io.EOF.
Advance(ID string) (string, error)
Advance(ID IndexInternalID) (IndexInternalID, error)
Close() error
}
@ -200,8 +227,3 @@ func (b *Batch) Reset() {
b.IndexOps = make(map[string]*document.Document)
b.InternalOps = make(map[string][]byte)
}
func (tfd *TermFieldDoc) Reset() *TermFieldDoc {
*tfd = TermFieldDoc{}
return tfd
}

2
index/upside_down/dump.go
View File

@ -151,7 +151,7 @@ func (udc *UpsideDownCouch) DumpDoc(id string) chan interface{} {
}
}()
back, err := udc.backIndexRowForDoc(kvreader, id)
back, err := udc.backIndexRowForDoc(kvreader, []byte(id))
if err != nil {
rv <- err
return

20
index/upside_down/index_reader.go
View File

@ -23,12 +23,12 @@ type IndexReader struct {
docCount uint64
}
func (i *IndexReader) TermFieldReader(term []byte, fieldName string) (index.TermFieldReader, error) {
func (i *IndexReader) TermFieldReader(term []byte, fieldName string, includeFreq, includeNorm, includeTermVectors bool) (index.TermFieldReader, error) {
fieldIndex, fieldExists := i.index.fieldCache.FieldNamed(fieldName, false)
if fieldExists {
return newUpsideDownCouchTermFieldReader(i, term, uint16(fieldIndex))
return newUpsideDownCouchTermFieldReader(i, term, uint16(fieldIndex), includeFreq, includeNorm, includeTermVectors)
}
return newUpsideDownCouchTermFieldReader(i, []byte{ByteSeparator}, ^uint16(0))
return newUpsideDownCouchTermFieldReader(i, []byte{ByteSeparator}, ^uint16(0), includeFreq, includeNorm, includeTermVectors)
}
func (i *IndexReader) FieldDict(fieldName string) (index.FieldDict, error) {
@ -51,10 +51,14 @@ func (i *IndexReader) DocIDReader(start, end string) (index.DocIDReader, error)
return newUpsideDownCouchDocIDReader(i, start, end)
}
func (i *IndexReader) DocIDReaderOnly(ids []string) (index.DocIDReader, error) {
return newUpsideDownCouchDocIDReaderOnly(i, ids)
}
func (i *IndexReader) Document(id string) (doc *document.Document, err error) {
// first hit the back index to confirm doc exists
var backIndexRow *BackIndexRow
backIndexRow, err = i.index.backIndexRowForDoc(i.kvreader, id)
backIndexRow, err = i.index.backIndexRowForDoc(i.kvreader, []byte(id))
if err != nil {
return
}
@ -94,7 +98,7 @@ func (i *IndexReader) Document(id string) (doc *document.Document, err error) {
return
}
func (i *IndexReader) DocumentFieldTerms(id string) (index.FieldTerms, error) {
func (i *IndexReader) DocumentFieldTerms(id index.IndexInternalID) (index.FieldTerms, error) {
back, err := i.index.backIndexRowForDoc(i.kvreader, id)
if err != nil {
return nil, err
@ -112,7 +116,7 @@ func (i *IndexReader) DocumentFieldTerms(id string) (index.FieldTerms, error) {
return rv, nil
}
func (i *IndexReader) DocumentFieldTermsForFields(id string, fields []string) (index.FieldTerms, error) {
func (i *IndexReader) DocumentFieldTermsForFields(id index.IndexInternalID, fields []string) (index.FieldTerms, error) {
back, err := i.index.backIndexRowForDoc(i.kvreader, id)
if err != nil {
return nil, err
@ -181,6 +185,10 @@ func (i *IndexReader) Close() error {
return i.kvreader.Close()
}
func (i *IndexReader) FinalizeDocID(id index.IndexInternalID) (string, error) {
return string(id), nil
}
func incrementBytes(in []byte) []byte {
rv := make([]byte, len(in))
copy(rv, in)

157
index/upside_down/reader.go
View File

@ -10,6 +10,8 @@
package upside_down
import (
"bytes"
"sort"
"sync/atomic"
"github.com/blevesearch/bleve/index"
@ -25,7 +27,7 @@ type UpsideDownCouchTermFieldReader struct {
field uint16
}
func newUpsideDownCouchTermFieldReader(indexReader *IndexReader, term []byte, field uint16) (*UpsideDownCouchTermFieldReader, error) {
func newUpsideDownCouchTermFieldReader(indexReader *IndexReader, term []byte, field uint16, includeFreq, includeNorm, includeTermVectors bool) (*UpsideDownCouchTermFieldReader, error) {
dictionaryRow := NewDictionaryRow(term, field, 0)
val, err := indexReader.kvreader.Get(dictionaryRow.Key())
if err != nil {
@ -81,7 +83,7 @@ func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (*
if rv == nil {
rv = &index.TermFieldDoc{}
}
rv.ID = string(tfr.doc)
rv.ID = append(rv.ID, tfr.doc...)
rv.Freq = tfr.freq
rv.Norm = float64(tfr.norm)
if tfr.vectors != nil {
@ -94,9 +96,9 @@ func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (*
return nil, nil
}
func (r *UpsideDownCouchTermFieldReader) Advance(docID string, preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) {
func (r *UpsideDownCouchTermFieldReader) Advance(docID index.IndexInternalID, preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) {
if r.iterator != nil {
tfr := NewTermFrequencyRow(r.term, r.field, []byte(docID), 0, 0)
tfr := NewTermFrequencyRow(r.term, r.field, docID, 0, 0)
r.iterator.Seek(tfr.Key())
key, val, valid := r.iterator.Current()
if valid {
@ -108,7 +110,7 @@ func (r *UpsideDownCouchTermFieldReader) Advance(docID string, preAlloced *index
if rv == nil {
rv = &index.TermFieldDoc{}
}
rv.ID = string(tfr.doc)
rv.ID = append(rv.ID, tfr.doc...)
rv.Freq = tfr.freq
rv.Norm = float64(tfr.norm)
if tfr.vectors != nil {
@ -131,6 +133,9 @@ func (r *UpsideDownCouchTermFieldReader) Close() error {
type UpsideDownCouchDocIDReader struct {
indexReader *IndexReader
iterator store.KVIterator
only []string
onlyPos int
onlyMode bool
}
func newUpsideDownCouchDocIDReader(indexReader *IndexReader, start, end string) (*UpsideDownCouchDocIDReader, error) {
@ -152,37 +157,137 @@ func newUpsideDownCouchDocIDReader(indexReader *IndexReader, start, end string)
}, nil
}
func (r *UpsideDownCouchDocIDReader) Next() (string, error) {
key, val, valid := r.iterator.Current()
if valid {
br, err := NewBackIndexRowKV(key, val)
if err != nil {
return "", err
}
rv := string(br.doc)
r.iterator.Next()
return rv, nil
func newUpsideDownCouchDocIDReaderOnly(indexReader *IndexReader, ids []string) (*UpsideDownCouchDocIDReader, error) {
// ensure ids are sorted
sort.Strings(ids)
startBytes := []byte{0x0}
if len(ids) > 0 {
startBytes = []byte(ids[0])
}
return "", nil
endBytes := []byte{0xff}
if len(ids) > 0 {
endBytes = incrementBytes([]byte(ids[len(ids)-1]))
}
bisr := NewBackIndexRow(startBytes, nil, nil)
bier := NewBackIndexRow(endBytes, nil, nil)
it := indexReader.kvreader.RangeIterator(bisr.Key(), bier.Key())
return &UpsideDownCouchDocIDReader{
indexReader: indexReader,
iterator: it,
only: ids,
onlyMode: true,
}, nil
}
func (r *UpsideDownCouchDocIDReader) Advance(docID string) (string, error) {
bir := NewBackIndexRow([]byte(docID), nil, nil)
func (r *UpsideDownCouchDocIDReader) Next() (index.IndexInternalID, error) {
key, val, valid := r.iterator.Current()
if r.onlyMode {
var rv index.IndexInternalID
for valid && r.onlyPos < len(r.only) {
br, err := NewBackIndexRowKV(key, val)
if err != nil {
return nil, err
}
if !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) {
ok := r.nextOnly()
if !ok {
return nil, nil
}
r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key())
key, val, valid = r.iterator.Current()
continue
} else {
rv = append([]byte(nil), br.doc...)
break
}
}
if valid && r.onlyPos < len(r.only) {
ok := r.nextOnly()
if ok {
r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key())
}
return rv, nil
}
} else {
if valid {
br, err := NewBackIndexRowKV(key, val)
if err != nil {
return nil, err
}
rv := append([]byte(nil), br.doc...)
r.iterator.Next()
return rv, nil
}
}
return nil, nil
}
func (r *UpsideDownCouchDocIDReader) Advance(docID index.IndexInternalID) (index.IndexInternalID, error) {
bir := NewBackIndexRow(docID, nil, nil)
r.iterator.Seek(bir.Key())
key, val, valid := r.iterator.Current()
if valid {
br, err := NewBackIndexRowKV(key, val)
if err != nil {
return "", err
r.onlyPos = sort.SearchStrings(r.only, string(docID))
if r.onlyMode {
var rv index.IndexInternalID
for valid && r.onlyPos < len(r.only) {
br, err := NewBackIndexRowKV(key, val)
if err != nil {
return nil, err
}
if !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) {
ok := r.nextOnly()
if !ok {
return nil, nil
}
r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key())
continue
} else {
rv = append([]byte(nil), br.doc...)
break
}
}
if valid && r.onlyPos < len(r.only) {
ok := r.nextOnly()
if ok {
r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key())
}
return rv, nil
}
} else {
if valid {
br, err := NewBackIndexRowKV(key, val)
if err != nil {
return nil, err
}
rv := append([]byte(nil), br.doc...)
r.iterator.Next()
return rv, nil
}
rv := string(br.doc)
r.iterator.Next()
return rv, nil
}
return "", nil
return nil, nil
}
func (r *UpsideDownCouchDocIDReader) Close() error {
atomic.AddUint64(&r.indexReader.index.stats.termSearchersFinished, uint64(1))
return r.iterator.Close()
}
// move the r.only pos forward one, skipping duplicates
// return true if there is more data, or false if we got to the end of the list
func (r *UpsideDownCouchDocIDReader) nextOnly() bool {
// advance 1 position, until we see a different key
// it's already sorted, so this skips duplicates
start := r.onlyPos
r.onlyPos++
for r.onlyPos < len(r.only) && r.only[r.onlyPos] == r.only[start] {
start = r.onlyPos
r.onlyPos++
}
// inidicate if we got to the end of the list
return r.onlyPos < len(r.only)
}

30
index/upside_down/reader_test.go
View File

@ -72,7 +72,7 @@ func TestIndexReader(t *testing.T) {
}()
// first look for a term that doesn't exist
reader, err := indexReader.TermFieldReader([]byte("nope"), "name")
reader, err := indexReader.TermFieldReader([]byte("nope"), "name", true, true, true)
if err != nil {
t.Errorf("Error accessing term field reader: %v", err)
}
@ -85,7 +85,7 @@ func TestIndexReader(t *testing.T) {
t.Fatal(err)
}
reader, err = indexReader.TermFieldReader([]byte("test"), "name")
reader, err = indexReader.TermFieldReader([]byte("test"), "name", true, true, true)
if err != nil {
t.Errorf("Error accessing term field reader: %v", err)
}
@ -111,7 +111,7 @@ func TestIndexReader(t *testing.T) {
}
expectedMatch := &index.TermFieldDoc{
ID: "2",
ID: index.IndexInternalID("2"),
Freq: 1,
Norm: 0.5773502588272095,
Vectors: []*index.TermFieldVector{
@ -123,7 +123,7 @@ func TestIndexReader(t *testing.T) {
},
},
}
tfr, err := indexReader.TermFieldReader([]byte("rice"), "desc")
tfr, err := indexReader.TermFieldReader([]byte("rice"), "desc", true, true, true)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
@ -140,22 +140,22 @@ func TestIndexReader(t *testing.T) {
}
// now test usage of advance
reader, err = indexReader.TermFieldReader([]byte("test"), "name")
reader, err = indexReader.TermFieldReader([]byte("test"), "name", true, true, true)
if err != nil {
t.Errorf("Error accessing term field reader: %v", err)
}
match, err = reader.Advance("2", nil)
match, err = reader.Advance(index.IndexInternalID("2"), nil)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if match == nil {
t.Fatalf("Expected match, got nil")
}
if match.ID != "2" {
if !match.ID.Equals(index.IndexInternalID("2")) {
t.Errorf("Expected ID '2', got '%s'", match.ID)
}
match, err = reader.Advance("3", nil)
match, err = reader.Advance(index.IndexInternalID("3"), nil)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
@ -168,7 +168,7 @@ func TestIndexReader(t *testing.T) {
}
// now test creating a reader for a field that doesn't exist
reader, err = indexReader.TermFieldReader([]byte("water"), "doesnotexist")
reader, err = indexReader.TermFieldReader([]byte("water"), "doesnotexist", true, true, true)
if err != nil {
t.Errorf("Error accessing term field reader: %v", err)
}
@ -183,7 +183,7 @@ func TestIndexReader(t *testing.T) {
if match != nil {
t.Errorf("expected nil, got %v", match)
}
match, err = reader.Advance("anywhere", nil)
match, err = reader.Advance(index.IndexInternalID("anywhere"), nil)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
@ -260,7 +260,7 @@ func TestIndexDocIdReader(t *testing.T) {
id, err := reader.Next()
count := uint64(0)
for id != "" {
for id != nil {
count++
id, err = reader.Next()
}
@ -280,19 +280,19 @@ func TestIndexDocIdReader(t *testing.T) {
}
}()
id, err = reader2.Advance("2")
id, err = reader2.Advance(index.IndexInternalID("2"))
if err != nil {
t.Error(err)
}
if id != "2" {
if !id.Equals(index.IndexInternalID("2")) {
t.Errorf("expected to find id '2', got '%s'", id)
}
id, err = reader2.Advance("3")
id, err = reader2.Advance(index.IndexInternalID("3"))
if err != nil {
t.Error(err)
}
if id != "" {
if id != nil {
t.Errorf("expected to find id '', got '%s'", id)
}
}

10
index/upside_down/upside_down.go
View File

@ -439,7 +439,7 @@ func (udc *UpsideDownCouch) Update(doc *document.Document) (err error) {
// first we lookup the backindex row for the doc id if it exists
// lookup the back index row
var backIndexRow *BackIndexRow
backIndexRow, err = udc.backIndexRowForDoc(kvreader, doc.ID)
backIndexRow, err = udc.backIndexRowForDoc(kvreader, index.IndexInternalID(doc.ID))
if err != nil {
_ = kvreader.Close()
atomic.AddUint64(&udc.stats.errors, 1)
@ -627,7 +627,7 @@ func (udc *UpsideDownCouch) Delete(id string) (err error) {
// first we lookup the backindex row for the doc id if it exists
// lookup the back index row
var backIndexRow *BackIndexRow
backIndexRow, err = udc.backIndexRowForDoc(kvreader, id)
backIndexRow, err = udc.backIndexRowForDoc(kvreader, index.IndexInternalID(id))
if err != nil {
_ = kvreader.Close()
atomic.AddUint64(&udc.stats.errors, 1)
@ -695,10 +695,10 @@ func (udc *UpsideDownCouch) deleteSingle(id string, backIndexRow *BackIndexRow,
return deleteRows
}
func (udc *UpsideDownCouch) backIndexRowForDoc(kvreader store.KVReader, docID string) (*BackIndexRow, error) {
func (udc *UpsideDownCouch) backIndexRowForDoc(kvreader store.KVReader, docID index.IndexInternalID) (*BackIndexRow, error) {
// use a temporary row structure to build key
tempRow := &BackIndexRow{
doc: []byte(docID),
doc: docID,
}
keyBuf := GetRowBuffer()
@ -833,7 +833,7 @@ func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) {
}
for docID, doc := range batch.IndexOps {
backIndexRow, err := udc.backIndexRowForDoc(kvreader, docID)
backIndexRow, err := udc.backIndexRowForDoc(kvreader, index.IndexInternalID(docID))
if err != nil {
docBackIndexRowErr = err
return

12
index/upside_down/upside_down_test.go
View File

@ -663,16 +663,16 @@ func TestIndexBatch(t *testing.T) {
if err != nil {
t.Error(err)
}
docIds := make([]string, 0)
docIds := make([]index.IndexInternalID, 0)
docID, err := docIDReader.Next()
for docID != "" && err == nil {
for docID != nil && err == nil {
docIds = append(docIds, docID)
docID, err = docIDReader.Next()
}
if err != nil {
t.Error(err)
}
expectedDocIds := []string{"2", "3"}
expectedDocIds := []index.IndexInternalID{index.IndexInternalID("2"), index.IndexInternalID("3")}
if !reflect.DeepEqual(docIds, expectedDocIds) {
t.Errorf("expected ids: %v, got ids: %v", expectedDocIds, docIds)
}
@ -1119,14 +1119,14 @@ func TestIndexTermReaderCompositeFields(t *testing.T) {
}
}()
termFieldReader, err := indexReader.TermFieldReader([]byte("mister"), "_all")
termFieldReader, err := indexReader.TermFieldReader([]byte("mister"), "_all", true, true, true)
if err != nil {
t.Error(err)
}
tfd, err := termFieldReader.Next(nil)
for tfd != nil && err == nil {
if tfd.ID != "1" {
if !tfd.ID.Equals(index.IndexInternalID("1")) {
t.Errorf("expected to find document id 1")
}
tfd, err = termFieldReader.Next(nil)
@ -1179,7 +1179,7 @@ func TestIndexDocumentFieldTerms(t *testing.T) {
}
}()
fieldTerms, err := indexReader.DocumentFieldTerms("1")
fieldTerms, err := indexReader.DocumentFieldTerms(index.IndexInternalID("1"))
if err != nil {
t.Error(err)
}

2
index_impl.go
View File

@ -435,7 +435,7 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
collector.SetFacetsBuilder(facetsBuilder)
}
err = collector.Collect(ctx, searcher)
err = collector.Collect(ctx, searcher, indexReader)
if err != nil {
return nil, err
}

4
search/collector.go
View File

@ -12,11 +12,13 @@ package search
import (
"time"
"github.com/blevesearch/bleve/index"
"golang.org/x/net/context"
)
type Collector interface {
Collect(ctx context.Context, searcher Searcher) error
Collect(ctx context.Context, searcher Searcher, reader index.IndexReader) error
Results() DocumentMatchCollection
Total() uint64
MaxScore() float64

9
search/collectors/bench_test.go
View File

@ -5,16 +5,17 @@ import (
"strconv"
"testing"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/search"
"golang.org/x/net/context"
)
func benchHelper(numOfMatches int, collector search.Collector, b *testing.B) {
matches := make(search.DocumentMatchCollection, 0, numOfMatches)
matches := make([]*search.DocumentMatch, 0, numOfMatches)
for i := 0; i < numOfMatches; i++ {
matches = append(matches, &search.DocumentMatch{
ID: strconv.Itoa(i),
Score: rand.Float64(),
IndexInternalID: index.IndexInternalID(strconv.Itoa(i)),
Score: rand.Float64(),
})
}
@ -24,7 +25,7 @@ func benchHelper(numOfMatches int, collector search.Collector, b *testing.B) {
searcher := &stubSearcher{
matches: matches,
}
err := collector.Collect(context.Background(), searcher)
err := collector.Collect(context.Background(), searcher, &stubReader{})
if err != nil {
b.Fatal(err)
}

66
search/collectors/collector_top_score.go
View File

@ -15,6 +15,7 @@ import (
"golang.org/x/net/context"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/search"
)
@ -27,6 +28,7 @@ type TopScoreCollector struct {
minScore float64
total uint64
facetsBuilder *search.FacetsBuilder
actualResults search.DocumentMatchCollection
}
func NewTopScorerCollector(k int) *TopScoreCollector {
@ -59,16 +61,21 @@ func (tksc *TopScoreCollector) Took() time.Duration {
var COLLECT_CHECK_DONE_EVERY = uint64(1024)
func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Searcher) error {
func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Searcher, reader index.IndexReader) error {
startTime := time.Now()
var err error
var pre search.DocumentMatch // A single pre-alloc'ed, reused instance.
var next *search.DocumentMatch
// search context with enough pre-allocated document matches
searchContext := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(tksc.k + tksc.skip + searcher.DocumentMatchPoolSize()),
}
select {
case <-ctx.Done():
return ctx.Err()
default:
next, err = searcher.Next(&pre)
next, err = searcher.Next(searchContext)
}
for err == nil && next != nil {
if tksc.total%COLLECT_CHECK_DONE_EVERY == 0 {
@ -78,15 +85,22 @@ func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Sear
default:
}
}
tksc.collectSingle(next)
if tksc.facetsBuilder != nil {
err = tksc.facetsBuilder.Update(next)
if err != nil {
break
}
}
next, err = searcher.Next(pre.Reset())
tksc.collectSingle(searchContext, next)
next, err = searcher.Next(searchContext)
}
// finalize actual results
tksc.actualResults, err = tksc.finalizeResults(reader)
if err != nil {
return err
}
// compute search duration
tksc.took = time.Since(startTime)
if err != nil {
@ -95,47 +109,50 @@ func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Sear
return nil
}
func (tksc *TopScoreCollector) collectSingle(dmIn *search.DocumentMatch) {
func (tksc *TopScoreCollector) collectSingle(ctx *search.SearchContext, d *search.DocumentMatch) {
// increment total hits
tksc.total++
// update max score
if dmIn.Score > tksc.maxScore {
tksc.maxScore = dmIn.Score
if d.Score > tksc.maxScore {
tksc.maxScore = d.Score
}
if dmIn.Score <= tksc.minScore {
if d.Score <= tksc.minScore {
ctx.DocumentMatchPool.Put(d)
return
}
// Because the dmIn will be the single, pre-allocated, reused
// instance, we need to copy the dmIn into a new, standalone
// instance before inserting into our candidate results list.
dm := &search.DocumentMatch{}
*dm = *dmIn
for e := tksc.results.Front(); e != nil; e = e.Next() {
curr := e.Value.(*search.DocumentMatch)
if dm.Score <= curr.Score {
if d.Score <= curr.Score {
tksc.results.InsertBefore(dm, e)
tksc.results.InsertBefore(d, e)
// if we just made the list too long
if tksc.results.Len() > (tksc.k + tksc.skip) {
// remove the head
tksc.minScore = tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch).Score
removed := tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch)
tksc.minScore = removed.Score
ctx.DocumentMatchPool.Put(removed)
}
return
}
}
// if we got to the end, we still have to add it
tksc.results.PushBack(dm)
tksc.results.PushBack(d)
if tksc.results.Len() > (tksc.k + tksc.skip) {
// remove the head
tksc.minScore = tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch).Score
removed := tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch)
tksc.minScore = removed.Score
ctx.DocumentMatchPool.Put(removed)
}
}
func (tksc *TopScoreCollector) Results() search.DocumentMatchCollection {
return tksc.actualResults
}
func (tksc *TopScoreCollector) finalizeResults(r index.IndexReader) (search.DocumentMatchCollection, error) {
if tksc.results.Len()-tksc.skip > 0 {
rv := make(search.DocumentMatchCollection, tksc.results.Len()-tksc.skip)
i := 0
@ -145,12 +162,17 @@ func (tksc *TopScoreCollector) Results() search.DocumentMatchCollection {
skipped++
continue
}
var err error
rv[i] = e.Value.(*search.DocumentMatch)
rv[i].ID, err = r.FinalizeDocID(rv[i].IndexInternalID)
if err != nil {
return nil, err
}
i++
}
return rv
return rv, nil
}
return search.DocumentMatchCollection{}
return search.DocumentMatchCollection{}, nil
}
func (tksc *TopScoreCollector) SetFacetsBuilder(facetsBuilder *search.FacetsBuilder) {

241
search/collectors/collector_top_score_test.go
View File

@ -14,6 +14,7 @@ import (
"golang.org/x/net/context"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/search"
)
@ -23,68 +24,68 @@ func TestTop10Scores(t *testing.T) {
// the top-10 scores are > 10
// everything else is less than 10
searcher := &stubSearcher{
matches: search.DocumentMatchCollection{
matches: []*search.DocumentMatch{
&search.DocumentMatch{
ID: "a",
Score: 11,
IndexInternalID: index.IndexInternalID("a"),
Score: 11,
},
&search.DocumentMatch{
ID: "b",
Score: 9,
IndexInternalID: index.IndexInternalID("b"),
Score: 9,
},
&search.DocumentMatch{
ID: "c",
Score: 11,
IndexInternalID: index.IndexInternalID("c"),
Score: 11,
},
&search.DocumentMatch{
ID: "d",
Score: 9,
IndexInternalID: index.IndexInternalID("d"),
Score: 9,
},
&search.DocumentMatch{
ID: "e",
Score: 11,
IndexInternalID: index.IndexInternalID("e"),
Score: 11,
},
&search.DocumentMatch{
ID: "f",
Score: 9,
IndexInternalID: index.IndexInternalID("f"),
Score: 9,
},
&search.DocumentMatch{
ID: "g",
Score: 11,
IndexInternalID: index.IndexInternalID("g"),
Score: 11,
},
&search.DocumentMatch{
ID: "h",
Score: 9,
IndexInternalID: index.IndexInternalID("h"),
Score: 9,
},
&search.DocumentMatch{
ID: "i",
Score: 11,
IndexInternalID: index.IndexInternalID("i"),
Score: 11,
},
&search.DocumentMatch{
ID: "j",
Score: 11,
IndexInternalID: index.IndexInternalID("j"),
Score: 11,
},
&search.DocumentMatch{
ID: "k",
Score: 11,
IndexInternalID: index.IndexInternalID("k"),
Score: 11,
},
&search.DocumentMatch{
ID: "l",
Score: 99,
IndexInternalID: index.IndexInternalID("l"),
Score: 99,
},
&search.DocumentMatch{
ID: "m",
Score: 11,
IndexInternalID: index.IndexInternalID("m"),
Score: 11,
},
&search.DocumentMatch{
ID: "n",
Score: 11,
IndexInternalID: index.IndexInternalID("n"),
Score: 11,
},
},
}
collector := NewTopScorerCollector(10)
err := collector.Collect(context.Background(), searcher)
err := collector.Collect(context.Background(), searcher, &stubReader{})
if err != nil {
t.Fatal(err)
}
@ -131,68 +132,68 @@ func TestTop10ScoresSkip10(t *testing.T) {
// the top-10 scores are > 10
// everything else is less than 10
searcher := &stubSearcher{
matches: search.DocumentMatchCollection{
matches: []*search.DocumentMatch{
&search.DocumentMatch{
ID: "a",
Score: 11,
IndexInternalID: index.IndexInternalID("a"),
Score: 11,
},
&search.DocumentMatch{
ID: "b",
Score: 9.5,
IndexInternalID: index.IndexInternalID("b"),
Score: 9.5,
},
&search.DocumentMatch{
ID: "c",
Score: 11,
IndexInternalID: index.IndexInternalID("c"),
Score: 11,
},
&search.DocumentMatch{
ID: "d",
Score: 9,
IndexInternalID: index.IndexInternalID("d"),
Score: 9,
},
&search.DocumentMatch{
ID: "e",
Score: 11,
IndexInternalID: index.IndexInternalID("e"),
Score: 11,
},
&search.DocumentMatch{
ID: "f",
Score: 9,
IndexInternalID: index.IndexInternalID("f"),
Score: 9,
},
&search.DocumentMatch{
ID: "g",
Score: 11,
IndexInternalID: index.IndexInternalID("g"),
Score: 11,
},
&search.DocumentMatch{
ID: "h",
Score: 9,
IndexInternalID: index.IndexInternalID("h"),
Score: 9,
},
&search.DocumentMatch{
ID: "i",
Score: 11,
IndexInternalID: index.IndexInternalID("i"),
Score: 11,
},
&search.DocumentMatch{
ID: "j",
Score: 11,
IndexInternalID: index.IndexInternalID("j"),
Score: 11,
},
&search.DocumentMatch{
ID: "k",
Score: 11,
IndexInternalID: index.IndexInternalID("k"),
Score: 11,
},
&search.DocumentMatch{
ID: "l",
Score: 99,
IndexInternalID: index.IndexInternalID("l"),
Score: 99,
},
&search.DocumentMatch{
ID: "m",
Score: 11,
IndexInternalID: index.IndexInternalID("m"),
Score: 11,
},
&search.DocumentMatch{
ID: "n",
Score: 11,
IndexInternalID: index.IndexInternalID("n"),
Score: 11,
},
},
}
collector := NewTopScorerSkipCollector(10, 10)
err := collector.Collect(context.Background(), searcher)
err := collector.Collect(context.Background(), searcher, &stubReader{})
if err != nil {
t.Fatal(err)
}
@ -227,69 +228,69 @@ func TestPaginationSameScores(t *testing.T) {
// a stub search with more than 10 matches
// all documents have the same score
searcher := &stubSearcher{
matches: search.DocumentMatchCollection{
matches: []*search.DocumentMatch{
&search.DocumentMatch{
ID: "a",
Score: 5,
IndexInternalID: index.IndexInternalID("a"),
Score: 5,
},
&search.DocumentMatch{
ID: "b",
Score: 5,
IndexInternalID: index.IndexInternalID("b"),
Score: 5,
},
&search.DocumentMatch{
ID: "c",
Score: 5,
IndexInternalID: index.IndexInternalID("c"),
Score: 5,
},
&search.DocumentMatch{
ID: "d",
Score: 5,
IndexInternalID: index.IndexInternalID("d"),
Score: 5,
},
&search.DocumentMatch{
ID: "e",
Score: 5,
IndexInternalID: index.IndexInternalID("e"),
Score: 5,
},
&search.DocumentMatch{
ID: "f",
Score: 5,
IndexInternalID: index.IndexInternalID("f"),
Score: 5,
},
&search.DocumentMatch{
ID: "g",
Score: 5,
IndexInternalID: index.IndexInternalID("g"),
Score: 5,
},
&search.DocumentMatch{
ID: "h",
Score: 5,
IndexInternalID: index.IndexInternalID("h"),
Score: 5,
},
&search.DocumentMatch{
ID: "i",
Score: 5,
IndexInternalID: index.IndexInternalID("i"),
Score: 5,
},
&search.DocumentMatch{
ID: "j",
Score: 5,
IndexInternalID: index.IndexInternalID("j"),
Score: 5,
},
&search.DocumentMatch{
ID: "k",
Score: 5,
IndexInternalID: index.IndexInternalID("k"),
Score: 5,
},
&search.DocumentMatch{
ID: "l",
Score: 5,
IndexInternalID: index.IndexInternalID("l"),
Score: 5,
},
&search.DocumentMatch{
ID: "m",
Score: 5,
IndexInternalID: index.IndexInternalID("m"),
Score: 5,
},
&search.DocumentMatch{
ID: "n",
Score: 5,
IndexInternalID: index.IndexInternalID("n"),
Score: 5,
},
},
}
// first get first 5 hits
collector := NewTopScorerSkipCollector(5, 0)
err := collector.Collect(context.Background(), searcher)
err := collector.Collect(context.Background(), searcher, &stubReader{})
if err != nil {
t.Fatal(err)
}
@ -313,69 +314,69 @@ func TestPaginationSameScores(t *testing.T) {
// a stub search with more than 10 matches
// all documents have the same score
searcher = &stubSearcher{
matches: search.DocumentMatchCollection{
matches: []*search.DocumentMatch{
&search.DocumentMatch{
ID: "a",
Score: 5,
IndexInternalID: index.IndexInternalID("a"),
Score: 5,
},
&search.DocumentMatch{
ID: "b",
Score: 5,
IndexInternalID: index.IndexInternalID("b"),
Score: 5,
},
&search.DocumentMatch{
ID: "c",
Score: 5,
IndexInternalID: index.IndexInternalID("c"),
Score: 5,
},
&search.DocumentMatch{
ID: "d",
Score: 5,
IndexInternalID: index.IndexInternalID("d"),
Score: 5,
},
&search.DocumentMatch{
ID: "e",
Score: 5,
IndexInternalID: index.IndexInternalID("e"),
Score: 5,
},
&search.DocumentMatch{
ID: "f",
Score: 5,
IndexInternalID: index.IndexInternalID("f"),
Score: 5,
},
&search.DocumentMatch{
ID: "g",
Score: 5,
IndexInternalID: index.IndexInternalID("g"),
Score: 5,
},
&search.DocumentMatch{
ID: "h",
Score: 5,
IndexInternalID: index.IndexInternalID("h"),
Score: 5,
},
&search.DocumentMatch{
ID: "i",
Score: 5,
IndexInternalID: index.IndexInternalID("i"),
Score: 5,
},
&search.DocumentMatch{
ID: "j",
Score: 5,
IndexInternalID: index.IndexInternalID("j"),
Score: 5,
},
&search.DocumentMatch{
ID: "k",
Score: 5,
IndexInternalID: index.IndexInternalID("k"),
Score: 5,
},
&search.DocumentMatch{
ID: "l",
Score: 5,
IndexInternalID: index.IndexInternalID("l"),
Score: 5,
},
&search.DocumentMatch{
ID: "m",
Score: 5,
IndexInternalID: index.IndexInternalID("m"),
Score: 5,
},
&search.DocumentMatch{
ID: "n",
Score: 5,
IndexInternalID: index.IndexInternalID("n"),
Score: 5,
},
},
}
// now get next 5 hits
collector = NewTopScorerSkipCollector(5, 5)
err = collector.Collect(context.Background(), searcher)
err = collector.Collect(context.Background(), searcher, &stubReader{})
if err != nil {
t.Fatal(err)
}

72
search/collectors/search_test.go
View File

@ -10,15 +10,17 @@
package collectors
import (
"github.com/blevesearch/bleve/document"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/search"
)
type stubSearcher struct {
index int
matches search.DocumentMatchCollection
matches []*search.DocumentMatch
}
func (ss *stubSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
func (ss *stubSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
if ss.index < len(ss.matches) {
rv := ss.matches[ss.index]
ss.index++
@ -27,9 +29,9 @@ func (ss *stubSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docume
return nil, nil
}
func (ss *stubSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
func (ss *stubSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
for ss.index < len(ss.matches) && ss.matches[ss.index].ID < ID {
for ss.index < len(ss.matches) && ss.matches[ss.index].IndexInternalID.Compare(ID) < 0 {
ss.index++
}
if ss.index < len(ss.matches) {
@ -58,3 +60,65 @@ func (ss *stubSearcher) Count() uint64 {
func (ss *stubSearcher) Min() int {
return 0
}
func (ss *stubSearcher) DocumentMatchPoolSize() int {
return 0
}
type stubReader struct{}
func (sr *stubReader) TermFieldReader(term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (index.TermFieldReader, error) {
return nil, nil
}
func (sr *stubReader) DocIDReader(start, end string) (index.DocIDReader, error) {
return nil, nil
}
func (sr *stubReader) DocIDReaderOnly(ids []string) (index.DocIDReader, error) {
return nil, nil
}
func (sr *stubReader) FieldDict(field string) (index.FieldDict, error) {
return nil, nil
}
func (sr *stubReader) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) {
return nil, nil
}
func (sr *stubReader) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) {
return nil, nil
}
func (sr *stubReader) Document(id string) (*document.Document, error) {
return nil, nil
}
func (sr *stubReader) DocumentFieldTerms(id index.IndexInternalID) (index.FieldTerms, error) {
return nil, nil
}
func (sr *stubReader) DocumentFieldTermsForFields(id index.IndexInternalID, fields []string) (index.FieldTerms, error) {
return nil, nil
}
func (sr *stubReader) Fields() ([]string, error) {
return nil, nil
}
func (sr *stubReader) GetInternal(key []byte) ([]byte, error) {
return nil, nil
}
func (sr *stubReader) DocCount() uint64 {
return 0
}
func (sr *stubReader) FinalizeDocID(id index.IndexInternalID) (string, error) {
return string(id), nil
}
func (sr *stubReader) Close() error {
return nil
}

2
search/facets_builder.go
View File

@ -42,7 +42,7 @@ func (fb *FacetsBuilder) Update(docMatch *DocumentMatch) error {
for _, facetBuilder := range fb.facets {
fields = append(fields, facetBuilder.Field())
}
fieldTerms, err := fb.indexReader.DocumentFieldTermsForFields(docMatch.ID, fields)
fieldTerms, err := fb.indexReader.DocumentFieldTermsForFields(docMatch.IndexInternalID, fields)
if err != nil {
return err
}

70
search/pool.go Normal file
View File

@ -0,0 +1,70 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package search
// DocumentMatchPoolTooSmall is a callback function that can be executed
// when the DocumentMatchPool does not have sufficient capacity
// By default we just perform just-in-time allocation, but you could log
// a message, or panic, etc.
type DocumentMatchPoolTooSmall func(p *DocumentMatchPool) *DocumentMatch
// DocumentMatchPool manages use/re-use of DocumentMatch instances
// it pre-allocates space from a single large block with the expected
// number of instances. It is not thread-safe as currently all
// aspects of search take place in a single goroutine.
type DocumentMatchPool struct {
avail DocumentMatchCollection
TooSmall DocumentMatchPoolTooSmall
}
func defaultDocumentMatchPoolTooSmall(p *DocumentMatchPool) *DocumentMatch {
return &DocumentMatch{}
}
// NewDocumentMatchPool will build a DocumentMatchPool with memory
// pre-allocated to accomodate the requested number of DocumentMatch
// instances
func NewDocumentMatchPool(size int) *DocumentMatchPool {
avail := make(DocumentMatchCollection, 0, size)
// pre-allocate the expected number of instances
startBlock := make([]DocumentMatch, size)
// make these initial instances available
for i := range startBlock {
avail = append(avail, &startBlock[i])
}
return &DocumentMatchPool{
avail: avail,
TooSmall: defaultDocumentMatchPoolTooSmall,
}
}
// Get returns an available DocumentMatch from the pool
// if the pool was not allocated with sufficient size, an allocation will
// occur to satisfy this request. As a side-effect this will grow the size
// of the pool.
func (p *DocumentMatchPool) Get() *DocumentMatch {
var rv *DocumentMatch
if len(p.avail) > 0 {
rv, p.avail = p.avail[len(p.avail)-1], p.avail[:len(p.avail)-1]
} else {
rv = p.TooSmall(p)
}
return rv
}
// Put returns a DocumentMatch to the pool
func (p *DocumentMatchPool) Put(d *DocumentMatch) {
if d == nil {
return
}
// reset DocumentMatch before returning it to available pool
d.Reset()
p.avail = append(p.avail, d)
}

65
search/pool_test.go Normal file
View File

@ -0,0 +1,65 @@
// Copyright (c) 2013 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package search
import "testing"
func TestDocumentMatchPool(t *testing.T) {
tooManyCalled := false
// create a pool
dmp := NewDocumentMatchPool(10)
dmp.TooSmall = func(inner *DocumentMatchPool) *DocumentMatch {
tooManyCalled = true
return &DocumentMatch{}
}
// get 10 instances without returning
returned := make(DocumentMatchCollection, 10)
for i := 0; i < 10; i++ {
returned[i] = dmp.Get()
if tooManyCalled {
t.Fatal("too many function called before expected")
}
}
// get one more and see if too many function is called
extra := dmp.Get()
if !tooManyCalled {
t.Fatal("expected too many function to be called, but wasnt")
}
// return the first 10
for i := 0; i < 10; i++ {
dmp.Put(returned[i])
}
// check len and cap
if len(dmp.avail) != 10 {
t.Fatalf("expected 10 available, got %d", len(dmp.avail))
}
if cap(dmp.avail) != 10 {
t.Fatalf("expected avail cap still 10, got %d", cap(dmp.avail))
}
// return the extra
dmp.Put(extra)
// check len and cap grown to 11
if len(dmp.avail) != 11 {
t.Fatalf("expected 11 available, got %d", len(dmp.avail))
}
// cap grows, but not by 1 (append behavior)
if cap(dmp.avail) <= 10 {
t.Fatalf("expected avail cap mpore than 10, got %d", cap(dmp.avail))
}
}

17
search/scorers/scorer_conjunction.go
View File

@ -23,11 +23,7 @@ func NewConjunctionQueryScorer(explain bool) *ConjunctionQueryScorer {
}
}
func (s *ConjunctionQueryScorer) Score(constituents []*search.DocumentMatch) *search.DocumentMatch {
rv := search.DocumentMatch{
ID: constituents[0].ID,
}
func (s *ConjunctionQueryScorer) Score(ctx *search.SearchContext, constituents []*search.DocumentMatch) *search.DocumentMatch {
var sum float64
var childrenExplanations []*search.Explanation
if s.explain {
@ -44,16 +40,21 @@ func (s *ConjunctionQueryScorer) Score(constituents []*search.DocumentMatch) *se
locations = append(locations, docMatch.Locations)
}
}
rv.Score = sum
newScore := sum
var newExpl *search.Explanation
if s.explain {
rv.Expl = &search.Explanation{Value: sum, Message: "sum of:", Children: childrenExplanations}
newExpl = &search.Explanation{Value: sum, Message: "sum of:", Children: childrenExplanations}
}
// reuse constituents[0] as the return value
rv := constituents[0]
rv.Score = newScore
rv.Expl = newExpl
if len(locations) == 1 {
rv.Locations = locations[0]
} else if len(locations) > 1 {
rv.Locations = search.MergeLocations(locations)
}
return &rv
return rv
}

12
search/scorers/scorer_constant.go
View File

@ -12,6 +12,7 @@ package scorers
import (
"fmt"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/search"
)
@ -64,7 +65,7 @@ func (s *ConstantScorer) SetQueryNorm(qnorm float64) {
}
}
func (s *ConstantScorer) Score(id string) *search.DocumentMatch {
func (s *ConstantScorer) Score(ctx *search.SearchContext, id index.IndexInternalID) *search.DocumentMatch {
var scoreExplanation *search.Explanation
score := s.constant
@ -91,13 +92,12 @@ func (s *ConstantScorer) Score(id string) *search.DocumentMatch {
}
}
rv := search.DocumentMatch{
ID: id,
Score: score,
}
rv := ctx.DocumentMatchPool.Get()
rv.IndexInternalID = id
rv.Score = score
if s.explain {
rv.Expl = scoreExplanation
}
return &rv
return rv
}

22
search/scorers/scorer_constant_test.go
View File

@ -28,7 +28,7 @@ func TestConstantScorer(t *testing.T) {
// test some simple math
{
termMatch: &index.TermFieldDoc{
ID: "one",
ID: index.IndexInternalID("one"),
Freq: 1,
Norm: 1.0,
Vectors: []*index.TermFieldVector{
@ -41,8 +41,8 @@ func TestConstantScorer(t *testing.T) {
},
},
result: &search.DocumentMatch{
ID: "one",
Score: 1.0,
IndexInternalID: index.IndexInternalID("one"),
Score: 1.0,
Expl: &search.Explanation{
Value: 1.0,
Message: "ConstantScore()",
@ -52,7 +52,10 @@ func TestConstantScorer(t *testing.T) {
}
for _, test := range tests {
actual := scorer.Score(test.termMatch.ID)
ctx := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(1),
}
actual := scorer.Score(ctx, test.termMatch.ID)
if !reflect.DeepEqual(actual, test.result) {
t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch)
@ -72,13 +75,13 @@ func TestConstantScorerWithQueryNorm(t *testing.T) {
}{
{
termMatch: &index.TermFieldDoc{
ID: "one",
ID: index.IndexInternalID("one"),
Freq: 1,
Norm: 1.0,
},
result: &search.DocumentMatch{
ID: "one",
Score: 2.0,
IndexInternalID: index.IndexInternalID("one"),
Score: 2.0,
Expl: &search.Explanation{
Value: 2.0,
Message: "weight(^1.000000), product of:",
@ -108,7 +111,10 @@ func TestConstantScorerWithQueryNorm(t *testing.T) {
}
for _, test := range tests {
actual := scorer.Score(test.termMatch.ID)
ctx := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(1),
}
actual := scorer.Score(ctx, test.termMatch.ID)
if !reflect.DeepEqual(actual, test.result) {
t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch)

17
search/scorers/scorer_disjunction.go
View File

@ -25,11 +25,7 @@ func NewDisjunctionQueryScorer(explain bool) *DisjunctionQueryScorer {
}
}
func (s *DisjunctionQueryScorer) Score(constituents []*search.DocumentMatch, countMatch, countTotal int) *search.DocumentMatch {
rv := search.DocumentMatch{
ID: constituents[0].ID,
}
func (s *DisjunctionQueryScorer) Score(ctx *search.SearchContext, constituents []*search.DocumentMatch, countMatch, countTotal int) *search.DocumentMatch {
var sum float64
var childrenExplanations []*search.Explanation
if s.explain {
@ -53,19 +49,24 @@ func (s *DisjunctionQueryScorer) Score(constituents []*search.DocumentMatch, cou
}
coord := float64(countMatch) / float64(countTotal)
rv.Score = sum * coord
newScore := sum * coord
var newExpl *search.Explanation
if s.explain {
ce := make([]*search.Explanation, 2)
ce[0] = rawExpl
ce[1] = &search.Explanation{Value: coord, Message: fmt.Sprintf("coord(%d/%d)", countMatch, countTotal)}
rv.Expl = &search.Explanation{Value: rv.Score, Message: "product of:", Children: ce}
newExpl = &search.Explanation{Value: newScore, Message: "product of:", Children: ce}
}
// reuse constituents[0] as the return value
rv := constituents[0]
rv.Score = newScore
rv.Expl = newExpl
if len(locations) == 1 {
rv.Locations = locations[0]
} else if len(locations) > 1 {
rv.Locations = search.MergeLocations(locations)
}
return &rv
return rv
}

9
search/scorers/scorer_term.go
View File

@ -83,7 +83,7 @@ func (s *TermQueryScorer) SetQueryNorm(qnorm float64) {
}
}
func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc, preAllocated *search.DocumentMatch) *search.DocumentMatch {
func (s *TermQueryScorer) Score(ctx *search.SearchContext, termMatch *index.TermFieldDoc) *search.DocumentMatch {
var scoreExplanation *search.Explanation
// need to compute score
@ -128,11 +128,8 @@ func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc, preAllocated *sea
}
}
rv := preAllocated
if rv == nil {
rv = &search.DocumentMatch{}
}
rv.ID = termMatch.ID
rv := ctx.DocumentMatchPool.Get()
rv.IndexInternalID = append(rv.IndexInternalID, termMatch.ID...)
rv.Score = score
if s.explain {
rv.Expl = scoreExplanation

34
search/scorers/scorer_term_test.go
View File

@ -35,7 +35,7 @@ func TestTermScorer(t *testing.T) {
// test some simple math
{
termMatch: &index.TermFieldDoc{
ID: "one",
ID: index.IndexInternalID("one"),
Freq: 1,
Norm: 1.0,
Vectors: []*index.TermFieldVector{
@ -48,8 +48,8 @@ func TestTermScorer(t *testing.T) {
},
},
result: &search.DocumentMatch{
ID: "one",
Score: math.Sqrt(1.0) * idf,
IndexInternalID: index.IndexInternalID("one"),
Score: math.Sqrt(1.0) * idf,
Expl: &search.Explanation{
Value: math.Sqrt(1.0) * idf,
Message: "fieldWeight(desc:beer in one), product of:",
@ -84,13 +84,13 @@ func TestTermScorer(t *testing.T) {
// test the same thing again (score should be cached this time)
{
termMatch: &index.TermFieldDoc{
ID: "one",
ID: index.IndexInternalID("one"),
Freq: 1,
Norm: 1.0,
},
result: &search.DocumentMatch{
ID: "one",
Score: math.Sqrt(1.0) * idf,
IndexInternalID: index.IndexInternalID("one"),
Score: math.Sqrt(1.0) * idf,
Expl: &search.Explanation{
Value: math.Sqrt(1.0) * idf,
Message: "fieldWeight(desc:beer in one), product of:",
@ -114,13 +114,13 @@ func TestTermScorer(t *testing.T) {
// test a case where the sqrt isn't precalculated
{
termMatch: &index.TermFieldDoc{
ID: "one",
ID: index.IndexInternalID("one"),
Freq: 65,
Norm: 1.0,
},
result: &search.DocumentMatch{
ID: "one",
Score: math.Sqrt(65) * idf,
IndexInternalID: index.IndexInternalID("one"),
Score: math.Sqrt(65) * idf,
Expl: &search.Explanation{
Value: math.Sqrt(65) * idf,
Message: "fieldWeight(desc:beer in one), product of:",
@ -144,7 +144,10 @@ func TestTermScorer(t *testing.T) {
}
for _, test := range tests {
actual := scorer.Score(test.termMatch, nil)
ctx := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(1),
}
actual := scorer.Score(ctx, test.termMatch)
if !reflect.DeepEqual(actual, test.result) {
t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch)
@ -177,13 +180,13 @@ func TestTermScorerWithQueryNorm(t *testing.T) {
}{
{
termMatch: &index.TermFieldDoc{
ID: "one",
ID: index.IndexInternalID("one"),
Freq: 1,
Norm: 1.0,
},
result: &search.DocumentMatch{
ID: "one",
Score: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0,
IndexInternalID: index.IndexInternalID("one"),
Score: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0,
Expl: &search.Explanation{
Value: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0,
Message: "weight(desc:beer^3.000000 in one), product of:",
@ -231,7 +234,10 @@ func TestTermScorerWithQueryNorm(t *testing.T) {
}
for _, test := range tests {
actual := scorer.Score(test.termMatch, nil)
ctx := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(1),
}
actual := scorer.Score(ctx, test.termMatch)
if !reflect.DeepEqual(actual, test.result) {
t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch)

32
search/search.go
View File

@ -9,6 +9,8 @@
package search
import "github.com/blevesearch/bleve/index"
type Location struct {
Pos float64 `json:"pos"`
Start float64 `json:"start"`
@ -51,12 +53,13 @@ type FieldTermLocationMap map[string]TermLocationMap
type FieldFragmentMap map[string][]string
type DocumentMatch struct {
Index string `json:"index,omitempty"`
ID string `json:"id"`
Score float64 `json:"score"`
Expl *Explanation `json:"explanation,omitempty"`
Locations FieldTermLocationMap `json:"locations,omitempty"`
Fragments FieldFragmentMap `json:"fragments,omitempty"`
Index string `json:"index,omitempty"`
ID string `json:"id"`
IndexInternalID index.IndexInternalID `json:"-"`
Score float64 `json:"score"`
Expl *Explanation `json:"explanation,omitempty"`
Locations FieldTermLocationMap `json:"locations,omitempty"`
Fragments FieldFragmentMap `json:"fragments,omitempty"`
// Fields contains the values for document fields listed in
// SearchRequest.Fields. Text fields are returned as strings, numeric
@ -85,8 +88,14 @@ func (dm *DocumentMatch) AddFieldValue(name string, value interface{}) {
dm.Fields[name] = valSlice
}
// Reset allows an already allocated DocumentMatch to be reused
func (dm *DocumentMatch) Reset() *DocumentMatch {
// remember the []byte used for the IndexInternalID
indexInternalId := dm.IndexInternalID
// idiom to copy over from empty DocumentMatch (0 allocations)
*dm = DocumentMatch{}
// reuse the []byte already allocated (and reset len to 0)
dm.IndexInternalID = indexInternalId[:0]
return dm
}
@ -97,11 +106,18 @@ func (c DocumentMatchCollection) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
func (c DocumentMatchCollection) Less(i, j int) bool { return c[i].Score > c[j].Score }
type Searcher interface {
Next(preAllocated *DocumentMatch) (*DocumentMatch, error)
Advance(ID string, preAllocated *DocumentMatch) (*DocumentMatch, error)
Next(ctx *SearchContext) (*DocumentMatch, error)
Advance(ctx *SearchContext, ID index.IndexInternalID) (*DocumentMatch, error)
Close() error
Weight() float64
SetQueryNorm(float64)
Count() uint64
Min() int
DocumentMatchPoolSize() int
}
// SearchContext represents the context around a single search
type SearchContext struct {
DocumentMatchPool *DocumentMatchPool
}

134
search/searchers/search_boolean.go
View File

@ -27,7 +27,7 @@ type BooleanSearcher struct {
currMust *search.DocumentMatch
currShould *search.DocumentMatch
currMustNot *search.DocumentMatch
currentID string
currentID index.IndexInternalID
min uint64
scorer *scorers.ConjunctionQueryScorer
}
@ -66,63 +66,78 @@ func (s *BooleanSearcher) computeQueryNorm() {
}
}
func (s *BooleanSearcher) initSearchers() error {
func (s *BooleanSearcher) initSearchers(ctx *search.SearchContext) error {
var err error
// get all searchers pointing at their first match
if s.mustSearcher != nil {
s.currMust, err = s.mustSearcher.Next(nil)
if s.currMust != nil {
ctx.DocumentMatchPool.Put(s.currMust)
}
s.currMust, err = s.mustSearcher.Next(ctx)
if err != nil {
return err
}
}
if s.shouldSearcher != nil {
s.currShould, err = s.shouldSearcher.Next(nil)
if s.currShould != nil {
ctx.DocumentMatchPool.Put(s.currShould)
}
s.currShould, err = s.shouldSearcher.Next(ctx)
if err != nil {
return err
}
}
if s.mustNotSearcher != nil {
s.currMustNot, err = s.mustNotSearcher.Next(nil)
if s.currMustNot != nil {
ctx.DocumentMatchPool.Put(s.currMustNot)
}
s.currMustNot, err = s.mustNotSearcher.Next(ctx)
if err != nil {
return err
}
}
if s.mustSearcher != nil && s.currMust != nil {
s.currentID = s.currMust.ID
s.currentID = s.currMust.IndexInternalID
} else if s.mustSearcher == nil && s.currShould != nil {
s.currentID = s.currShould.ID
s.currentID = s.currShould.IndexInternalID
} else {
s.currentID = ""
s.currentID = nil
}
s.initialized = true
return nil
}
func (s *BooleanSearcher) advanceNextMust() error {
func (s *BooleanSearcher) advanceNextMust(ctx *search.SearchContext, skipReturn *search.DocumentMatch) error {
var err error
if s.mustSearcher != nil {
s.currMust, err = s.mustSearcher.Next(nil)
if s.currMust != skipReturn {
ctx.DocumentMatchPool.Put(s.currMust)
}
s.currMust, err = s.mustSearcher.Next(ctx)
if err != nil {
return err
}
} else if s.mustSearcher == nil {
s.currShould, err = s.shouldSearcher.Next(nil)
if s.currShould != skipReturn {
ctx.DocumentMatchPool.Put(s.currShould)
}
s.currShould, err = s.shouldSearcher.Next(ctx)
if err != nil {
return err
}
}
if s.mustSearcher != nil && s.currMust != nil {
s.currentID = s.currMust.ID
s.currentID = s.currMust.IndexInternalID
} else if s.mustSearcher == nil && s.currShould != nil {
s.currentID = s.currShould.ID
s.currentID = s.currShould.IndexInternalID
} else {
s.currentID = ""
s.currentID = nil
}
return nil
}
@ -148,10 +163,10 @@ func (s *BooleanSearcher) SetQueryNorm(qnorm float64) {
}
}
func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
func (s *BooleanSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
if !s.initialized {
err := s.initSearchers()
err := s.initSearchers(ctx)
if err != nil {
return nil, err
}
@ -160,37 +175,43 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu
var err error
var rv *search.DocumentMatch
for s.currentID != "" {
if s.currMustNot != nil && s.currMustNot.ID < s.currentID {
for s.currentID != nil {
if s.currMustNot != nil && s.currMustNot.IndexInternalID.Compare(s.currentID) < 0 {
if s.currMustNot != nil {
ctx.DocumentMatchPool.Put(s.currMustNot)
}
// advance must not searcher to our candidate entry
s.currMustNot, err = s.mustNotSearcher.Advance(s.currentID, nil)
s.currMustNot, err = s.mustNotSearcher.Advance(ctx, s.currentID)
if err != nil {
return nil, err
}
if s.currMustNot != nil && s.currMustNot.ID == s.currentID {
if s.currMustNot != nil && s.currMustNot.IndexInternalID.Equals(s.currentID) {
// the candidate is excluded
err = s.advanceNextMust()
err = s.advanceNextMust(ctx, nil)
if err != nil {
return nil, err
}
continue
}
} else if s.currMustNot != nil && s.currMustNot.ID == s.currentID {
} else if s.currMustNot != nil && s.currMustNot.IndexInternalID.Equals(s.currentID) {
// the candidate is excluded
err = s.advanceNextMust()
err = s.advanceNextMust(ctx, nil)
if err != nil {
return nil, err
}
continue
}
if s.currShould != nil && s.currShould.ID < s.currentID {
if s.currShould != nil && s.currShould.IndexInternalID.Compare(s.currentID) < 0 {
// advance should searcher to our candidate entry
s.currShould, err = s.shouldSearcher.Advance(s.currentID, nil)
if s.currShould != nil {
ctx.DocumentMatchPool.Put(s.currShould)
}
s.currShould, err = s.shouldSearcher.Advance(ctx, s.currentID)
if err != nil {
return nil, err
}
if s.currShould != nil && s.currShould.ID == s.currentID {
if s.currShould != nil && s.currShould.IndexInternalID.Equals(s.currentID) {
// score bonus matches should
var cons []*search.DocumentMatch
if s.currMust != nil {
@ -203,22 +224,22 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu
s.currShould,
}
}
rv = s.scorer.Score(cons)
err = s.advanceNextMust()
rv = s.scorer.Score(ctx, cons)
err = s.advanceNextMust(ctx, rv)
if err != nil {
return nil, err
}
break
} else if s.shouldSearcher.Min() == 0 {
// match is OK anyway
rv = s.scorer.Score([]*search.DocumentMatch{s.currMust})
err = s.advanceNextMust()
rv = s.scorer.Score(ctx, []*search.DocumentMatch{s.currMust})
err = s.advanceNextMust(ctx, rv)
if err != nil {
return nil, err
}
break
}
} else if s.currShould != nil && s.currShould.ID == s.currentID {
} else if s.currShould != nil && s.currShould.IndexInternalID.Equals(s.currentID) {
// score bonus matches should
var cons []*search.DocumentMatch
if s.currMust != nil {
@ -231,23 +252,23 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu
s.currShould,
}
}
rv = s.scorer.Score(cons)
err = s.advanceNextMust()
rv = s.scorer.Score(ctx, cons)
err = s.advanceNextMust(ctx, rv)
if err != nil {
return nil, err
}
break
} else if s.shouldSearcher == nil || s.shouldSearcher.Min() == 0 {
// match is OK anyway
rv = s.scorer.Score([]*search.DocumentMatch{s.currMust})
err = s.advanceNextMust()
rv = s.scorer.Score(ctx, []*search.DocumentMatch{s.currMust})
err = s.advanceNextMust(ctx, rv)
if err != nil {
return nil, err
}
break
}
err = s.advanceNextMust()
err = s.advanceNextMust(ctx, nil)
if err != nil {
return nil, err
}
@ -255,10 +276,10 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu
return rv, nil
}
func (s *BooleanSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
func (s *BooleanSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
if !s.initialized {
err := s.initSearchers()
err := s.initSearchers(ctx)
if err != nil {
return nil, err
}
@ -266,33 +287,42 @@ func (s *BooleanSearcher) Advance(ID string, preAllocated *search.DocumentMatch)
var err error
if s.mustSearcher != nil {
s.currMust, err = s.mustSearcher.Advance(ID, nil)
if s.currMust != nil {
ctx.DocumentMatchPool.Put(s.currMust)
}
s.currMust, err = s.mustSearcher.Advance(ctx, ID)
if err != nil {
return nil, err
}
}
if s.shouldSearcher != nil {
s.currShould, err = s.shouldSearcher.Advance(ID, nil)
if s.currShould != nil {
ctx.DocumentMatchPool.Put(s.currShould)
}
s.currShould, err = s.shouldSearcher.Advance(ctx, ID)
if err != nil {
return nil, err
}
}
if s.mustNotSearcher != nil {
s.currMustNot, err = s.mustNotSearcher.Advance(ID, nil)
if s.currMustNot != nil {
ctx.DocumentMatchPool.Put(s.currMustNot)
}
s.currMustNot, err = s.mustNotSearcher.Advance(ctx, ID)
if err != nil {
return nil, err
}
}
if s.mustSearcher != nil && s.currMust != nil {
s.currentID = s.currMust.ID
s.currentID = s.currMust.IndexInternalID
} else if s.mustSearcher == nil && s.currShould != nil {
s.currentID = s.currShould.ID
s.currentID = s.currShould.IndexInternalID
} else {
s.currentID = ""
s.currentID = nil
}
return s.Next(preAllocated)
return s.Next(ctx)
}
func (s *BooleanSearcher) Count() uint64 {
@ -333,3 +363,17 @@ func (s *BooleanSearcher) Close() error {
func (s *BooleanSearcher) Min() int {
return 0
}
func (s *BooleanSearcher) DocumentMatchPoolSize() int {
rv := 3
if s.mustSearcher != nil {
rv += s.mustSearcher.DocumentMatchPoolSize()
}
if s.shouldSearcher != nil {
rv += s.shouldSearcher.DocumentMatchPoolSize()
}
if s.mustNotSearcher != nil {
rv += s.mustNotSearcher.DocumentMatchPoolSize()
}
return rv
}

61
search/searchers/search_boolean_test.go
View File

@ -12,6 +12,7 @@ package searchers
import (
"testing"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/search"
)
@ -248,16 +249,16 @@ func TestBooleanSearch(t *testing.T) {
searcher: booleanSearcher,
results: []*search.DocumentMatch{
{
ID: "1",
Score: 0.9818005051949021,
IndexInternalID: index.IndexInternalID("1"),
Score: 0.9818005051949021,
},
{
ID: "3",
Score: 0.808709699395535,
IndexInternalID: index.IndexInternalID("3"),
Score: 0.808709699395535,
},
{
ID: "4",
Score: 0.34618161159873423,
IndexInternalID: index.IndexInternalID("4"),
Score: 0.34618161159873423,
},
},
},
@ -265,12 +266,12 @@ func TestBooleanSearch(t *testing.T) {
searcher: booleanSearcher2,
results: []*search.DocumentMatch{
{
ID: "1",
Score: 0.6775110856165737,
IndexInternalID: index.IndexInternalID("1"),
Score: 0.6775110856165737,
},
{
ID: "3",
Score: 0.6775110856165737,
IndexInternalID: index.IndexInternalID("3"),
Score: 0.6775110856165737,
},
},
},
@ -283,16 +284,16 @@ func TestBooleanSearch(t *testing.T) {
searcher: booleanSearcher4,
results: []*search.DocumentMatch{
{
ID: "1",
Score: 1.0,
IndexInternalID: index.IndexInternalID("1"),
Score: 1.0,
},
{
ID: "3",
Score: 0.5,
IndexInternalID: index.IndexInternalID("3"),
Score: 0.5,
},
{
ID: "4",
Score: 1.0,
IndexInternalID: index.IndexInternalID("4"),
Score: 1.0,
},
},
},
@ -300,12 +301,12 @@ func TestBooleanSearch(t *testing.T) {
searcher: booleanSearcher5,
results: []*search.DocumentMatch{
{
ID: "3",
Score: 0.5,
IndexInternalID: index.IndexInternalID("3"),
Score: 0.5,
},
{
ID: "4",
Score: 1.0,
IndexInternalID: index.IndexInternalID("4"),
Score: 1.0,
},
},
},
@ -318,8 +319,8 @@ func TestBooleanSearch(t *testing.T) {
searcher: conjunctionSearcher7,
results: []*search.DocumentMatch{
{
ID: "1",
Score: 2.0097428702814377,
IndexInternalID: index.IndexInternalID("1"),
Score: 2.0097428702814377,
},
},
},
@ -327,8 +328,8 @@ func TestBooleanSearch(t *testing.T) {
searcher: conjunctionSearcher8,
results: []*search.DocumentMatch{
{
ID: "3",
Score: 2.0681575785068107,
IndexInternalID: index.IndexInternalID("3"),
Score: 2.0681575785068107,
},
},
},
@ -342,19 +343,23 @@ func TestBooleanSearch(t *testing.T) {
}
}()
next, err := test.searcher.Next(nil)
ctx := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()),
}
next, err := test.searcher.Next(ctx)
i := 0
for err == nil && next != nil {
if i < len(test.results) {
if next.ID != test.results[i].ID {
t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex)
if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) {
t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex)
}
if !scoresCloseEnough(next.Score, test.results[i].Score) {
t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex)
t.Logf("scoring explanation: %s", next.Expl)
}
}
next, err = test.searcher.Next(nil)
ctx.DocumentMatchPool.Put(next)
next, err = test.searcher.Next(ctx)
i++
}
if err != nil {

79
search/searchers/search_conjunction.go
View File

@ -25,7 +25,7 @@ type ConjunctionSearcher struct {
explain bool
queryNorm float64
currs []*search.DocumentMatch
currentID string
currentID index.IndexInternalID
scorer *scorers.ConjunctionQueryScorer
}
@ -63,11 +63,14 @@ func (s *ConjunctionSearcher) computeQueryNorm() {
}
}
func (s *ConjunctionSearcher) initSearchers() error {
func (s *ConjunctionSearcher) initSearchers(ctx *search.SearchContext) error {
var err error
// get all searchers pointing at their first match
for i, termSearcher := range s.searchers {
s.currs[i], err = termSearcher.Next(nil)
if s.currs[i] != nil {
ctx.DocumentMatchPool.Put(s.currs[i])
}
s.currs[i], err = termSearcher.Next(ctx)
if err != nil {
return err
}
@ -75,9 +78,9 @@ func (s *ConjunctionSearcher) initSearchers() error {
if len(s.currs) > 0 {
if s.currs[0] != nil {
s.currentID = s.currs[0].ID
s.currentID = s.currs[0].IndexInternalID
} else {
s.currentID = ""
s.currentID = nil
}
}
@ -99,9 +102,9 @@ func (s *ConjunctionSearcher) SetQueryNorm(qnorm float64) {
}
}
func (s *ConjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
func (s *ConjunctionSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
if !s.initialized {
err := s.initSearchers()
err := s.initSearchers(ctx)
if err != nil {
return nil, err
}
@ -109,68 +112,82 @@ func (s *ConjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search.
var rv *search.DocumentMatch
var err error
OUTER:
for s.currentID != "" {
for s.currentID != nil {
for i, termSearcher := range s.searchers {
if s.currs[i] != nil && s.currs[i].ID != s.currentID {
if s.currentID < s.currs[i].ID {
s.currentID = s.currs[i].ID
if s.currs[i] != nil && !s.currs[i].IndexInternalID.Equals(s.currentID) {
if s.currentID.Compare(s.currs[i].IndexInternalID) < 0 {
s.currentID = s.currs[i].IndexInternalID
continue OUTER
}
// this reader doesn't have the currentID, try to advance
s.currs[i], err = termSearcher.Advance(s.currentID, nil)
if s.currs[i] != nil {
ctx.DocumentMatchPool.Put(s.currs[i])
}
s.currs[i], err = termSearcher.Advance(ctx, s.currentID)
if err != nil {
return nil, err
}
if s.currs[i] == nil {
s.currentID = ""
s.currentID = nil
continue OUTER
}
if s.currs[i].ID != s.currentID {
if !s.currs[i].IndexInternalID.Equals(s.currentID) {
// we just advanced, so it doesn't match, it must be greater
// no need to call next
s.currentID = s.currs[i].ID
s.currentID = s.currs[i].IndexInternalID
continue OUTER
}
} else if s.currs[i] == nil {
s.currentID = ""
s.currentID = nil
continue OUTER
}
}
// if we get here, a doc matched all readers, sum the score and add it
rv = s.scorer.Score(s.currs)
rv = s.scorer.Score(ctx, s.currs)
// prepare for next entry
s.currs[0], err = s.searchers[0].Next(nil)
if err != nil {
return nil, err
// we know all the searchers are pointing at the same thing
// so they all need to be advanced
for i, termSearcher := range s.searchers {
if s.currs[i] != rv {
ctx.DocumentMatchPool.Put(s.currs[i])
}
s.currs[i], err = termSearcher.Next(ctx)
if err != nil {
return nil, err
}
}
if s.currs[0] == nil {
s.currentID = ""
s.currentID = nil
} else {
s.currentID = s.currs[0].ID
s.currentID = s.currs[0].IndexInternalID
}
// don't continue now, wait for the next call to Next()
break
}
return rv, nil
}
func (s *ConjunctionSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
func (s *ConjunctionSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
if !s.initialized {
err := s.initSearchers()
err := s.initSearchers(ctx)
if err != nil {
return nil, err
}
}
var err error
for i, searcher := range s.searchers {
s.currs[i], err = searcher.Advance(ID, nil)
if s.currs[i] != nil {
ctx.DocumentMatchPool.Put(s.currs[i])
}
s.currs[i], err = searcher.Advance(ctx, ID)
if err != nil {
return nil, err
}
}
s.currentID = ID
return s.Next(preAllocated)
return s.Next(ctx)
}
func (s *ConjunctionSearcher) Count() uint64 {
@ -195,3 +212,11 @@ func (s *ConjunctionSearcher) Close() error {
func (s *ConjunctionSearcher) Min() int {
return 0
}
func (s *ConjunctionSearcher) DocumentMatchPoolSize() int {
rv := len(s.currs)
for _, s := range s.searchers {
rv += s.DocumentMatchPoolSize()
}
return rv
}

36
search/searchers/search_conjunction_test.go
View File

@ -12,6 +12,7 @@ package searchers
import (
"testing"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/search"
)
@ -128,8 +129,8 @@ func TestConjunctionSearch(t *testing.T) {
searcher: beerAndMartySearcher,
results: []*search.DocumentMatch{
{
ID: "1",
Score: 2.0097428702814377,
IndexInternalID: index.IndexInternalID("1"),
Score: 2.0097428702814377,
},
},
},
@ -137,8 +138,8 @@ func TestConjunctionSearch(t *testing.T) {
searcher: angstAndBeerSearcher,
results: []*search.DocumentMatch{
{
ID: "2",
Score: 1.0807601687084403,
IndexInternalID: index.IndexInternalID("2"),
Score: 1.0807601687084403,
},
},
},
@ -150,12 +151,12 @@ func TestConjunctionSearch(t *testing.T) {
searcher: beerAndMisterSearcher,
results: []*search.DocumentMatch{
{
ID: "2",
Score: 1.2877980334016337,
IndexInternalID: index.IndexInternalID("2"),
Score: 1.2877980334016337,
},
{
ID: "3",
Score: 1.2877980334016337,
IndexInternalID: index.IndexInternalID("3"),
Score: 1.2877980334016337,
},
},
},
@ -163,8 +164,8 @@ func TestConjunctionSearch(t *testing.T) {
searcher: couchbaseAndMisterSearcher,
results: []*search.DocumentMatch{
{
ID: "2",
Score: 1.4436599157093672,
IndexInternalID: index.IndexInternalID("2"),
Score: 1.4436599157093672,
},
},
},
@ -172,8 +173,8 @@ func TestConjunctionSearch(t *testing.T) {
searcher: beerAndCouchbaseAndMisterSearcher,
results: []*search.DocumentMatch{
{
ID: "2",
Score: 1.441614953806971,
IndexInternalID: index.IndexInternalID("2"),
Score: 1.441614953806971,
},
},
},
@ -187,19 +188,22 @@ func TestConjunctionSearch(t *testing.T) {
}
}()
next, err := test.searcher.Next(nil)
ctx := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(10),
}
next, err := test.searcher.Next(ctx)
i := 0
for err == nil && next != nil {
if i < len(test.results) {
if next.ID != test.results[i].ID {
t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex)
if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) {
t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex)
}
if !scoresCloseEnough(next.Score, test.results[i].Score) {
t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex)
t.Logf("scoring explanation: %s", next.Expl)
}
}
next, err = test.searcher.Next(nil)
next, err = test.searcher.Next(ctx)
i++
}
if err != nil {

53
search/searchers/search_disjunction.go
View File

@ -30,7 +30,7 @@ type DisjunctionSearcher struct {
searchers OrderedSearcherList
queryNorm float64
currs []*search.DocumentMatch
currentID string
currentID index.IndexInternalID
scorer *scorers.DisjunctionQueryScorer
min float64
}
@ -83,11 +83,14 @@ func (s *DisjunctionSearcher) computeQueryNorm() {
}
}
func (s *DisjunctionSearcher) initSearchers() error {
func (s *DisjunctionSearcher) initSearchers(ctx *search.SearchContext) error {
var err error
// get all searchers pointing at their first match
for i, termSearcher := range s.searchers {
s.currs[i], err = termSearcher.Next(nil)
if s.currs[i] != nil {
ctx.DocumentMatchPool.Put(s.currs[i])
}
s.currs[i], err = termSearcher.Next(ctx)
if err != nil {
return err
}
@ -98,11 +101,11 @@ func (s *DisjunctionSearcher) initSearchers() error {
return nil
}
func (s *DisjunctionSearcher) nextSmallestID() string {
rv := ""
func (s *DisjunctionSearcher) nextSmallestID() index.IndexInternalID {
var rv index.IndexInternalID
for _, curr := range s.currs {
if curr != nil && (curr.ID < rv || rv == "") {
rv = curr.ID
if curr != nil && (curr.IndexInternalID.Compare(rv) < 0 || rv == nil) {
rv = curr.IndexInternalID
}
}
return rv
@ -122,9 +125,9 @@ func (s *DisjunctionSearcher) SetQueryNorm(qnorm float64) {
}
}
func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
func (s *DisjunctionSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
if !s.initialized {
err := s.initSearchers()
err := s.initSearchers(ctx)
if err != nil {
return nil, err
}
@ -134,9 +137,9 @@ func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search.
matching := make([]*search.DocumentMatch, 0, len(s.searchers))
found := false
for !found && s.currentID != "" {
for !found && s.currentID != nil {
for _, curr := range s.currs {
if curr != nil && curr.ID == s.currentID {
if curr != nil && curr.IndexInternalID.Equals(s.currentID) {
matching = append(matching, curr)
}
}
@ -144,16 +147,19 @@ func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search.
if len(matching) >= int(s.min) {
found = true
// score this match
rv = s.scorer.Score(matching, len(matching), len(s.searchers))
rv = s.scorer.Score(ctx, matching, len(matching), len(s.searchers))
}
// reset matching
matching = make([]*search.DocumentMatch, 0)
// invoke next on all the matching searchers
for i, curr := range s.currs {
if curr != nil && curr.ID == s.currentID {
if curr != nil && curr.IndexInternalID.Equals(s.currentID) {
searcher := s.searchers[i]
s.currs[i], err = searcher.Next(nil)
if s.currs[i] != rv {
ctx.DocumentMatchPool.Put(s.currs[i])
}
s.currs[i], err = searcher.Next(ctx)
if err != nil {
return nil, err
}
@ -164,9 +170,9 @@ func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search.
return rv, nil
}
func (s *DisjunctionSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
func (s *DisjunctionSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
if !s.initialized {
err := s.initSearchers()
err := s.initSearchers(ctx)
if err != nil {
return nil, err
}
@ -174,7 +180,10 @@ func (s *DisjunctionSearcher) Advance(ID string, preAllocated *search.DocumentMa
// get all searchers pointing at their first match
var err error
for i, termSearcher := range s.searchers {
s.currs[i], err = termSearcher.Advance(ID, nil)
if s.currs[i] != nil {
ctx.DocumentMatchPool.Put(s.currs[i])
}
s.currs[i], err = termSearcher.Advance(ctx, ID)
if err != nil {
return nil, err
}
@ -182,7 +191,7 @@ func (s *DisjunctionSearcher) Advance(ID string, preAllocated *search.DocumentMa
s.currentID = s.nextSmallestID()
return s.Next(preAllocated)
return s.Next(ctx)
}
func (s *DisjunctionSearcher) Count() uint64 {
@ -207,3 +216,11 @@ func (s *DisjunctionSearcher) Close() error {
func (s *DisjunctionSearcher) Min() int {
return int(s.min) // FIXME just make this an int
}
func (s *DisjunctionSearcher) DocumentMatchPoolSize() int {
rv := len(s.currs)
for _, s := range s.searchers {
rv += s.DocumentMatchPoolSize()
}
return rv
}

38
search/searchers/search_disjunction_test.go
View File

@ -12,6 +12,7 @@ package searchers
import (
"testing"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/search"
)
@ -71,12 +72,12 @@ func TestDisjunctionSearch(t *testing.T) {
searcher: martyOrDustinSearcher,
results: []*search.DocumentMatch{
{
ID: "1",
Score: 0.6775110856165737,
IndexInternalID: index.IndexInternalID("1"),
Score: 0.6775110856165737,
},
{
ID: "3",
Score: 0.6775110856165737,
IndexInternalID: index.IndexInternalID("3"),
Score: 0.6775110856165737,
},
},
},
@ -85,16 +86,16 @@ func TestDisjunctionSearch(t *testing.T) {
searcher: nestedRaviOrMartyOrDustinSearcher,
results: []*search.DocumentMatch{
{
ID: "1",
Score: 0.2765927424732821,
IndexInternalID: index.IndexInternalID("1"),
Score: 0.2765927424732821,
},
{
ID: "3",
Score: 0.2765927424732821,
IndexInternalID: index.IndexInternalID("3"),
Score: 0.2765927424732821,
},
{
ID: "4",
Score: 0.5531854849465642,
IndexInternalID: index.IndexInternalID("4"),
Score: 0.5531854849465642,
},
},
},
@ -108,19 +109,23 @@ func TestDisjunctionSearch(t *testing.T) {
}
}()
next, err := test.searcher.Next(nil)
ctx := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()),
}
next, err := test.searcher.Next(ctx)
i := 0
for err == nil && next != nil {
if i < len(test.results) {
if next.ID != test.results[i].ID {
t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex)
if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) {
t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex)
}
if !scoresCloseEnough(next.Score, test.results[i].Score) {
t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex)
t.Logf("scoring explanation: %s", next.Expl)
}
}
next, err = test.searcher.Next(nil)
ctx.DocumentMatchPool.Put(next)
next, err = test.searcher.Next(ctx)
i++
}
if err != nil {
@ -158,7 +163,10 @@ func TestDisjunctionAdvance(t *testing.T) {
t.Fatal(err)
}
match, err := martyOrDustinSearcher.Advance("3", nil)
ctx := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(martyOrDustinSearcher.DocumentMatchPoolSize()),
}
match, err := martyOrDustinSearcher.Advance(ctx, index.IndexInternalID("3"))
if err != nil {
t.Errorf("unexpected error: %v", err)
}

78
search/searchers/search_docid.go
View File

@ -10,8 +10,6 @@
package searchers
import (
"sort"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/scorers"
@ -19,54 +17,28 @@ import (
// DocIDSearcher returns documents matching a predefined set of identifiers.
type DocIDSearcher struct {
ids []string
current int
scorer *scorers.ConstantScorer
reader index.DocIDReader
scorer *scorers.ConstantScorer
count int
}
func NewDocIDSearcher(indexReader index.IndexReader, ids []string, boost float64,
explain bool) (searcher *DocIDSearcher, err error) {
kept := make([]string, len(ids))
copy(kept, ids)
sort.Strings(kept)
if len(ids) > 0 {
var idReader index.DocIDReader
endTerm := string(incrementBytes([]byte(kept[len(kept)-1])))
idReader, err = indexReader.DocIDReader(kept[0], endTerm)
if err != nil {
return nil, err
}
defer func() {
if cerr := idReader.Close(); err == nil && cerr != nil {
err = cerr
}
}()
j := 0
for _, id := range kept {
doc, err := idReader.Advance(id)
if err != nil {
return nil, err
}
// Non-duplicate match
if doc == id && (j == 0 || kept[j-1] != id) {
kept[j] = id
j++
}
}
kept = kept[:j]
reader, err := indexReader.DocIDReaderOnly(ids)
if err != nil {
return nil, err
}
scorer := scorers.NewConstantScorer(1.0, boost, explain)
return &DocIDSearcher{
ids: kept,
scorer: scorer,
reader: reader,
count: len(ids),
}, nil
}
func (s *DocIDSearcher) Count() uint64 {
return uint64(len(s.ids))
return uint64(s.count)
}
func (s *DocIDSearcher) Weight() float64 {
@ -77,20 +49,30 @@ func (s *DocIDSearcher) SetQueryNorm(qnorm float64) {
s.scorer.SetQueryNorm(qnorm)
}
func (s *DocIDSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
if s.current >= len(s.ids) {
func (s *DocIDSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
docidMatch, err := s.reader.Next()
if err != nil {
return nil, err
}
if docidMatch == nil {
return nil, nil
}
id := s.ids[s.current]
s.current++
docMatch := s.scorer.Score(id)
return docMatch, nil
docMatch := s.scorer.Score(ctx, docidMatch)
return docMatch, nil
}
func (s *DocIDSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
s.current = sort.SearchStrings(s.ids, ID)
return s.Next(preAllocated)
func (s *DocIDSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
docidMatch, err := s.reader.Advance(ID)
if err != nil {
return nil, err
}
if docidMatch == nil {
return nil, nil
}
docMatch := s.scorer.Score(ctx, docidMatch)
return docMatch, nil
}
func (s *DocIDSearcher) Close() error {
@ -100,3 +82,7 @@ func (s *DocIDSearcher) Close() error {
func (s *DocIDSearcher) Min() int {
return 0
}
func (s *DocIDSearcher) DocumentMatchPoolSize() int {
return 1
}

25
search/searchers/search_docid_test.go
View File

@ -16,6 +16,7 @@ import (
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store/gtreap"
"github.com/blevesearch/bleve/index/upside_down"
"github.com/blevesearch/bleve/search"
)
func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) {
@ -62,27 +63,29 @@ func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) {
}
}()
if searcher.Count() != uint64(len(wanted)) {
t.Fatalf("expected count %v got %v", len(wanted), searcher.Count())
ctx := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(searcher.DocumentMatchPoolSize()),
}
// Check the sequence
for i, id := range wanted {
m, err := searcher.Next(nil)
m, err := searcher.Next(ctx)
if err != nil {
t.Fatal(err)
}
if id != m.ID {
t.Fatalf("expected %v at position %v, got %v", id, i, m.ID)
if !index.IndexInternalID(id).Equals(m.IndexInternalID) {
t.Fatalf("expected %v at position %v, got %v", id, i, m.IndexInternalID)
}
ctx.DocumentMatchPool.Put(m)
}
m, err := searcher.Next(nil)
m, err := searcher.Next(ctx)
if err != nil {
t.Fatal(err)
}
if m != nil {
t.Fatalf("expected nil past the end of the sequence, got %v", m.ID)
t.Fatalf("expected nil past the end of the sequence, got %v", m.IndexInternalID)
}
ctx.DocumentMatchPool.Put(m)
// Check seeking
for _, id := range wanted {
@ -91,24 +94,26 @@ func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) {
}
before := id[:1]
for _, target := range []string{before, id} {
m, err := searcher.Advance(target, nil)
m, err := searcher.Advance(ctx, index.IndexInternalID(target))
if err != nil {
t.Fatal(err)
}
if m == nil || m.ID != id {
if m == nil || !m.IndexInternalID.Equals(index.IndexInternalID(id)) {
t.Fatalf("advancing to %v returned %v instead of %v", before, m, id)
}
ctx.DocumentMatchPool.Put(m)
}
}
// Seek after the end of the sequence
after := "zzz"
m, err = searcher.Advance(after, nil)
m, err = searcher.Advance(ctx, index.IndexInternalID(after))
if err != nil {
t.Fatal(err)
}
if m != nil {
t.Fatalf("advancing past the end of the sequence should return nil, got %v", m)
}
ctx.DocumentMatchPool.Put(m)
}
func TestDocIDSearcherEmptySearchEmptyIndex(t *testing.T) {

12
search/searchers/search_fuzzy.go
View File

@ -107,13 +107,13 @@ func (s *FuzzySearcher) SetQueryNorm(qnorm float64) {
s.searcher.SetQueryNorm(qnorm)
}
func (s *FuzzySearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
return s.searcher.Next(preAllocated)
func (s *FuzzySearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
return s.searcher.Next(ctx)
}
func (s *FuzzySearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
return s.searcher.Advance(ID, preAllocated)
func (s *FuzzySearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
return s.searcher.Advance(ctx, ID)
}
func (s *FuzzySearcher) Close() error {
@ -123,3 +123,7 @@ func (s *FuzzySearcher) Close() error {
func (s *FuzzySearcher) Min() int {
return 0
}
func (s *FuzzySearcher) DocumentMatchPoolSize() int {
return s.searcher.DocumentMatchPoolSize()
}

37
search/searchers/search_fuzzy_test.go
View File

@ -12,6 +12,7 @@ package searchers
import (
"testing"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/search"
)
@ -56,20 +57,20 @@ func TestFuzzySearch(t *testing.T) {
searcher: fuzzySearcherbeet,
results: []*search.DocumentMatch{
{
ID: "1",
Score: 1.0,
IndexInternalID: index.IndexInternalID("1"),
Score: 1.0,
},
{
ID: "2",
Score: 0.5,
IndexInternalID: index.IndexInternalID("2"),
Score: 0.5,
},
{
ID: "3",
Score: 0.5,
IndexInternalID: index.IndexInternalID("3"),
Score: 0.5,
},
{
ID: "4",
Score: 0.9999999838027345,
IndexInternalID: index.IndexInternalID("4"),
Score: 0.9999999838027345,
},
},
},
@ -81,8 +82,8 @@ func TestFuzzySearch(t *testing.T) {
searcher: fuzzySearcheraplee,
results: []*search.DocumentMatch{
{
ID: "3",
Score: 0.9581453659370776,
IndexInternalID: index.IndexInternalID("3"),
Score: 0.9581453659370776,
},
},
},
@ -90,8 +91,8 @@ func TestFuzzySearch(t *testing.T) {
searcher: fuzzySearcherprefix,
results: []*search.DocumentMatch{
{
ID: "5",
Score: 1.916290731874155,
IndexInternalID: index.IndexInternalID("5"),
Score: 1.916290731874155,
},
},
},
@ -105,19 +106,23 @@ func TestFuzzySearch(t *testing.T) {
}
}()
next, err := test.searcher.Next(nil)
ctx := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()),
}
next, err := test.searcher.Next(ctx)
i := 0
for err == nil && next != nil {
if i < len(test.results) {
if next.ID != test.results[i].ID {
t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex)
if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) {
t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex)
}
if next.Score != test.results[i].Score {
t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex)
t.Logf("scoring explanation: %s", next.Expl)
}
}
next, err = test.searcher.Next(nil)
ctx.DocumentMatchPool.Put(next)
next, err = test.searcher.Next(ctx)
i++
}
if err != nil {

16
search/searchers/search_match_all.go
View File

@ -46,35 +46,35 @@ func (s *MatchAllSearcher) SetQueryNorm(qnorm float64) {
s.scorer.SetQueryNorm(qnorm)
}
func (s *MatchAllSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
func (s *MatchAllSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
id, err := s.reader.Next()
if err != nil {
return nil, err
}
if id == "" {
if id == nil {
return nil, nil
}
// score match
docMatch := s.scorer.Score(id)
docMatch := s.scorer.Score(ctx, id)
// return doc match
return docMatch, nil
}
func (s *MatchAllSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
func (s *MatchAllSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
id, err := s.reader.Advance(ID)
if err != nil {
return nil, err
}
if id == "" {
if id == nil {
return nil, nil
}
// score match
docMatch := s.scorer.Score(id)
docMatch := s.scorer.Score(ctx, id)
// return doc match
return docMatch, nil
@ -87,3 +87,7 @@ func (s *MatchAllSearcher) Close() error {
func (s *MatchAllSearcher) Min() int {
return 0
}
func (s *MatchAllSearcher) DocumentMatchPoolSize() int {
return 1
}

53
search/searchers/search_match_all_test.go
View File

@ -12,6 +12,7 @@ package searchers
import (
"testing"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/search"
)
@ -48,24 +49,24 @@ func TestMatchAllSearch(t *testing.T) {
queryNorm: 1.0,
results: []*search.DocumentMatch{
{
ID: "1",
Score: 1.0,
IndexInternalID: index.IndexInternalID("1"),
Score: 1.0,
},
{
ID: "2",
Score: 1.0,
IndexInternalID: index.IndexInternalID("2"),
Score: 1.0,
},
{
ID: "3",
Score: 1.0,
IndexInternalID: index.IndexInternalID("3"),
Score: 1.0,
},
{
ID: "4",
Score: 1.0,
IndexInternalID: index.IndexInternalID("4"),
Score: 1.0,
},
{
ID: "5",
Score: 1.0,
IndexInternalID: index.IndexInternalID("5"),
Score: 1.0,
},
},
},
@ -74,24 +75,24 @@ func TestMatchAllSearch(t *testing.T) {
queryNorm: 0.8333333,
results: []*search.DocumentMatch{
{
ID: "1",
Score: 1.0,
IndexInternalID: index.IndexInternalID("1"),
Score: 1.0,
},
{
ID: "2",
Score: 1.0,
IndexInternalID: index.IndexInternalID("2"),
Score: 1.0,
},
{
ID: "3",
Score: 1.0,
IndexInternalID: index.IndexInternalID("3"),
Score: 1.0,
},
{
ID: "4",
Score: 1.0,
IndexInternalID: index.IndexInternalID("4"),
Score: 1.0,
},
{
ID: "5",
Score: 1.0,
IndexInternalID: index.IndexInternalID("5"),
Score: 1.0,
},
},
},
@ -109,19 +110,23 @@ func TestMatchAllSearch(t *testing.T) {
}
}()
next, err := test.searcher.Next(nil)
ctx := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()),
}
next, err := test.searcher.Next(ctx)
i := 0
for err == nil && next != nil {
if i < len(test.results) {
if next.ID != test.results[i].ID {
t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex)
if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) {
t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex)
}
if !scoresCloseEnough(next.Score, test.results[i].Score) {
t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex)
t.Logf("scoring explanation: %s", next.Expl)
}
}
next, err = test.searcher.Next(nil)
ctx.DocumentMatchPool.Put(next)
next, err = test.searcher.Next(ctx)
i++
}
if err != nil {

8
search/searchers/search_match_none.go
View File

@ -36,11 +36,11 @@ func (s *MatchNoneSearcher) SetQueryNorm(qnorm float64) {
}
func (s *MatchNoneSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
func (s *MatchNoneSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
return nil, nil
}
func (s *MatchNoneSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
func (s *MatchNoneSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
return nil, nil
}
@ -51,3 +51,7 @@ func (s *MatchNoneSearcher) Close() error {
func (s *MatchNoneSearcher) Min() int {
return 0
}
func (s *MatchNoneSearcher) DocumentMatchPoolSize() int {
return 0
}

12
search/searchers/search_match_none_test.go
View File

@ -51,19 +51,23 @@ func TestMatchNoneSearch(t *testing.T) {
}
}()
next, err := test.searcher.Next(nil)
ctx := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()),
}
next, err := test.searcher.Next(ctx)
i := 0
for err == nil && next != nil {
if i < len(test.results) {
if next.ID != test.results[i].ID {
t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex)
if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) {
t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex)
}
if !scoresCloseEnough(next.Score, test.results[i].Score) {
t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex)
t.Logf("scoring explanation: %s", next.Expl)
}
}
next, err = test.searcher.Next(nil)
ctx.DocumentMatchPool.Put(next)
next, err = test.searcher.Next(ctx)
i++
}
if err != nil {

12
search/searchers/search_numeric_range.go
View File

@ -96,12 +96,12 @@ func (s *NumericRangeSearcher) SetQueryNorm(qnorm float64) {
s.searcher.SetQueryNorm(qnorm)
}
func (s *NumericRangeSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
return s.searcher.Next(preAllocated)
func (s *NumericRangeSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
return s.searcher.Next(ctx)
}
func (s *NumericRangeSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
return s.searcher.Advance(ID, preAllocated)
func (s *NumericRangeSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
return s.searcher.Advance(ctx, ID)
}
func (s *NumericRangeSearcher) Close() error {
@ -215,3 +215,7 @@ func newRangeBytes(minBytes, maxBytes []byte) *termRange {
func (s *NumericRangeSearcher) Min() int {
return 0
}
func (s *NumericRangeSearcher) DocumentMatchPoolSize() int {
return s.searcher.DocumentMatchPoolSize()
}

28
search/searchers/search_phrase.go
View File

@ -52,11 +52,11 @@ func (s *PhraseSearcher) computeQueryNorm() {
}
}
func (s *PhraseSearcher) initSearchers() error {
func (s *PhraseSearcher) initSearchers(ctx *search.SearchContext) error {
var err error
// get all searchers pointing at their first match
if s.mustSearcher != nil {
s.currMust, err = s.mustSearcher.Next(nil)
s.currMust, err = s.mustSearcher.Next(ctx)
if err != nil {
return err
}
@ -66,11 +66,11 @@ func (s *PhraseSearcher) initSearchers() error {
return nil
}
func (s *PhraseSearcher) advanceNextMust() error {
func (s *PhraseSearcher) advanceNextMust(ctx *search.SearchContext) error {
var err error
if s.mustSearcher != nil {
s.currMust, err = s.mustSearcher.Next(nil)
s.currMust, err = s.mustSearcher.Next(ctx)
if err != nil {
return err
}
@ -90,9 +90,9 @@ func (s *PhraseSearcher) SetQueryNorm(qnorm float64) {
s.mustSearcher.SetQueryNorm(qnorm)
}
func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
func (s *PhraseSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
if !s.initialized {
err := s.initSearchers()
err := s.initSearchers(ctx)
if err != nil {
return nil, err
}
@ -144,14 +144,14 @@ func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docum
// return match
rv = s.currMust
rv.Locations = rvftlm
err := s.advanceNextMust()
err := s.advanceNextMust(ctx)
if err != nil {
return nil, err
}
return rv, nil
}
err := s.advanceNextMust()
err := s.advanceNextMust(ctx)
if err != nil {
return nil, err
}
@ -160,19 +160,19 @@ func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docum
return nil, nil
}
func (s *PhraseSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
func (s *PhraseSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
if !s.initialized {
err := s.initSearchers()
err := s.initSearchers(ctx)
if err != nil {
return nil, err
}
}
var err error
s.currMust, err = s.mustSearcher.Advance(ID, nil)
s.currMust, err = s.mustSearcher.Advance(ctx, ID)
if err != nil {
return nil, err
}
return s.Next(preAllocated)
return s.Next(ctx)
}
func (s *PhraseSearcher) Count() uint64 {
@ -195,3 +195,7 @@ func (s *PhraseSearcher) Close() error {
func (s *PhraseSearcher) Min() int {
return 0
}
func (s *PhraseSearcher) DocumentMatchPoolSize() int {
return s.mustSearcher.DocumentMatchPoolSize() + 1
}

17
search/searchers/search_phrase_test.go
View File

@ -12,6 +12,7 @@ package searchers
import (
"testing"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/search"
)
@ -53,8 +54,8 @@ func TestPhraseSearch(t *testing.T) {
searcher: phraseSearcher,
results: []*search.DocumentMatch{
{
ID: "2",
Score: 1.0807601687084403,
IndexInternalID: index.IndexInternalID("2"),
Score: 1.0807601687084403,
},
},
},
@ -68,19 +69,23 @@ func TestPhraseSearch(t *testing.T) {
}
}()
next, err := test.searcher.Next(nil)
ctx := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()),
}
next, err := test.searcher.Next(ctx)
i := 0
for err == nil && next != nil {
if i < len(test.results) {
if next.ID != test.results[i].ID {
t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex)
if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) {
t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex)
}
if next.Score != test.results[i].Score {
t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex)
t.Logf("scoring explanation: %s", next.Expl)
}
}
next, err = test.searcher.Next(nil)
ctx.DocumentMatchPool.Put(next)
next, err = test.searcher.Next(ctx)
i++
}
if err != nil {

12
search/searchers/search_regexp.go
View File

@ -106,13 +106,13 @@ func (s *RegexpSearcher) SetQueryNorm(qnorm float64) {
s.searcher.SetQueryNorm(qnorm)
}
func (s *RegexpSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
return s.searcher.Next(preAllocated)
func (s *RegexpSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
return s.searcher.Next(ctx)
}
func (s *RegexpSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
return s.searcher.Advance(ID, preAllocated)
func (s *RegexpSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
return s.searcher.Advance(ctx, ID)
}
func (s *RegexpSearcher) Close() error {
@ -122,3 +122,7 @@ func (s *RegexpSearcher) Close() error {
func (s *RegexpSearcher) Min() int {
return 0
}
func (s *RegexpSearcher) DocumentMatchPoolSize() int {
return s.searcher.DocumentMatchPoolSize()
}

25
search/searchers/search_regexp_test.go
View File

@ -13,6 +13,7 @@ import (
"regexp"
"testing"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/search"
)
@ -57,8 +58,8 @@ func TestRegexpSearch(t *testing.T) {
searcher: regexpSearcher,
results: []*search.DocumentMatch{
{
ID: "1",
Score: 1.916290731874155,
IndexInternalID: index.IndexInternalID("1"),
Score: 1.916290731874155,
},
},
},
@ -66,12 +67,12 @@ func TestRegexpSearch(t *testing.T) {
searcher: regexpSearcherCo,
results: []*search.DocumentMatch{
{
ID: "2",
Score: 0.33875554280828685,
IndexInternalID: index.IndexInternalID("2"),
Score: 0.33875554280828685,
},
{
ID: "3",
Score: 0.33875554280828685,
IndexInternalID: index.IndexInternalID("3"),
Score: 0.33875554280828685,
},
},
},
@ -85,19 +86,23 @@ func TestRegexpSearch(t *testing.T) {
}
}()
next, err := test.searcher.Next(nil)
ctx := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()),
}
next, err := test.searcher.Next(ctx)
i := 0
for err == nil && next != nil {
if i < len(test.results) {
if next.ID != test.results[i].ID {
t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex)
if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) {
t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex)
}
if next.Score != test.results[i].Score {
t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex)
t.Logf("scoring explanation: %s", next.Expl)
}
}
next, err = test.searcher.Next(nil)
ctx.DocumentMatchPool.Put(next)
next, err = test.searcher.Next(ctx)
i++
}
if err != nil {

14
search/searchers/search_term.go
View File

@ -26,7 +26,7 @@ type TermSearcher struct {
}
func NewTermSearcher(indexReader index.IndexReader, term string, field string, boost float64, explain bool) (*TermSearcher, error) {
reader, err := indexReader.TermFieldReader([]byte(term), field)
reader, err := indexReader.TermFieldReader([]byte(term), field, true, true, true)
if err != nil {
return nil, err
}
@ -53,7 +53,7 @@ func (s *TermSearcher) SetQueryNorm(qnorm float64) {
s.scorer.SetQueryNorm(qnorm)
}
func (s *TermSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
func (s *TermSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
termMatch, err := s.reader.Next(s.tfd.Reset())
if err != nil {
return nil, err
@ -64,13 +64,13 @@ func (s *TermSearcher) Next(preAllocated *search.DocumentMatch) (*search.Documen
}
// score match
docMatch := s.scorer.Score(termMatch, preAllocated)
docMatch := s.scorer.Score(ctx, termMatch)
// return doc match
return docMatch, nil
}
func (s *TermSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
func (s *TermSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
termMatch, err := s.reader.Advance(ID, s.tfd.Reset())
if err != nil {
return nil, err
@ -81,7 +81,7 @@ func (s *TermSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*
}
// score match
docMatch := s.scorer.Score(termMatch, preAllocated)
docMatch := s.scorer.Score(ctx, termMatch)
// return doc match
return docMatch, nil
@ -94,3 +94,7 @@ func (s *TermSearcher) Close() error {
func (s *TermSearcher) Min() int {
return 0
}
func (s *TermSearcher) DocumentMatchPoolSize() int {
return 1
}

12
search/searchers/search_term_prefix.go
View File

@ -70,13 +70,13 @@ func (s *TermPrefixSearcher) SetQueryNorm(qnorm float64) {
s.searcher.SetQueryNorm(qnorm)
}
func (s *TermPrefixSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
return s.searcher.Next(preAllocated)
func (s *TermPrefixSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
return s.searcher.Next(ctx)
}
func (s *TermPrefixSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
return s.searcher.Advance(ID, preAllocated)
func (s *TermPrefixSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
return s.searcher.Advance(ctx, ID)
}
func (s *TermPrefixSearcher) Close() error {
@ -86,3 +86,7 @@ func (s *TermPrefixSearcher) Close() error {
func (s *TermPrefixSearcher) Min() int {
return 0
}
func (s *TermPrefixSearcher) DocumentMatchPoolSize() int {
return s.searcher.DocumentMatchPoolSize()
}

23
search/searchers/search_term_test.go
View File

@ -17,6 +17,7 @@ import (
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store/gtreap"
"github.com/blevesearch/bleve/index/upside_down"
"github.com/blevesearch/bleve/search"
)
func TestTermSearcher(t *testing.T) {
@ -163,23 +164,28 @@ func TestTermSearcher(t *testing.T) {
t.Errorf("expected count of 9, got %d", searcher.Count())
}
docMatch, err := searcher.Next(nil)
ctx := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(1),
}
docMatch, err := searcher.Next(ctx)
if err != nil {
t.Errorf("expected result, got %v", err)
}
if docMatch.ID != "a" {
t.Errorf("expected result ID to be 'a', got '%s", docMatch.ID)
if !docMatch.IndexInternalID.Equals(index.IndexInternalID("a")) {
t.Errorf("expected result ID to be 'a', got '%s", docMatch.IndexInternalID)
}
docMatch, err = searcher.Advance("c", nil)
ctx.DocumentMatchPool.Put(docMatch)
docMatch, err = searcher.Advance(ctx, index.IndexInternalID("c"))
if err != nil {
t.Errorf("expected result, got %v", err)
}
if docMatch.ID != "c" {
t.Errorf("expected result ID to be 'c' got '%s'", docMatch.ID)
if !docMatch.IndexInternalID.Equals(index.IndexInternalID("c")) {
t.Errorf("expected result ID to be 'c' got '%s'", docMatch.IndexInternalID)
}
// try advancing past end
docMatch, err = searcher.Advance("z", nil)
ctx.DocumentMatchPool.Put(docMatch)
docMatch, err = searcher.Advance(ctx, index.IndexInternalID("z"))
if err != nil {
t.Fatal(err)
}
@ -188,7 +194,8 @@ func TestTermSearcher(t *testing.T) {
}
// try pushing next past end
docMatch, err = searcher.Next(nil)
ctx.DocumentMatchPool.Put(docMatch)
docMatch, err = searcher.Next(ctx)
if err != nil {
t.Fatal(err)
}