diff --git a/index/index.go b/index/index.go index bf9f394a..ac86f20a 100644 --- a/index/index.go +++ b/index/index.go @@ -10,6 +10,7 @@ package index import ( + "bytes" "encoding/json" "fmt" "time" @@ -60,7 +61,7 @@ type AsyncIndex interface { } type IndexReader interface { - TermFieldReader(term []byte, field string) (TermFieldReader, error) + TermFieldReader(term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (TermFieldReader, error) // DocIDReader returns an iterator over documents which identifiers are // greater than or equal to start and smaller than end. Set start to the @@ -69,6 +70,8 @@ type IndexReader interface { // The caller must close returned instance to release associated resources. DocIDReader(start, end string) (DocIDReader, error) + DocIDReaderOnly(ids []string) (DocIDReader, error) + FieldDict(field string) (FieldDict, error) // FieldDictRange is currently defined to include the start and end terms @@ -76,8 +79,8 @@ type IndexReader interface { FieldDictPrefix(field string, termPrefix []byte) (FieldDict, error) Document(id string) (*document.Document, error) - DocumentFieldTerms(id string) (FieldTerms, error) - DocumentFieldTermsForFields(id string, fields []string) (FieldTerms, error) + DocumentFieldTerms(id IndexInternalID) (FieldTerms, error) + DocumentFieldTermsForFields(id IndexInternalID, fields []string) (FieldTerms, error) Fields() ([]string, error) @@ -85,6 +88,8 @@ type IndexReader interface { DocCount() uint64 + FinalizeDocID(id IndexInternalID) (string, error) + Close() error } @@ -98,14 +103,36 @@ type TermFieldVector struct { End uint64 } +// IndexInternalID is an opaque document identifier interal to the index impl +type IndexInternalID []byte + +func (id IndexInternalID) Equals(other IndexInternalID) bool { + return id.Compare(other) == 0 +} + +func (id IndexInternalID) Compare(other IndexInternalID) int { + return bytes.Compare(id, other) +} + type TermFieldDoc struct { Term string - ID string + ID IndexInternalID Freq uint64 Norm float64 Vectors []*TermFieldVector } +// Reset allows an already allocated TermFieldDoc to be reused +func (tfd *TermFieldDoc) Reset() *TermFieldDoc { + // remember the []byte used for the ID + id := tfd.ID + // idiom to copy over from empty TermFieldDoc (0 allocations) + *tfd = TermFieldDoc{} + // reuse the []byte already allocated (and reset len to 0) + tfd.ID = id[:0] + return tfd +} + // TermFieldReader is the interface exposing the enumeration of documents // containing a given term in a given field. Documents are returned in byte // lexicographic order over their identifiers. @@ -117,7 +144,7 @@ type TermFieldReader interface { // Advance resets the enumeration at specified document or its immediate // follower. - Advance(ID string, preAlloced *TermFieldDoc) (*TermFieldDoc, error) + Advance(ID IndexInternalID, preAlloced *TermFieldDoc) (*TermFieldDoc, error) // Count returns the number of documents contains the term in this field. Count() uint64 @@ -137,15 +164,15 @@ type FieldDict interface { // DocIDReader is the interface exposing enumeration of documents identifiers. // Close the reader to release associated resources. type DocIDReader interface { - // Next returns the next document identifier in ascending lexicographic - // byte order, or io.EOF when the end of the sequence is reached. - Next() (string, error) + // Next returns the next document internal identifier in the natural + // index order, or io.EOF when the end of the sequence is reached. + Next() (IndexInternalID, error) - // Advance resets the iteration to the first identifier greater than or - // equal to ID. If ID is smaller than the start of the range, the iteration + // Advance resets the iteration to the first internal identifier greater than + // or equal to ID. If ID is smaller than the start of the range, the iteration // will start there instead. If ID is greater than or equal to the end of // the range, Next() call will return io.EOF. - Advance(ID string) (string, error) + Advance(ID IndexInternalID) (IndexInternalID, error) Close() error } @@ -200,8 +227,3 @@ func (b *Batch) Reset() { b.IndexOps = make(map[string]*document.Document) b.InternalOps = make(map[string][]byte) } - -func (tfd *TermFieldDoc) Reset() *TermFieldDoc { - *tfd = TermFieldDoc{} - return tfd -} diff --git a/index/upside_down/dump.go b/index/upside_down/dump.go index 023ae458..fd2be837 100644 --- a/index/upside_down/dump.go +++ b/index/upside_down/dump.go @@ -151,7 +151,7 @@ func (udc *UpsideDownCouch) DumpDoc(id string) chan interface{} { } }() - back, err := udc.backIndexRowForDoc(kvreader, id) + back, err := udc.backIndexRowForDoc(kvreader, []byte(id)) if err != nil { rv <- err return diff --git a/index/upside_down/index_reader.go b/index/upside_down/index_reader.go index 49655adf..181b8b14 100644 --- a/index/upside_down/index_reader.go +++ b/index/upside_down/index_reader.go @@ -23,12 +23,12 @@ type IndexReader struct { docCount uint64 } -func (i *IndexReader) TermFieldReader(term []byte, fieldName string) (index.TermFieldReader, error) { +func (i *IndexReader) TermFieldReader(term []byte, fieldName string, includeFreq, includeNorm, includeTermVectors bool) (index.TermFieldReader, error) { fieldIndex, fieldExists := i.index.fieldCache.FieldNamed(fieldName, false) if fieldExists { - return newUpsideDownCouchTermFieldReader(i, term, uint16(fieldIndex)) + return newUpsideDownCouchTermFieldReader(i, term, uint16(fieldIndex), includeFreq, includeNorm, includeTermVectors) } - return newUpsideDownCouchTermFieldReader(i, []byte{ByteSeparator}, ^uint16(0)) + return newUpsideDownCouchTermFieldReader(i, []byte{ByteSeparator}, ^uint16(0), includeFreq, includeNorm, includeTermVectors) } func (i *IndexReader) FieldDict(fieldName string) (index.FieldDict, error) { @@ -51,10 +51,14 @@ func (i *IndexReader) DocIDReader(start, end string) (index.DocIDReader, error) return newUpsideDownCouchDocIDReader(i, start, end) } +func (i *IndexReader) DocIDReaderOnly(ids []string) (index.DocIDReader, error) { + return newUpsideDownCouchDocIDReaderOnly(i, ids) +} + func (i *IndexReader) Document(id string) (doc *document.Document, err error) { // first hit the back index to confirm doc exists var backIndexRow *BackIndexRow - backIndexRow, err = i.index.backIndexRowForDoc(i.kvreader, id) + backIndexRow, err = i.index.backIndexRowForDoc(i.kvreader, []byte(id)) if err != nil { return } @@ -94,7 +98,7 @@ func (i *IndexReader) Document(id string) (doc *document.Document, err error) { return } -func (i *IndexReader) DocumentFieldTerms(id string) (index.FieldTerms, error) { +func (i *IndexReader) DocumentFieldTerms(id index.IndexInternalID) (index.FieldTerms, error) { back, err := i.index.backIndexRowForDoc(i.kvreader, id) if err != nil { return nil, err @@ -112,7 +116,7 @@ func (i *IndexReader) DocumentFieldTerms(id string) (index.FieldTerms, error) { return rv, nil } -func (i *IndexReader) DocumentFieldTermsForFields(id string, fields []string) (index.FieldTerms, error) { +func (i *IndexReader) DocumentFieldTermsForFields(id index.IndexInternalID, fields []string) (index.FieldTerms, error) { back, err := i.index.backIndexRowForDoc(i.kvreader, id) if err != nil { return nil, err @@ -181,6 +185,10 @@ func (i *IndexReader) Close() error { return i.kvreader.Close() } +func (i *IndexReader) FinalizeDocID(id index.IndexInternalID) (string, error) { + return string(id), nil +} + func incrementBytes(in []byte) []byte { rv := make([]byte, len(in)) copy(rv, in) diff --git a/index/upside_down/reader.go b/index/upside_down/reader.go index 53007ea0..caca3fac 100644 --- a/index/upside_down/reader.go +++ b/index/upside_down/reader.go @@ -10,6 +10,8 @@ package upside_down import ( + "bytes" + "sort" "sync/atomic" "github.com/blevesearch/bleve/index" @@ -25,7 +27,7 @@ type UpsideDownCouchTermFieldReader struct { field uint16 } -func newUpsideDownCouchTermFieldReader(indexReader *IndexReader, term []byte, field uint16) (*UpsideDownCouchTermFieldReader, error) { +func newUpsideDownCouchTermFieldReader(indexReader *IndexReader, term []byte, field uint16, includeFreq, includeNorm, includeTermVectors bool) (*UpsideDownCouchTermFieldReader, error) { dictionaryRow := NewDictionaryRow(term, field, 0) val, err := indexReader.kvreader.Get(dictionaryRow.Key()) if err != nil { @@ -81,7 +83,7 @@ func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (* if rv == nil { rv = &index.TermFieldDoc{} } - rv.ID = string(tfr.doc) + rv.ID = append(rv.ID, tfr.doc...) rv.Freq = tfr.freq rv.Norm = float64(tfr.norm) if tfr.vectors != nil { @@ -94,9 +96,9 @@ func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (* return nil, nil } -func (r *UpsideDownCouchTermFieldReader) Advance(docID string, preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) { +func (r *UpsideDownCouchTermFieldReader) Advance(docID index.IndexInternalID, preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) { if r.iterator != nil { - tfr := NewTermFrequencyRow(r.term, r.field, []byte(docID), 0, 0) + tfr := NewTermFrequencyRow(r.term, r.field, docID, 0, 0) r.iterator.Seek(tfr.Key()) key, val, valid := r.iterator.Current() if valid { @@ -108,7 +110,7 @@ func (r *UpsideDownCouchTermFieldReader) Advance(docID string, preAlloced *index if rv == nil { rv = &index.TermFieldDoc{} } - rv.ID = string(tfr.doc) + rv.ID = append(rv.ID, tfr.doc...) rv.Freq = tfr.freq rv.Norm = float64(tfr.norm) if tfr.vectors != nil { @@ -131,6 +133,9 @@ func (r *UpsideDownCouchTermFieldReader) Close() error { type UpsideDownCouchDocIDReader struct { indexReader *IndexReader iterator store.KVIterator + only []string + onlyPos int + onlyMode bool } func newUpsideDownCouchDocIDReader(indexReader *IndexReader, start, end string) (*UpsideDownCouchDocIDReader, error) { @@ -152,37 +157,137 @@ func newUpsideDownCouchDocIDReader(indexReader *IndexReader, start, end string) }, nil } -func (r *UpsideDownCouchDocIDReader) Next() (string, error) { - key, val, valid := r.iterator.Current() - if valid { - br, err := NewBackIndexRowKV(key, val) - if err != nil { - return "", err - } - rv := string(br.doc) - r.iterator.Next() - return rv, nil +func newUpsideDownCouchDocIDReaderOnly(indexReader *IndexReader, ids []string) (*UpsideDownCouchDocIDReader, error) { + // ensure ids are sorted + sort.Strings(ids) + startBytes := []byte{0x0} + if len(ids) > 0 { + startBytes = []byte(ids[0]) } - return "", nil + endBytes := []byte{0xff} + if len(ids) > 0 { + endBytes = incrementBytes([]byte(ids[len(ids)-1])) + } + bisr := NewBackIndexRow(startBytes, nil, nil) + bier := NewBackIndexRow(endBytes, nil, nil) + it := indexReader.kvreader.RangeIterator(bisr.Key(), bier.Key()) + + return &UpsideDownCouchDocIDReader{ + indexReader: indexReader, + iterator: it, + only: ids, + onlyMode: true, + }, nil } -func (r *UpsideDownCouchDocIDReader) Advance(docID string) (string, error) { - bir := NewBackIndexRow([]byte(docID), nil, nil) +func (r *UpsideDownCouchDocIDReader) Next() (index.IndexInternalID, error) { + key, val, valid := r.iterator.Current() + + if r.onlyMode { + var rv index.IndexInternalID + for valid && r.onlyPos < len(r.only) { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + if !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) { + ok := r.nextOnly() + if !ok { + return nil, nil + } + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + key, val, valid = r.iterator.Current() + continue + } else { + rv = append([]byte(nil), br.doc...) + break + } + } + if valid && r.onlyPos < len(r.only) { + ok := r.nextOnly() + if ok { + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + } + return rv, nil + } + + } else { + if valid { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + rv := append([]byte(nil), br.doc...) + r.iterator.Next() + return rv, nil + } + } + return nil, nil +} + +func (r *UpsideDownCouchDocIDReader) Advance(docID index.IndexInternalID) (index.IndexInternalID, error) { + bir := NewBackIndexRow(docID, nil, nil) r.iterator.Seek(bir.Key()) key, val, valid := r.iterator.Current() - if valid { - br, err := NewBackIndexRowKV(key, val) - if err != nil { - return "", err + r.onlyPos = sort.SearchStrings(r.only, string(docID)) + + if r.onlyMode { + var rv index.IndexInternalID + for valid && r.onlyPos < len(r.only) { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + if !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) { + ok := r.nextOnly() + if !ok { + return nil, nil + } + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + continue + } else { + rv = append([]byte(nil), br.doc...) + break + } + } + if valid && r.onlyPos < len(r.only) { + ok := r.nextOnly() + if ok { + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + } + return rv, nil + } + } else { + if valid { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + rv := append([]byte(nil), br.doc...) + r.iterator.Next() + return rv, nil } - rv := string(br.doc) - r.iterator.Next() - return rv, nil } - return "", nil + return nil, nil } func (r *UpsideDownCouchDocIDReader) Close() error { atomic.AddUint64(&r.indexReader.index.stats.termSearchersFinished, uint64(1)) return r.iterator.Close() } + +// move the r.only pos forward one, skipping duplicates +// return true if there is more data, or false if we got to the end of the list +func (r *UpsideDownCouchDocIDReader) nextOnly() bool { + + // advance 1 position, until we see a different key + // it's already sorted, so this skips duplicates + start := r.onlyPos + r.onlyPos++ + for r.onlyPos < len(r.only) && r.only[r.onlyPos] == r.only[start] { + start = r.onlyPos + r.onlyPos++ + } + // inidicate if we got to the end of the list + return r.onlyPos < len(r.only) +} diff --git a/index/upside_down/reader_test.go b/index/upside_down/reader_test.go index 124b1462..0e27e78f 100644 --- a/index/upside_down/reader_test.go +++ b/index/upside_down/reader_test.go @@ -72,7 +72,7 @@ func TestIndexReader(t *testing.T) { }() // first look for a term that doesn't exist - reader, err := indexReader.TermFieldReader([]byte("nope"), "name") + reader, err := indexReader.TermFieldReader([]byte("nope"), "name", true, true, true) if err != nil { t.Errorf("Error accessing term field reader: %v", err) } @@ -85,7 +85,7 @@ func TestIndexReader(t *testing.T) { t.Fatal(err) } - reader, err = indexReader.TermFieldReader([]byte("test"), "name") + reader, err = indexReader.TermFieldReader([]byte("test"), "name", true, true, true) if err != nil { t.Errorf("Error accessing term field reader: %v", err) } @@ -111,7 +111,7 @@ func TestIndexReader(t *testing.T) { } expectedMatch := &index.TermFieldDoc{ - ID: "2", + ID: index.IndexInternalID("2"), Freq: 1, Norm: 0.5773502588272095, Vectors: []*index.TermFieldVector{ @@ -123,7 +123,7 @@ func TestIndexReader(t *testing.T) { }, }, } - tfr, err := indexReader.TermFieldReader([]byte("rice"), "desc") + tfr, err := indexReader.TermFieldReader([]byte("rice"), "desc", true, true, true) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -140,22 +140,22 @@ func TestIndexReader(t *testing.T) { } // now test usage of advance - reader, err = indexReader.TermFieldReader([]byte("test"), "name") + reader, err = indexReader.TermFieldReader([]byte("test"), "name", true, true, true) if err != nil { t.Errorf("Error accessing term field reader: %v", err) } - match, err = reader.Advance("2", nil) + match, err = reader.Advance(index.IndexInternalID("2"), nil) if err != nil { t.Errorf("unexpected error: %v", err) } if match == nil { t.Fatalf("Expected match, got nil") } - if match.ID != "2" { + if !match.ID.Equals(index.IndexInternalID("2")) { t.Errorf("Expected ID '2', got '%s'", match.ID) } - match, err = reader.Advance("3", nil) + match, err = reader.Advance(index.IndexInternalID("3"), nil) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -168,7 +168,7 @@ func TestIndexReader(t *testing.T) { } // now test creating a reader for a field that doesn't exist - reader, err = indexReader.TermFieldReader([]byte("water"), "doesnotexist") + reader, err = indexReader.TermFieldReader([]byte("water"), "doesnotexist", true, true, true) if err != nil { t.Errorf("Error accessing term field reader: %v", err) } @@ -183,7 +183,7 @@ func TestIndexReader(t *testing.T) { if match != nil { t.Errorf("expected nil, got %v", match) } - match, err = reader.Advance("anywhere", nil) + match, err = reader.Advance(index.IndexInternalID("anywhere"), nil) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -260,7 +260,7 @@ func TestIndexDocIdReader(t *testing.T) { id, err := reader.Next() count := uint64(0) - for id != "" { + for id != nil { count++ id, err = reader.Next() } @@ -280,19 +280,19 @@ func TestIndexDocIdReader(t *testing.T) { } }() - id, err = reader2.Advance("2") + id, err = reader2.Advance(index.IndexInternalID("2")) if err != nil { t.Error(err) } - if id != "2" { + if !id.Equals(index.IndexInternalID("2")) { t.Errorf("expected to find id '2', got '%s'", id) } - id, err = reader2.Advance("3") + id, err = reader2.Advance(index.IndexInternalID("3")) if err != nil { t.Error(err) } - if id != "" { + if id != nil { t.Errorf("expected to find id '', got '%s'", id) } } diff --git a/index/upside_down/upside_down.go b/index/upside_down/upside_down.go index 7fec5724..7c67e157 100644 --- a/index/upside_down/upside_down.go +++ b/index/upside_down/upside_down.go @@ -439,7 +439,7 @@ func (udc *UpsideDownCouch) Update(doc *document.Document) (err error) { // first we lookup the backindex row for the doc id if it exists // lookup the back index row var backIndexRow *BackIndexRow - backIndexRow, err = udc.backIndexRowForDoc(kvreader, doc.ID) + backIndexRow, err = udc.backIndexRowForDoc(kvreader, index.IndexInternalID(doc.ID)) if err != nil { _ = kvreader.Close() atomic.AddUint64(&udc.stats.errors, 1) @@ -627,7 +627,7 @@ func (udc *UpsideDownCouch) Delete(id string) (err error) { // first we lookup the backindex row for the doc id if it exists // lookup the back index row var backIndexRow *BackIndexRow - backIndexRow, err = udc.backIndexRowForDoc(kvreader, id) + backIndexRow, err = udc.backIndexRowForDoc(kvreader, index.IndexInternalID(id)) if err != nil { _ = kvreader.Close() atomic.AddUint64(&udc.stats.errors, 1) @@ -695,10 +695,10 @@ func (udc *UpsideDownCouch) deleteSingle(id string, backIndexRow *BackIndexRow, return deleteRows } -func (udc *UpsideDownCouch) backIndexRowForDoc(kvreader store.KVReader, docID string) (*BackIndexRow, error) { +func (udc *UpsideDownCouch) backIndexRowForDoc(kvreader store.KVReader, docID index.IndexInternalID) (*BackIndexRow, error) { // use a temporary row structure to build key tempRow := &BackIndexRow{ - doc: []byte(docID), + doc: docID, } keyBuf := GetRowBuffer() @@ -833,7 +833,7 @@ func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) { } for docID, doc := range batch.IndexOps { - backIndexRow, err := udc.backIndexRowForDoc(kvreader, docID) + backIndexRow, err := udc.backIndexRowForDoc(kvreader, index.IndexInternalID(docID)) if err != nil { docBackIndexRowErr = err return diff --git a/index/upside_down/upside_down_test.go b/index/upside_down/upside_down_test.go index 85a62bc0..099a2b0c 100644 --- a/index/upside_down/upside_down_test.go +++ b/index/upside_down/upside_down_test.go @@ -663,16 +663,16 @@ func TestIndexBatch(t *testing.T) { if err != nil { t.Error(err) } - docIds := make([]string, 0) + docIds := make([]index.IndexInternalID, 0) docID, err := docIDReader.Next() - for docID != "" && err == nil { + for docID != nil && err == nil { docIds = append(docIds, docID) docID, err = docIDReader.Next() } if err != nil { t.Error(err) } - expectedDocIds := []string{"2", "3"} + expectedDocIds := []index.IndexInternalID{index.IndexInternalID("2"), index.IndexInternalID("3")} if !reflect.DeepEqual(docIds, expectedDocIds) { t.Errorf("expected ids: %v, got ids: %v", expectedDocIds, docIds) } @@ -1119,14 +1119,14 @@ func TestIndexTermReaderCompositeFields(t *testing.T) { } }() - termFieldReader, err := indexReader.TermFieldReader([]byte("mister"), "_all") + termFieldReader, err := indexReader.TermFieldReader([]byte("mister"), "_all", true, true, true) if err != nil { t.Error(err) } tfd, err := termFieldReader.Next(nil) for tfd != nil && err == nil { - if tfd.ID != "1" { + if !tfd.ID.Equals(index.IndexInternalID("1")) { t.Errorf("expected to find document id 1") } tfd, err = termFieldReader.Next(nil) @@ -1179,7 +1179,7 @@ func TestIndexDocumentFieldTerms(t *testing.T) { } }() - fieldTerms, err := indexReader.DocumentFieldTerms("1") + fieldTerms, err := indexReader.DocumentFieldTerms(index.IndexInternalID("1")) if err != nil { t.Error(err) } diff --git a/index_impl.go b/index_impl.go index f914ec94..66c33f8c 100644 --- a/index_impl.go +++ b/index_impl.go @@ -435,7 +435,7 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr collector.SetFacetsBuilder(facetsBuilder) } - err = collector.Collect(ctx, searcher) + err = collector.Collect(ctx, searcher, indexReader) if err != nil { return nil, err } diff --git a/search/collector.go b/search/collector.go index 773c8d55..a6d9148c 100644 --- a/search/collector.go +++ b/search/collector.go @@ -12,11 +12,13 @@ package search import ( "time" + "github.com/blevesearch/bleve/index" + "golang.org/x/net/context" ) type Collector interface { - Collect(ctx context.Context, searcher Searcher) error + Collect(ctx context.Context, searcher Searcher, reader index.IndexReader) error Results() DocumentMatchCollection Total() uint64 MaxScore() float64 diff --git a/search/collectors/bench_test.go b/search/collectors/bench_test.go index d8daeb9e..cddf5e7f 100644 --- a/search/collectors/bench_test.go +++ b/search/collectors/bench_test.go @@ -5,16 +5,17 @@ import ( "strconv" "testing" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" "golang.org/x/net/context" ) func benchHelper(numOfMatches int, collector search.Collector, b *testing.B) { - matches := make(search.DocumentMatchCollection, 0, numOfMatches) + matches := make([]*search.DocumentMatch, 0, numOfMatches) for i := 0; i < numOfMatches; i++ { matches = append(matches, &search.DocumentMatch{ - ID: strconv.Itoa(i), - Score: rand.Float64(), + IndexInternalID: index.IndexInternalID(strconv.Itoa(i)), + Score: rand.Float64(), }) } @@ -24,7 +25,7 @@ func benchHelper(numOfMatches int, collector search.Collector, b *testing.B) { searcher := &stubSearcher{ matches: matches, } - err := collector.Collect(context.Background(), searcher) + err := collector.Collect(context.Background(), searcher, &stubReader{}) if err != nil { b.Fatal(err) } diff --git a/search/collectors/collector_top_score.go b/search/collectors/collector_top_score.go index 90b0c75e..7d7131ce 100644 --- a/search/collectors/collector_top_score.go +++ b/search/collectors/collector_top_score.go @@ -15,6 +15,7 @@ import ( "golang.org/x/net/context" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -27,6 +28,7 @@ type TopScoreCollector struct { minScore float64 total uint64 facetsBuilder *search.FacetsBuilder + actualResults search.DocumentMatchCollection } func NewTopScorerCollector(k int) *TopScoreCollector { @@ -59,16 +61,21 @@ func (tksc *TopScoreCollector) Took() time.Duration { var COLLECT_CHECK_DONE_EVERY = uint64(1024) -func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Searcher) error { +func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Searcher, reader index.IndexReader) error { startTime := time.Now() var err error - var pre search.DocumentMatch // A single pre-alloc'ed, reused instance. var next *search.DocumentMatch + + // search context with enough pre-allocated document matches + searchContext := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(tksc.k + tksc.skip + searcher.DocumentMatchPoolSize()), + } + select { case <-ctx.Done(): return ctx.Err() default: - next, err = searcher.Next(&pre) + next, err = searcher.Next(searchContext) } for err == nil && next != nil { if tksc.total%COLLECT_CHECK_DONE_EVERY == 0 { @@ -78,15 +85,22 @@ func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Sear default: } } - tksc.collectSingle(next) if tksc.facetsBuilder != nil { err = tksc.facetsBuilder.Update(next) if err != nil { break } } - next, err = searcher.Next(pre.Reset()) + tksc.collectSingle(searchContext, next) + + next, err = searcher.Next(searchContext) } + // finalize actual results + tksc.actualResults, err = tksc.finalizeResults(reader) + if err != nil { + return err + } + // compute search duration tksc.took = time.Since(startTime) if err != nil { @@ -95,47 +109,50 @@ func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Sear return nil } -func (tksc *TopScoreCollector) collectSingle(dmIn *search.DocumentMatch) { +func (tksc *TopScoreCollector) collectSingle(ctx *search.SearchContext, d *search.DocumentMatch) { // increment total hits tksc.total++ // update max score - if dmIn.Score > tksc.maxScore { - tksc.maxScore = dmIn.Score + if d.Score > tksc.maxScore { + tksc.maxScore = d.Score } - if dmIn.Score <= tksc.minScore { + if d.Score <= tksc.minScore { + ctx.DocumentMatchPool.Put(d) return } - // Because the dmIn will be the single, pre-allocated, reused - // instance, we need to copy the dmIn into a new, standalone - // instance before inserting into our candidate results list. - dm := &search.DocumentMatch{} - *dm = *dmIn - for e := tksc.results.Front(); e != nil; e = e.Next() { curr := e.Value.(*search.DocumentMatch) - if dm.Score <= curr.Score { + if d.Score <= curr.Score { - tksc.results.InsertBefore(dm, e) + tksc.results.InsertBefore(d, e) // if we just made the list too long if tksc.results.Len() > (tksc.k + tksc.skip) { // remove the head - tksc.minScore = tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch).Score + removed := tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch) + tksc.minScore = removed.Score + ctx.DocumentMatchPool.Put(removed) } return } } // if we got to the end, we still have to add it - tksc.results.PushBack(dm) + tksc.results.PushBack(d) if tksc.results.Len() > (tksc.k + tksc.skip) { // remove the head - tksc.minScore = tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch).Score + removed := tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch) + tksc.minScore = removed.Score + ctx.DocumentMatchPool.Put(removed) } } func (tksc *TopScoreCollector) Results() search.DocumentMatchCollection { + return tksc.actualResults +} + +func (tksc *TopScoreCollector) finalizeResults(r index.IndexReader) (search.DocumentMatchCollection, error) { if tksc.results.Len()-tksc.skip > 0 { rv := make(search.DocumentMatchCollection, tksc.results.Len()-tksc.skip) i := 0 @@ -145,12 +162,17 @@ func (tksc *TopScoreCollector) Results() search.DocumentMatchCollection { skipped++ continue } + var err error rv[i] = e.Value.(*search.DocumentMatch) + rv[i].ID, err = r.FinalizeDocID(rv[i].IndexInternalID) + if err != nil { + return nil, err + } i++ } - return rv + return rv, nil } - return search.DocumentMatchCollection{} + return search.DocumentMatchCollection{}, nil } func (tksc *TopScoreCollector) SetFacetsBuilder(facetsBuilder *search.FacetsBuilder) { diff --git a/search/collectors/collector_top_score_test.go b/search/collectors/collector_top_score_test.go index 4bf76140..66c8978c 100644 --- a/search/collectors/collector_top_score_test.go +++ b/search/collectors/collector_top_score_test.go @@ -14,6 +14,7 @@ import ( "golang.org/x/net/context" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -23,68 +24,68 @@ func TestTop10Scores(t *testing.T) { // the top-10 scores are > 10 // everything else is less than 10 searcher := &stubSearcher{ - matches: search.DocumentMatchCollection{ + matches: []*search.DocumentMatch{ &search.DocumentMatch{ - ID: "a", - Score: 11, + IndexInternalID: index.IndexInternalID("a"), + Score: 11, }, &search.DocumentMatch{ - ID: "b", - Score: 9, + IndexInternalID: index.IndexInternalID("b"), + Score: 9, }, &search.DocumentMatch{ - ID: "c", - Score: 11, + IndexInternalID: index.IndexInternalID("c"), + Score: 11, }, &search.DocumentMatch{ - ID: "d", - Score: 9, + IndexInternalID: index.IndexInternalID("d"), + Score: 9, }, &search.DocumentMatch{ - ID: "e", - Score: 11, + IndexInternalID: index.IndexInternalID("e"), + Score: 11, }, &search.DocumentMatch{ - ID: "f", - Score: 9, + IndexInternalID: index.IndexInternalID("f"), + Score: 9, }, &search.DocumentMatch{ - ID: "g", - Score: 11, + IndexInternalID: index.IndexInternalID("g"), + Score: 11, }, &search.DocumentMatch{ - ID: "h", - Score: 9, + IndexInternalID: index.IndexInternalID("h"), + Score: 9, }, &search.DocumentMatch{ - ID: "i", - Score: 11, + IndexInternalID: index.IndexInternalID("i"), + Score: 11, }, &search.DocumentMatch{ - ID: "j", - Score: 11, + IndexInternalID: index.IndexInternalID("j"), + Score: 11, }, &search.DocumentMatch{ - ID: "k", - Score: 11, + IndexInternalID: index.IndexInternalID("k"), + Score: 11, }, &search.DocumentMatch{ - ID: "l", - Score: 99, + IndexInternalID: index.IndexInternalID("l"), + Score: 99, }, &search.DocumentMatch{ - ID: "m", - Score: 11, + IndexInternalID: index.IndexInternalID("m"), + Score: 11, }, &search.DocumentMatch{ - ID: "n", - Score: 11, + IndexInternalID: index.IndexInternalID("n"), + Score: 11, }, }, } collector := NewTopScorerCollector(10) - err := collector.Collect(context.Background(), searcher) + err := collector.Collect(context.Background(), searcher, &stubReader{}) if err != nil { t.Fatal(err) } @@ -131,68 +132,68 @@ func TestTop10ScoresSkip10(t *testing.T) { // the top-10 scores are > 10 // everything else is less than 10 searcher := &stubSearcher{ - matches: search.DocumentMatchCollection{ + matches: []*search.DocumentMatch{ &search.DocumentMatch{ - ID: "a", - Score: 11, + IndexInternalID: index.IndexInternalID("a"), + Score: 11, }, &search.DocumentMatch{ - ID: "b", - Score: 9.5, + IndexInternalID: index.IndexInternalID("b"), + Score: 9.5, }, &search.DocumentMatch{ - ID: "c", - Score: 11, + IndexInternalID: index.IndexInternalID("c"), + Score: 11, }, &search.DocumentMatch{ - ID: "d", - Score: 9, + IndexInternalID: index.IndexInternalID("d"), + Score: 9, }, &search.DocumentMatch{ - ID: "e", - Score: 11, + IndexInternalID: index.IndexInternalID("e"), + Score: 11, }, &search.DocumentMatch{ - ID: "f", - Score: 9, + IndexInternalID: index.IndexInternalID("f"), + Score: 9, }, &search.DocumentMatch{ - ID: "g", - Score: 11, + IndexInternalID: index.IndexInternalID("g"), + Score: 11, }, &search.DocumentMatch{ - ID: "h", - Score: 9, + IndexInternalID: index.IndexInternalID("h"), + Score: 9, }, &search.DocumentMatch{ - ID: "i", - Score: 11, + IndexInternalID: index.IndexInternalID("i"), + Score: 11, }, &search.DocumentMatch{ - ID: "j", - Score: 11, + IndexInternalID: index.IndexInternalID("j"), + Score: 11, }, &search.DocumentMatch{ - ID: "k", - Score: 11, + IndexInternalID: index.IndexInternalID("k"), + Score: 11, }, &search.DocumentMatch{ - ID: "l", - Score: 99, + IndexInternalID: index.IndexInternalID("l"), + Score: 99, }, &search.DocumentMatch{ - ID: "m", - Score: 11, + IndexInternalID: index.IndexInternalID("m"), + Score: 11, }, &search.DocumentMatch{ - ID: "n", - Score: 11, + IndexInternalID: index.IndexInternalID("n"), + Score: 11, }, }, } collector := NewTopScorerSkipCollector(10, 10) - err := collector.Collect(context.Background(), searcher) + err := collector.Collect(context.Background(), searcher, &stubReader{}) if err != nil { t.Fatal(err) } @@ -227,69 +228,69 @@ func TestPaginationSameScores(t *testing.T) { // a stub search with more than 10 matches // all documents have the same score searcher := &stubSearcher{ - matches: search.DocumentMatchCollection{ + matches: []*search.DocumentMatch{ &search.DocumentMatch{ - ID: "a", - Score: 5, + IndexInternalID: index.IndexInternalID("a"), + Score: 5, }, &search.DocumentMatch{ - ID: "b", - Score: 5, + IndexInternalID: index.IndexInternalID("b"), + Score: 5, }, &search.DocumentMatch{ - ID: "c", - Score: 5, + IndexInternalID: index.IndexInternalID("c"), + Score: 5, }, &search.DocumentMatch{ - ID: "d", - Score: 5, + IndexInternalID: index.IndexInternalID("d"), + Score: 5, }, &search.DocumentMatch{ - ID: "e", - Score: 5, + IndexInternalID: index.IndexInternalID("e"), + Score: 5, }, &search.DocumentMatch{ - ID: "f", - Score: 5, + IndexInternalID: index.IndexInternalID("f"), + Score: 5, }, &search.DocumentMatch{ - ID: "g", - Score: 5, + IndexInternalID: index.IndexInternalID("g"), + Score: 5, }, &search.DocumentMatch{ - ID: "h", - Score: 5, + IndexInternalID: index.IndexInternalID("h"), + Score: 5, }, &search.DocumentMatch{ - ID: "i", - Score: 5, + IndexInternalID: index.IndexInternalID("i"), + Score: 5, }, &search.DocumentMatch{ - ID: "j", - Score: 5, + IndexInternalID: index.IndexInternalID("j"), + Score: 5, }, &search.DocumentMatch{ - ID: "k", - Score: 5, + IndexInternalID: index.IndexInternalID("k"), + Score: 5, }, &search.DocumentMatch{ - ID: "l", - Score: 5, + IndexInternalID: index.IndexInternalID("l"), + Score: 5, }, &search.DocumentMatch{ - ID: "m", - Score: 5, + IndexInternalID: index.IndexInternalID("m"), + Score: 5, }, &search.DocumentMatch{ - ID: "n", - Score: 5, + IndexInternalID: index.IndexInternalID("n"), + Score: 5, }, }, } // first get first 5 hits collector := NewTopScorerSkipCollector(5, 0) - err := collector.Collect(context.Background(), searcher) + err := collector.Collect(context.Background(), searcher, &stubReader{}) if err != nil { t.Fatal(err) } @@ -313,69 +314,69 @@ func TestPaginationSameScores(t *testing.T) { // a stub search with more than 10 matches // all documents have the same score searcher = &stubSearcher{ - matches: search.DocumentMatchCollection{ + matches: []*search.DocumentMatch{ &search.DocumentMatch{ - ID: "a", - Score: 5, + IndexInternalID: index.IndexInternalID("a"), + Score: 5, }, &search.DocumentMatch{ - ID: "b", - Score: 5, + IndexInternalID: index.IndexInternalID("b"), + Score: 5, }, &search.DocumentMatch{ - ID: "c", - Score: 5, + IndexInternalID: index.IndexInternalID("c"), + Score: 5, }, &search.DocumentMatch{ - ID: "d", - Score: 5, + IndexInternalID: index.IndexInternalID("d"), + Score: 5, }, &search.DocumentMatch{ - ID: "e", - Score: 5, + IndexInternalID: index.IndexInternalID("e"), + Score: 5, }, &search.DocumentMatch{ - ID: "f", - Score: 5, + IndexInternalID: index.IndexInternalID("f"), + Score: 5, }, &search.DocumentMatch{ - ID: "g", - Score: 5, + IndexInternalID: index.IndexInternalID("g"), + Score: 5, }, &search.DocumentMatch{ - ID: "h", - Score: 5, + IndexInternalID: index.IndexInternalID("h"), + Score: 5, }, &search.DocumentMatch{ - ID: "i", - Score: 5, + IndexInternalID: index.IndexInternalID("i"), + Score: 5, }, &search.DocumentMatch{ - ID: "j", - Score: 5, + IndexInternalID: index.IndexInternalID("j"), + Score: 5, }, &search.DocumentMatch{ - ID: "k", - Score: 5, + IndexInternalID: index.IndexInternalID("k"), + Score: 5, }, &search.DocumentMatch{ - ID: "l", - Score: 5, + IndexInternalID: index.IndexInternalID("l"), + Score: 5, }, &search.DocumentMatch{ - ID: "m", - Score: 5, + IndexInternalID: index.IndexInternalID("m"), + Score: 5, }, &search.DocumentMatch{ - ID: "n", - Score: 5, + IndexInternalID: index.IndexInternalID("n"), + Score: 5, }, }, } // now get next 5 hits collector = NewTopScorerSkipCollector(5, 5) - err = collector.Collect(context.Background(), searcher) + err = collector.Collect(context.Background(), searcher, &stubReader{}) if err != nil { t.Fatal(err) } diff --git a/search/collectors/search_test.go b/search/collectors/search_test.go index 629a1eca..c5faa243 100644 --- a/search/collectors/search_test.go +++ b/search/collectors/search_test.go @@ -10,15 +10,17 @@ package collectors import ( + "github.com/blevesearch/bleve/document" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) type stubSearcher struct { index int - matches search.DocumentMatchCollection + matches []*search.DocumentMatch } -func (ss *stubSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (ss *stubSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { if ss.index < len(ss.matches) { rv := ss.matches[ss.index] ss.index++ @@ -27,9 +29,9 @@ func (ss *stubSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docume return nil, nil } -func (ss *stubSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (ss *stubSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { - for ss.index < len(ss.matches) && ss.matches[ss.index].ID < ID { + for ss.index < len(ss.matches) && ss.matches[ss.index].IndexInternalID.Compare(ID) < 0 { ss.index++ } if ss.index < len(ss.matches) { @@ -58,3 +60,65 @@ func (ss *stubSearcher) Count() uint64 { func (ss *stubSearcher) Min() int { return 0 } + +func (ss *stubSearcher) DocumentMatchPoolSize() int { + return 0 +} + +type stubReader struct{} + +func (sr *stubReader) TermFieldReader(term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (index.TermFieldReader, error) { + return nil, nil +} + +func (sr *stubReader) DocIDReader(start, end string) (index.DocIDReader, error) { + return nil, nil +} + +func (sr *stubReader) DocIDReaderOnly(ids []string) (index.DocIDReader, error) { + return nil, nil +} + +func (sr *stubReader) FieldDict(field string) (index.FieldDict, error) { + return nil, nil +} + +func (sr *stubReader) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) { + return nil, nil +} + +func (sr *stubReader) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) { + return nil, nil +} + +func (sr *stubReader) Document(id string) (*document.Document, error) { + return nil, nil +} + +func (sr *stubReader) DocumentFieldTerms(id index.IndexInternalID) (index.FieldTerms, error) { + return nil, nil +} + +func (sr *stubReader) DocumentFieldTermsForFields(id index.IndexInternalID, fields []string) (index.FieldTerms, error) { + return nil, nil +} + +func (sr *stubReader) Fields() ([]string, error) { + return nil, nil +} + +func (sr *stubReader) GetInternal(key []byte) ([]byte, error) { + return nil, nil +} + +func (sr *stubReader) DocCount() uint64 { + return 0 +} + +func (sr *stubReader) FinalizeDocID(id index.IndexInternalID) (string, error) { + return string(id), nil +} + +func (sr *stubReader) Close() error { + return nil +} diff --git a/search/facets_builder.go b/search/facets_builder.go index 4d52ec2b..55cb33ab 100644 --- a/search/facets_builder.go +++ b/search/facets_builder.go @@ -42,7 +42,7 @@ func (fb *FacetsBuilder) Update(docMatch *DocumentMatch) error { for _, facetBuilder := range fb.facets { fields = append(fields, facetBuilder.Field()) } - fieldTerms, err := fb.indexReader.DocumentFieldTermsForFields(docMatch.ID, fields) + fieldTerms, err := fb.indexReader.DocumentFieldTermsForFields(docMatch.IndexInternalID, fields) if err != nil { return err } diff --git a/search/pool.go b/search/pool.go new file mode 100644 index 00000000..108d494b --- /dev/null +++ b/search/pool.go @@ -0,0 +1,70 @@ +// Copyright (c) 2014 Couchbase, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file +// except in compliance with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed under the +// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +// either express or implied. See the License for the specific language governing permissions +// and limitations under the License. + +package search + +// DocumentMatchPoolTooSmall is a callback function that can be executed +// when the DocumentMatchPool does not have sufficient capacity +// By default we just perform just-in-time allocation, but you could log +// a message, or panic, etc. +type DocumentMatchPoolTooSmall func(p *DocumentMatchPool) *DocumentMatch + +// DocumentMatchPool manages use/re-use of DocumentMatch instances +// it pre-allocates space from a single large block with the expected +// number of instances. It is not thread-safe as currently all +// aspects of search take place in a single goroutine. +type DocumentMatchPool struct { + avail DocumentMatchCollection + TooSmall DocumentMatchPoolTooSmall +} + +func defaultDocumentMatchPoolTooSmall(p *DocumentMatchPool) *DocumentMatch { + return &DocumentMatch{} +} + +// NewDocumentMatchPool will build a DocumentMatchPool with memory +// pre-allocated to accomodate the requested number of DocumentMatch +// instances +func NewDocumentMatchPool(size int) *DocumentMatchPool { + avail := make(DocumentMatchCollection, 0, size) + // pre-allocate the expected number of instances + startBlock := make([]DocumentMatch, size) + // make these initial instances available + for i := range startBlock { + avail = append(avail, &startBlock[i]) + } + return &DocumentMatchPool{ + avail: avail, + TooSmall: defaultDocumentMatchPoolTooSmall, + } +} + +// Get returns an available DocumentMatch from the pool +// if the pool was not allocated with sufficient size, an allocation will +// occur to satisfy this request. As a side-effect this will grow the size +// of the pool. +func (p *DocumentMatchPool) Get() *DocumentMatch { + var rv *DocumentMatch + if len(p.avail) > 0 { + rv, p.avail = p.avail[len(p.avail)-1], p.avail[:len(p.avail)-1] + } else { + rv = p.TooSmall(p) + } + return rv +} + +// Put returns a DocumentMatch to the pool +func (p *DocumentMatchPool) Put(d *DocumentMatch) { + if d == nil { + return + } + // reset DocumentMatch before returning it to available pool + d.Reset() + p.avail = append(p.avail, d) +} diff --git a/search/pool_test.go b/search/pool_test.go new file mode 100644 index 00000000..875f607c --- /dev/null +++ b/search/pool_test.go @@ -0,0 +1,65 @@ +// Copyright (c) 2013 Couchbase, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file +// except in compliance with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed under the +// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +// either express or implied. See the License for the specific language governing permissions +// and limitations under the License. + +package search + +import "testing" + +func TestDocumentMatchPool(t *testing.T) { + + tooManyCalled := false + + // create a pool + dmp := NewDocumentMatchPool(10) + dmp.TooSmall = func(inner *DocumentMatchPool) *DocumentMatch { + tooManyCalled = true + return &DocumentMatch{} + } + + // get 10 instances without returning + returned := make(DocumentMatchCollection, 10) + + for i := 0; i < 10; i++ { + returned[i] = dmp.Get() + if tooManyCalled { + t.Fatal("too many function called before expected") + } + } + + // get one more and see if too many function is called + extra := dmp.Get() + if !tooManyCalled { + t.Fatal("expected too many function to be called, but wasnt") + } + + // return the first 10 + for i := 0; i < 10; i++ { + dmp.Put(returned[i]) + } + + // check len and cap + if len(dmp.avail) != 10 { + t.Fatalf("expected 10 available, got %d", len(dmp.avail)) + } + if cap(dmp.avail) != 10 { + t.Fatalf("expected avail cap still 10, got %d", cap(dmp.avail)) + } + + // return the extra + dmp.Put(extra) + + // check len and cap grown to 11 + if len(dmp.avail) != 11 { + t.Fatalf("expected 11 available, got %d", len(dmp.avail)) + } + // cap grows, but not by 1 (append behavior) + if cap(dmp.avail) <= 10 { + t.Fatalf("expected avail cap mpore than 10, got %d", cap(dmp.avail)) + } +} diff --git a/search/scorers/scorer_conjunction.go b/search/scorers/scorer_conjunction.go index 422f282c..c941c69e 100644 --- a/search/scorers/scorer_conjunction.go +++ b/search/scorers/scorer_conjunction.go @@ -23,11 +23,7 @@ func NewConjunctionQueryScorer(explain bool) *ConjunctionQueryScorer { } } -func (s *ConjunctionQueryScorer) Score(constituents []*search.DocumentMatch) *search.DocumentMatch { - rv := search.DocumentMatch{ - ID: constituents[0].ID, - } - +func (s *ConjunctionQueryScorer) Score(ctx *search.SearchContext, constituents []*search.DocumentMatch) *search.DocumentMatch { var sum float64 var childrenExplanations []*search.Explanation if s.explain { @@ -44,16 +40,21 @@ func (s *ConjunctionQueryScorer) Score(constituents []*search.DocumentMatch) *se locations = append(locations, docMatch.Locations) } } - rv.Score = sum + newScore := sum + var newExpl *search.Explanation if s.explain { - rv.Expl = &search.Explanation{Value: sum, Message: "sum of:", Children: childrenExplanations} + newExpl = &search.Explanation{Value: sum, Message: "sum of:", Children: childrenExplanations} } + // reuse constituents[0] as the return value + rv := constituents[0] + rv.Score = newScore + rv.Expl = newExpl if len(locations) == 1 { rv.Locations = locations[0] } else if len(locations) > 1 { rv.Locations = search.MergeLocations(locations) } - return &rv + return rv } diff --git a/search/scorers/scorer_constant.go b/search/scorers/scorer_constant.go index 1434bd5e..adbaf478 100644 --- a/search/scorers/scorer_constant.go +++ b/search/scorers/scorer_constant.go @@ -12,6 +12,7 @@ package scorers import ( "fmt" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -64,7 +65,7 @@ func (s *ConstantScorer) SetQueryNorm(qnorm float64) { } } -func (s *ConstantScorer) Score(id string) *search.DocumentMatch { +func (s *ConstantScorer) Score(ctx *search.SearchContext, id index.IndexInternalID) *search.DocumentMatch { var scoreExplanation *search.Explanation score := s.constant @@ -91,13 +92,12 @@ func (s *ConstantScorer) Score(id string) *search.DocumentMatch { } } - rv := search.DocumentMatch{ - ID: id, - Score: score, - } + rv := ctx.DocumentMatchPool.Get() + rv.IndexInternalID = id + rv.Score = score if s.explain { rv.Expl = scoreExplanation } - return &rv + return rv } diff --git a/search/scorers/scorer_constant_test.go b/search/scorers/scorer_constant_test.go index 4c8740e0..e1e6e9c6 100644 --- a/search/scorers/scorer_constant_test.go +++ b/search/scorers/scorer_constant_test.go @@ -28,7 +28,7 @@ func TestConstantScorer(t *testing.T) { // test some simple math { termMatch: &index.TermFieldDoc{ - ID: "one", + ID: index.IndexInternalID("one"), Freq: 1, Norm: 1.0, Vectors: []*index.TermFieldVector{ @@ -41,8 +41,8 @@ func TestConstantScorer(t *testing.T) { }, }, result: &search.DocumentMatch{ - ID: "one", - Score: 1.0, + IndexInternalID: index.IndexInternalID("one"), + Score: 1.0, Expl: &search.Explanation{ Value: 1.0, Message: "ConstantScore()", @@ -52,7 +52,10 @@ func TestConstantScorer(t *testing.T) { } for _, test := range tests { - actual := scorer.Score(test.termMatch.ID) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(1), + } + actual := scorer.Score(ctx, test.termMatch.ID) if !reflect.DeepEqual(actual, test.result) { t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) @@ -72,13 +75,13 @@ func TestConstantScorerWithQueryNorm(t *testing.T) { }{ { termMatch: &index.TermFieldDoc{ - ID: "one", + ID: index.IndexInternalID("one"), Freq: 1, Norm: 1.0, }, result: &search.DocumentMatch{ - ID: "one", - Score: 2.0, + IndexInternalID: index.IndexInternalID("one"), + Score: 2.0, Expl: &search.Explanation{ Value: 2.0, Message: "weight(^1.000000), product of:", @@ -108,7 +111,10 @@ func TestConstantScorerWithQueryNorm(t *testing.T) { } for _, test := range tests { - actual := scorer.Score(test.termMatch.ID) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(1), + } + actual := scorer.Score(ctx, test.termMatch.ID) if !reflect.DeepEqual(actual, test.result) { t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) diff --git a/search/scorers/scorer_disjunction.go b/search/scorers/scorer_disjunction.go index 00bc8cd0..4868fc4f 100644 --- a/search/scorers/scorer_disjunction.go +++ b/search/scorers/scorer_disjunction.go @@ -25,11 +25,7 @@ func NewDisjunctionQueryScorer(explain bool) *DisjunctionQueryScorer { } } -func (s *DisjunctionQueryScorer) Score(constituents []*search.DocumentMatch, countMatch, countTotal int) *search.DocumentMatch { - rv := search.DocumentMatch{ - ID: constituents[0].ID, - } - +func (s *DisjunctionQueryScorer) Score(ctx *search.SearchContext, constituents []*search.DocumentMatch, countMatch, countTotal int) *search.DocumentMatch { var sum float64 var childrenExplanations []*search.Explanation if s.explain { @@ -53,19 +49,24 @@ func (s *DisjunctionQueryScorer) Score(constituents []*search.DocumentMatch, cou } coord := float64(countMatch) / float64(countTotal) - rv.Score = sum * coord + newScore := sum * coord + var newExpl *search.Explanation if s.explain { ce := make([]*search.Explanation, 2) ce[0] = rawExpl ce[1] = &search.Explanation{Value: coord, Message: fmt.Sprintf("coord(%d/%d)", countMatch, countTotal)} - rv.Expl = &search.Explanation{Value: rv.Score, Message: "product of:", Children: ce} + newExpl = &search.Explanation{Value: newScore, Message: "product of:", Children: ce} } + // reuse constituents[0] as the return value + rv := constituents[0] + rv.Score = newScore + rv.Expl = newExpl if len(locations) == 1 { rv.Locations = locations[0] } else if len(locations) > 1 { rv.Locations = search.MergeLocations(locations) } - return &rv + return rv } diff --git a/search/scorers/scorer_term.go b/search/scorers/scorer_term.go index ce926221..4304d067 100644 --- a/search/scorers/scorer_term.go +++ b/search/scorers/scorer_term.go @@ -83,7 +83,7 @@ func (s *TermQueryScorer) SetQueryNorm(qnorm float64) { } } -func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc, preAllocated *search.DocumentMatch) *search.DocumentMatch { +func (s *TermQueryScorer) Score(ctx *search.SearchContext, termMatch *index.TermFieldDoc) *search.DocumentMatch { var scoreExplanation *search.Explanation // need to compute score @@ -128,11 +128,8 @@ func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc, preAllocated *sea } } - rv := preAllocated - if rv == nil { - rv = &search.DocumentMatch{} - } - rv.ID = termMatch.ID + rv := ctx.DocumentMatchPool.Get() + rv.IndexInternalID = append(rv.IndexInternalID, termMatch.ID...) rv.Score = score if s.explain { rv.Expl = scoreExplanation diff --git a/search/scorers/scorer_term_test.go b/search/scorers/scorer_term_test.go index 612b02d9..0241d163 100644 --- a/search/scorers/scorer_term_test.go +++ b/search/scorers/scorer_term_test.go @@ -35,7 +35,7 @@ func TestTermScorer(t *testing.T) { // test some simple math { termMatch: &index.TermFieldDoc{ - ID: "one", + ID: index.IndexInternalID("one"), Freq: 1, Norm: 1.0, Vectors: []*index.TermFieldVector{ @@ -48,8 +48,8 @@ func TestTermScorer(t *testing.T) { }, }, result: &search.DocumentMatch{ - ID: "one", - Score: math.Sqrt(1.0) * idf, + IndexInternalID: index.IndexInternalID("one"), + Score: math.Sqrt(1.0) * idf, Expl: &search.Explanation{ Value: math.Sqrt(1.0) * idf, Message: "fieldWeight(desc:beer in one), product of:", @@ -84,13 +84,13 @@ func TestTermScorer(t *testing.T) { // test the same thing again (score should be cached this time) { termMatch: &index.TermFieldDoc{ - ID: "one", + ID: index.IndexInternalID("one"), Freq: 1, Norm: 1.0, }, result: &search.DocumentMatch{ - ID: "one", - Score: math.Sqrt(1.0) * idf, + IndexInternalID: index.IndexInternalID("one"), + Score: math.Sqrt(1.0) * idf, Expl: &search.Explanation{ Value: math.Sqrt(1.0) * idf, Message: "fieldWeight(desc:beer in one), product of:", @@ -114,13 +114,13 @@ func TestTermScorer(t *testing.T) { // test a case where the sqrt isn't precalculated { termMatch: &index.TermFieldDoc{ - ID: "one", + ID: index.IndexInternalID("one"), Freq: 65, Norm: 1.0, }, result: &search.DocumentMatch{ - ID: "one", - Score: math.Sqrt(65) * idf, + IndexInternalID: index.IndexInternalID("one"), + Score: math.Sqrt(65) * idf, Expl: &search.Explanation{ Value: math.Sqrt(65) * idf, Message: "fieldWeight(desc:beer in one), product of:", @@ -144,7 +144,10 @@ func TestTermScorer(t *testing.T) { } for _, test := range tests { - actual := scorer.Score(test.termMatch, nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(1), + } + actual := scorer.Score(ctx, test.termMatch) if !reflect.DeepEqual(actual, test.result) { t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) @@ -177,13 +180,13 @@ func TestTermScorerWithQueryNorm(t *testing.T) { }{ { termMatch: &index.TermFieldDoc{ - ID: "one", + ID: index.IndexInternalID("one"), Freq: 1, Norm: 1.0, }, result: &search.DocumentMatch{ - ID: "one", - Score: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0, + IndexInternalID: index.IndexInternalID("one"), + Score: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0, Expl: &search.Explanation{ Value: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0, Message: "weight(desc:beer^3.000000 in one), product of:", @@ -231,7 +234,10 @@ func TestTermScorerWithQueryNorm(t *testing.T) { } for _, test := range tests { - actual := scorer.Score(test.termMatch, nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(1), + } + actual := scorer.Score(ctx, test.termMatch) if !reflect.DeepEqual(actual, test.result) { t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) diff --git a/search/search.go b/search/search.go index 4b3b988b..22abc064 100644 --- a/search/search.go +++ b/search/search.go @@ -9,6 +9,8 @@ package search +import "github.com/blevesearch/bleve/index" + type Location struct { Pos float64 `json:"pos"` Start float64 `json:"start"` @@ -51,12 +53,13 @@ type FieldTermLocationMap map[string]TermLocationMap type FieldFragmentMap map[string][]string type DocumentMatch struct { - Index string `json:"index,omitempty"` - ID string `json:"id"` - Score float64 `json:"score"` - Expl *Explanation `json:"explanation,omitempty"` - Locations FieldTermLocationMap `json:"locations,omitempty"` - Fragments FieldFragmentMap `json:"fragments,omitempty"` + Index string `json:"index,omitempty"` + ID string `json:"id"` + IndexInternalID index.IndexInternalID `json:"-"` + Score float64 `json:"score"` + Expl *Explanation `json:"explanation,omitempty"` + Locations FieldTermLocationMap `json:"locations,omitempty"` + Fragments FieldFragmentMap `json:"fragments,omitempty"` // Fields contains the values for document fields listed in // SearchRequest.Fields. Text fields are returned as strings, numeric @@ -85,8 +88,14 @@ func (dm *DocumentMatch) AddFieldValue(name string, value interface{}) { dm.Fields[name] = valSlice } +// Reset allows an already allocated DocumentMatch to be reused func (dm *DocumentMatch) Reset() *DocumentMatch { + // remember the []byte used for the IndexInternalID + indexInternalId := dm.IndexInternalID + // idiom to copy over from empty DocumentMatch (0 allocations) *dm = DocumentMatch{} + // reuse the []byte already allocated (and reset len to 0) + dm.IndexInternalID = indexInternalId[:0] return dm } @@ -97,11 +106,18 @@ func (c DocumentMatchCollection) Swap(i, j int) { c[i], c[j] = c[j], c[i] } func (c DocumentMatchCollection) Less(i, j int) bool { return c[i].Score > c[j].Score } type Searcher interface { - Next(preAllocated *DocumentMatch) (*DocumentMatch, error) - Advance(ID string, preAllocated *DocumentMatch) (*DocumentMatch, error) + Next(ctx *SearchContext) (*DocumentMatch, error) + Advance(ctx *SearchContext, ID index.IndexInternalID) (*DocumentMatch, error) Close() error Weight() float64 SetQueryNorm(float64) Count() uint64 Min() int + + DocumentMatchPoolSize() int +} + +// SearchContext represents the context around a single search +type SearchContext struct { + DocumentMatchPool *DocumentMatchPool } diff --git a/search/searchers/search_boolean.go b/search/searchers/search_boolean.go index 0ae96483..a66ba44f 100644 --- a/search/searchers/search_boolean.go +++ b/search/searchers/search_boolean.go @@ -27,7 +27,7 @@ type BooleanSearcher struct { currMust *search.DocumentMatch currShould *search.DocumentMatch currMustNot *search.DocumentMatch - currentID string + currentID index.IndexInternalID min uint64 scorer *scorers.ConjunctionQueryScorer } @@ -66,63 +66,78 @@ func (s *BooleanSearcher) computeQueryNorm() { } } -func (s *BooleanSearcher) initSearchers() error { +func (s *BooleanSearcher) initSearchers(ctx *search.SearchContext) error { var err error // get all searchers pointing at their first match if s.mustSearcher != nil { - s.currMust, err = s.mustSearcher.Next(nil) + if s.currMust != nil { + ctx.DocumentMatchPool.Put(s.currMust) + } + s.currMust, err = s.mustSearcher.Next(ctx) if err != nil { return err } } if s.shouldSearcher != nil { - s.currShould, err = s.shouldSearcher.Next(nil) + if s.currShould != nil { + ctx.DocumentMatchPool.Put(s.currShould) + } + s.currShould, err = s.shouldSearcher.Next(ctx) if err != nil { return err } } if s.mustNotSearcher != nil { - s.currMustNot, err = s.mustNotSearcher.Next(nil) + if s.currMustNot != nil { + ctx.DocumentMatchPool.Put(s.currMustNot) + } + s.currMustNot, err = s.mustNotSearcher.Next(ctx) if err != nil { return err } } if s.mustSearcher != nil && s.currMust != nil { - s.currentID = s.currMust.ID + s.currentID = s.currMust.IndexInternalID } else if s.mustSearcher == nil && s.currShould != nil { - s.currentID = s.currShould.ID + s.currentID = s.currShould.IndexInternalID } else { - s.currentID = "" + s.currentID = nil } s.initialized = true return nil } -func (s *BooleanSearcher) advanceNextMust() error { +func (s *BooleanSearcher) advanceNextMust(ctx *search.SearchContext, skipReturn *search.DocumentMatch) error { var err error if s.mustSearcher != nil { - s.currMust, err = s.mustSearcher.Next(nil) + if s.currMust != skipReturn { + ctx.DocumentMatchPool.Put(s.currMust) + } + s.currMust, err = s.mustSearcher.Next(ctx) if err != nil { return err } } else if s.mustSearcher == nil { - s.currShould, err = s.shouldSearcher.Next(nil) + if s.currShould != skipReturn { + ctx.DocumentMatchPool.Put(s.currShould) + } + s.currShould, err = s.shouldSearcher.Next(ctx) if err != nil { return err } } if s.mustSearcher != nil && s.currMust != nil { - s.currentID = s.currMust.ID + s.currentID = s.currMust.IndexInternalID } else if s.mustSearcher == nil && s.currShould != nil { - s.currentID = s.currShould.ID + s.currentID = s.currShould.IndexInternalID } else { - s.currentID = "" + s.currentID = nil } return nil } @@ -148,10 +163,10 @@ func (s *BooleanSearcher) SetQueryNorm(qnorm float64) { } } -func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *BooleanSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { if !s.initialized { - err := s.initSearchers() + err := s.initSearchers(ctx) if err != nil { return nil, err } @@ -160,37 +175,43 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu var err error var rv *search.DocumentMatch - for s.currentID != "" { - if s.currMustNot != nil && s.currMustNot.ID < s.currentID { + for s.currentID != nil { + if s.currMustNot != nil && s.currMustNot.IndexInternalID.Compare(s.currentID) < 0 { + if s.currMustNot != nil { + ctx.DocumentMatchPool.Put(s.currMustNot) + } // advance must not searcher to our candidate entry - s.currMustNot, err = s.mustNotSearcher.Advance(s.currentID, nil) + s.currMustNot, err = s.mustNotSearcher.Advance(ctx, s.currentID) if err != nil { return nil, err } - if s.currMustNot != nil && s.currMustNot.ID == s.currentID { + if s.currMustNot != nil && s.currMustNot.IndexInternalID.Equals(s.currentID) { // the candidate is excluded - err = s.advanceNextMust() + err = s.advanceNextMust(ctx, nil) if err != nil { return nil, err } continue } - } else if s.currMustNot != nil && s.currMustNot.ID == s.currentID { + } else if s.currMustNot != nil && s.currMustNot.IndexInternalID.Equals(s.currentID) { // the candidate is excluded - err = s.advanceNextMust() + err = s.advanceNextMust(ctx, nil) if err != nil { return nil, err } continue } - if s.currShould != nil && s.currShould.ID < s.currentID { + if s.currShould != nil && s.currShould.IndexInternalID.Compare(s.currentID) < 0 { // advance should searcher to our candidate entry - s.currShould, err = s.shouldSearcher.Advance(s.currentID, nil) + if s.currShould != nil { + ctx.DocumentMatchPool.Put(s.currShould) + } + s.currShould, err = s.shouldSearcher.Advance(ctx, s.currentID) if err != nil { return nil, err } - if s.currShould != nil && s.currShould.ID == s.currentID { + if s.currShould != nil && s.currShould.IndexInternalID.Equals(s.currentID) { // score bonus matches should var cons []*search.DocumentMatch if s.currMust != nil { @@ -203,22 +224,22 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu s.currShould, } } - rv = s.scorer.Score(cons) - err = s.advanceNextMust() + rv = s.scorer.Score(ctx, cons) + err = s.advanceNextMust(ctx, rv) if err != nil { return nil, err } break } else if s.shouldSearcher.Min() == 0 { // match is OK anyway - rv = s.scorer.Score([]*search.DocumentMatch{s.currMust}) - err = s.advanceNextMust() + rv = s.scorer.Score(ctx, []*search.DocumentMatch{s.currMust}) + err = s.advanceNextMust(ctx, rv) if err != nil { return nil, err } break } - } else if s.currShould != nil && s.currShould.ID == s.currentID { + } else if s.currShould != nil && s.currShould.IndexInternalID.Equals(s.currentID) { // score bonus matches should var cons []*search.DocumentMatch if s.currMust != nil { @@ -231,23 +252,23 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu s.currShould, } } - rv = s.scorer.Score(cons) - err = s.advanceNextMust() + rv = s.scorer.Score(ctx, cons) + err = s.advanceNextMust(ctx, rv) if err != nil { return nil, err } break } else if s.shouldSearcher == nil || s.shouldSearcher.Min() == 0 { // match is OK anyway - rv = s.scorer.Score([]*search.DocumentMatch{s.currMust}) - err = s.advanceNextMust() + rv = s.scorer.Score(ctx, []*search.DocumentMatch{s.currMust}) + err = s.advanceNextMust(ctx, rv) if err != nil { return nil, err } break } - err = s.advanceNextMust() + err = s.advanceNextMust(ctx, nil) if err != nil { return nil, err } @@ -255,10 +276,10 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu return rv, nil } -func (s *BooleanSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *BooleanSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { if !s.initialized { - err := s.initSearchers() + err := s.initSearchers(ctx) if err != nil { return nil, err } @@ -266,33 +287,42 @@ func (s *BooleanSearcher) Advance(ID string, preAllocated *search.DocumentMatch) var err error if s.mustSearcher != nil { - s.currMust, err = s.mustSearcher.Advance(ID, nil) + if s.currMust != nil { + ctx.DocumentMatchPool.Put(s.currMust) + } + s.currMust, err = s.mustSearcher.Advance(ctx, ID) if err != nil { return nil, err } } if s.shouldSearcher != nil { - s.currShould, err = s.shouldSearcher.Advance(ID, nil) + if s.currShould != nil { + ctx.DocumentMatchPool.Put(s.currShould) + } + s.currShould, err = s.shouldSearcher.Advance(ctx, ID) if err != nil { return nil, err } } if s.mustNotSearcher != nil { - s.currMustNot, err = s.mustNotSearcher.Advance(ID, nil) + if s.currMustNot != nil { + ctx.DocumentMatchPool.Put(s.currMustNot) + } + s.currMustNot, err = s.mustNotSearcher.Advance(ctx, ID) if err != nil { return nil, err } } if s.mustSearcher != nil && s.currMust != nil { - s.currentID = s.currMust.ID + s.currentID = s.currMust.IndexInternalID } else if s.mustSearcher == nil && s.currShould != nil { - s.currentID = s.currShould.ID + s.currentID = s.currShould.IndexInternalID } else { - s.currentID = "" + s.currentID = nil } - return s.Next(preAllocated) + return s.Next(ctx) } func (s *BooleanSearcher) Count() uint64 { @@ -333,3 +363,17 @@ func (s *BooleanSearcher) Close() error { func (s *BooleanSearcher) Min() int { return 0 } + +func (s *BooleanSearcher) DocumentMatchPoolSize() int { + rv := 3 + if s.mustSearcher != nil { + rv += s.mustSearcher.DocumentMatchPoolSize() + } + if s.shouldSearcher != nil { + rv += s.shouldSearcher.DocumentMatchPoolSize() + } + if s.mustNotSearcher != nil { + rv += s.mustNotSearcher.DocumentMatchPoolSize() + } + return rv +} diff --git a/search/searchers/search_boolean_test.go b/search/searchers/search_boolean_test.go index ac6dc7ea..ddd31d2e 100644 --- a/search/searchers/search_boolean_test.go +++ b/search/searchers/search_boolean_test.go @@ -12,6 +12,7 @@ package searchers import ( "testing" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -248,16 +249,16 @@ func TestBooleanSearch(t *testing.T) { searcher: booleanSearcher, results: []*search.DocumentMatch{ { - ID: "1", - Score: 0.9818005051949021, + IndexInternalID: index.IndexInternalID("1"), + Score: 0.9818005051949021, }, { - ID: "3", - Score: 0.808709699395535, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.808709699395535, }, { - ID: "4", - Score: 0.34618161159873423, + IndexInternalID: index.IndexInternalID("4"), + Score: 0.34618161159873423, }, }, }, @@ -265,12 +266,12 @@ func TestBooleanSearch(t *testing.T) { searcher: booleanSearcher2, results: []*search.DocumentMatch{ { - ID: "1", - Score: 0.6775110856165737, + IndexInternalID: index.IndexInternalID("1"), + Score: 0.6775110856165737, }, { - ID: "3", - Score: 0.6775110856165737, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.6775110856165737, }, }, }, @@ -283,16 +284,16 @@ func TestBooleanSearch(t *testing.T) { searcher: booleanSearcher4, results: []*search.DocumentMatch{ { - ID: "1", - Score: 1.0, + IndexInternalID: index.IndexInternalID("1"), + Score: 1.0, }, { - ID: "3", - Score: 0.5, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.5, }, { - ID: "4", - Score: 1.0, + IndexInternalID: index.IndexInternalID("4"), + Score: 1.0, }, }, }, @@ -300,12 +301,12 @@ func TestBooleanSearch(t *testing.T) { searcher: booleanSearcher5, results: []*search.DocumentMatch{ { - ID: "3", - Score: 0.5, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.5, }, { - ID: "4", - Score: 1.0, + IndexInternalID: index.IndexInternalID("4"), + Score: 1.0, }, }, }, @@ -318,8 +319,8 @@ func TestBooleanSearch(t *testing.T) { searcher: conjunctionSearcher7, results: []*search.DocumentMatch{ { - ID: "1", - Score: 2.0097428702814377, + IndexInternalID: index.IndexInternalID("1"), + Score: 2.0097428702814377, }, }, }, @@ -327,8 +328,8 @@ func TestBooleanSearch(t *testing.T) { searcher: conjunctionSearcher8, results: []*search.DocumentMatch{ { - ID: "3", - Score: 2.0681575785068107, + IndexInternalID: index.IndexInternalID("3"), + Score: 2.0681575785068107, }, }, }, @@ -342,19 +343,23 @@ func TestBooleanSearch(t *testing.T) { } }() - next, err := test.searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()), + } + next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { - if next.ID != test.results[i].ID { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) t.Logf("scoring explanation: %s", next.Expl) } } - next, err = test.searcher.Next(nil) + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) i++ } if err != nil { diff --git a/search/searchers/search_conjunction.go b/search/searchers/search_conjunction.go index fe37b80b..c773f6a5 100644 --- a/search/searchers/search_conjunction.go +++ b/search/searchers/search_conjunction.go @@ -25,7 +25,7 @@ type ConjunctionSearcher struct { explain bool queryNorm float64 currs []*search.DocumentMatch - currentID string + currentID index.IndexInternalID scorer *scorers.ConjunctionQueryScorer } @@ -63,11 +63,14 @@ func (s *ConjunctionSearcher) computeQueryNorm() { } } -func (s *ConjunctionSearcher) initSearchers() error { +func (s *ConjunctionSearcher) initSearchers(ctx *search.SearchContext) error { var err error // get all searchers pointing at their first match for i, termSearcher := range s.searchers { - s.currs[i], err = termSearcher.Next(nil) + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = termSearcher.Next(ctx) if err != nil { return err } @@ -75,9 +78,9 @@ func (s *ConjunctionSearcher) initSearchers() error { if len(s.currs) > 0 { if s.currs[0] != nil { - s.currentID = s.currs[0].ID + s.currentID = s.currs[0].IndexInternalID } else { - s.currentID = "" + s.currentID = nil } } @@ -99,9 +102,9 @@ func (s *ConjunctionSearcher) SetQueryNorm(qnorm float64) { } } -func (s *ConjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *ConjunctionSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { if !s.initialized { - err := s.initSearchers() + err := s.initSearchers(ctx) if err != nil { return nil, err } @@ -109,68 +112,82 @@ func (s *ConjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search. var rv *search.DocumentMatch var err error OUTER: - for s.currentID != "" { + for s.currentID != nil { for i, termSearcher := range s.searchers { - if s.currs[i] != nil && s.currs[i].ID != s.currentID { - if s.currentID < s.currs[i].ID { - s.currentID = s.currs[i].ID + if s.currs[i] != nil && !s.currs[i].IndexInternalID.Equals(s.currentID) { + if s.currentID.Compare(s.currs[i].IndexInternalID) < 0 { + s.currentID = s.currs[i].IndexInternalID continue OUTER } // this reader doesn't have the currentID, try to advance - s.currs[i], err = termSearcher.Advance(s.currentID, nil) + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = termSearcher.Advance(ctx, s.currentID) if err != nil { return nil, err } if s.currs[i] == nil { - s.currentID = "" + s.currentID = nil continue OUTER } - if s.currs[i].ID != s.currentID { + if !s.currs[i].IndexInternalID.Equals(s.currentID) { // we just advanced, so it doesn't match, it must be greater // no need to call next - s.currentID = s.currs[i].ID + s.currentID = s.currs[i].IndexInternalID continue OUTER } } else if s.currs[i] == nil { - s.currentID = "" + s.currentID = nil continue OUTER } } // if we get here, a doc matched all readers, sum the score and add it - rv = s.scorer.Score(s.currs) + rv = s.scorer.Score(ctx, s.currs) - // prepare for next entry - s.currs[0], err = s.searchers[0].Next(nil) - if err != nil { - return nil, err + // we know all the searchers are pointing at the same thing + // so they all need to be advanced + for i, termSearcher := range s.searchers { + if s.currs[i] != rv { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = termSearcher.Next(ctx) + if err != nil { + return nil, err + } } + if s.currs[0] == nil { - s.currentID = "" + s.currentID = nil } else { - s.currentID = s.currs[0].ID + s.currentID = s.currs[0].IndexInternalID } + // don't continue now, wait for the next call to Next() break } return rv, nil } -func (s *ConjunctionSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *ConjunctionSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { if !s.initialized { - err := s.initSearchers() + err := s.initSearchers(ctx) if err != nil { return nil, err } } var err error for i, searcher := range s.searchers { - s.currs[i], err = searcher.Advance(ID, nil) + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = searcher.Advance(ctx, ID) if err != nil { return nil, err } } s.currentID = ID - return s.Next(preAllocated) + return s.Next(ctx) } func (s *ConjunctionSearcher) Count() uint64 { @@ -195,3 +212,11 @@ func (s *ConjunctionSearcher) Close() error { func (s *ConjunctionSearcher) Min() int { return 0 } + +func (s *ConjunctionSearcher) DocumentMatchPoolSize() int { + rv := len(s.currs) + for _, s := range s.searchers { + rv += s.DocumentMatchPoolSize() + } + return rv +} diff --git a/search/searchers/search_conjunction_test.go b/search/searchers/search_conjunction_test.go index 1256059b..227a70ba 100644 --- a/search/searchers/search_conjunction_test.go +++ b/search/searchers/search_conjunction_test.go @@ -12,6 +12,7 @@ package searchers import ( "testing" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -128,8 +129,8 @@ func TestConjunctionSearch(t *testing.T) { searcher: beerAndMartySearcher, results: []*search.DocumentMatch{ { - ID: "1", - Score: 2.0097428702814377, + IndexInternalID: index.IndexInternalID("1"), + Score: 2.0097428702814377, }, }, }, @@ -137,8 +138,8 @@ func TestConjunctionSearch(t *testing.T) { searcher: angstAndBeerSearcher, results: []*search.DocumentMatch{ { - ID: "2", - Score: 1.0807601687084403, + IndexInternalID: index.IndexInternalID("2"), + Score: 1.0807601687084403, }, }, }, @@ -150,12 +151,12 @@ func TestConjunctionSearch(t *testing.T) { searcher: beerAndMisterSearcher, results: []*search.DocumentMatch{ { - ID: "2", - Score: 1.2877980334016337, + IndexInternalID: index.IndexInternalID("2"), + Score: 1.2877980334016337, }, { - ID: "3", - Score: 1.2877980334016337, + IndexInternalID: index.IndexInternalID("3"), + Score: 1.2877980334016337, }, }, }, @@ -163,8 +164,8 @@ func TestConjunctionSearch(t *testing.T) { searcher: couchbaseAndMisterSearcher, results: []*search.DocumentMatch{ { - ID: "2", - Score: 1.4436599157093672, + IndexInternalID: index.IndexInternalID("2"), + Score: 1.4436599157093672, }, }, }, @@ -172,8 +173,8 @@ func TestConjunctionSearch(t *testing.T) { searcher: beerAndCouchbaseAndMisterSearcher, results: []*search.DocumentMatch{ { - ID: "2", - Score: 1.441614953806971, + IndexInternalID: index.IndexInternalID("2"), + Score: 1.441614953806971, }, }, }, @@ -187,19 +188,22 @@ func TestConjunctionSearch(t *testing.T) { } }() - next, err := test.searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(10), + } + next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { - if next.ID != test.results[i].ID { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) t.Logf("scoring explanation: %s", next.Expl) } } - next, err = test.searcher.Next(nil) + next, err = test.searcher.Next(ctx) i++ } if err != nil { diff --git a/search/searchers/search_disjunction.go b/search/searchers/search_disjunction.go index d7cd9408..65a42d81 100644 --- a/search/searchers/search_disjunction.go +++ b/search/searchers/search_disjunction.go @@ -30,7 +30,7 @@ type DisjunctionSearcher struct { searchers OrderedSearcherList queryNorm float64 currs []*search.DocumentMatch - currentID string + currentID index.IndexInternalID scorer *scorers.DisjunctionQueryScorer min float64 } @@ -83,11 +83,14 @@ func (s *DisjunctionSearcher) computeQueryNorm() { } } -func (s *DisjunctionSearcher) initSearchers() error { +func (s *DisjunctionSearcher) initSearchers(ctx *search.SearchContext) error { var err error // get all searchers pointing at their first match for i, termSearcher := range s.searchers { - s.currs[i], err = termSearcher.Next(nil) + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = termSearcher.Next(ctx) if err != nil { return err } @@ -98,11 +101,11 @@ func (s *DisjunctionSearcher) initSearchers() error { return nil } -func (s *DisjunctionSearcher) nextSmallestID() string { - rv := "" +func (s *DisjunctionSearcher) nextSmallestID() index.IndexInternalID { + var rv index.IndexInternalID for _, curr := range s.currs { - if curr != nil && (curr.ID < rv || rv == "") { - rv = curr.ID + if curr != nil && (curr.IndexInternalID.Compare(rv) < 0 || rv == nil) { + rv = curr.IndexInternalID } } return rv @@ -122,9 +125,9 @@ func (s *DisjunctionSearcher) SetQueryNorm(qnorm float64) { } } -func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *DisjunctionSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { if !s.initialized { - err := s.initSearchers() + err := s.initSearchers(ctx) if err != nil { return nil, err } @@ -134,9 +137,9 @@ func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search. matching := make([]*search.DocumentMatch, 0, len(s.searchers)) found := false - for !found && s.currentID != "" { + for !found && s.currentID != nil { for _, curr := range s.currs { - if curr != nil && curr.ID == s.currentID { + if curr != nil && curr.IndexInternalID.Equals(s.currentID) { matching = append(matching, curr) } } @@ -144,16 +147,19 @@ func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search. if len(matching) >= int(s.min) { found = true // score this match - rv = s.scorer.Score(matching, len(matching), len(s.searchers)) + rv = s.scorer.Score(ctx, matching, len(matching), len(s.searchers)) } // reset matching matching = make([]*search.DocumentMatch, 0) // invoke next on all the matching searchers for i, curr := range s.currs { - if curr != nil && curr.ID == s.currentID { + if curr != nil && curr.IndexInternalID.Equals(s.currentID) { searcher := s.searchers[i] - s.currs[i], err = searcher.Next(nil) + if s.currs[i] != rv { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = searcher.Next(ctx) if err != nil { return nil, err } @@ -164,9 +170,9 @@ func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search. return rv, nil } -func (s *DisjunctionSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *DisjunctionSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { if !s.initialized { - err := s.initSearchers() + err := s.initSearchers(ctx) if err != nil { return nil, err } @@ -174,7 +180,10 @@ func (s *DisjunctionSearcher) Advance(ID string, preAllocated *search.DocumentMa // get all searchers pointing at their first match var err error for i, termSearcher := range s.searchers { - s.currs[i], err = termSearcher.Advance(ID, nil) + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = termSearcher.Advance(ctx, ID) if err != nil { return nil, err } @@ -182,7 +191,7 @@ func (s *DisjunctionSearcher) Advance(ID string, preAllocated *search.DocumentMa s.currentID = s.nextSmallestID() - return s.Next(preAllocated) + return s.Next(ctx) } func (s *DisjunctionSearcher) Count() uint64 { @@ -207,3 +216,11 @@ func (s *DisjunctionSearcher) Close() error { func (s *DisjunctionSearcher) Min() int { return int(s.min) // FIXME just make this an int } + +func (s *DisjunctionSearcher) DocumentMatchPoolSize() int { + rv := len(s.currs) + for _, s := range s.searchers { + rv += s.DocumentMatchPoolSize() + } + return rv +} diff --git a/search/searchers/search_disjunction_test.go b/search/searchers/search_disjunction_test.go index 19b0dd42..5f194065 100644 --- a/search/searchers/search_disjunction_test.go +++ b/search/searchers/search_disjunction_test.go @@ -12,6 +12,7 @@ package searchers import ( "testing" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -71,12 +72,12 @@ func TestDisjunctionSearch(t *testing.T) { searcher: martyOrDustinSearcher, results: []*search.DocumentMatch{ { - ID: "1", - Score: 0.6775110856165737, + IndexInternalID: index.IndexInternalID("1"), + Score: 0.6775110856165737, }, { - ID: "3", - Score: 0.6775110856165737, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.6775110856165737, }, }, }, @@ -85,16 +86,16 @@ func TestDisjunctionSearch(t *testing.T) { searcher: nestedRaviOrMartyOrDustinSearcher, results: []*search.DocumentMatch{ { - ID: "1", - Score: 0.2765927424732821, + IndexInternalID: index.IndexInternalID("1"), + Score: 0.2765927424732821, }, { - ID: "3", - Score: 0.2765927424732821, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.2765927424732821, }, { - ID: "4", - Score: 0.5531854849465642, + IndexInternalID: index.IndexInternalID("4"), + Score: 0.5531854849465642, }, }, }, @@ -108,19 +109,23 @@ func TestDisjunctionSearch(t *testing.T) { } }() - next, err := test.searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()), + } + next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { - if next.ID != test.results[i].ID { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) t.Logf("scoring explanation: %s", next.Expl) } } - next, err = test.searcher.Next(nil) + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) i++ } if err != nil { @@ -158,7 +163,10 @@ func TestDisjunctionAdvance(t *testing.T) { t.Fatal(err) } - match, err := martyOrDustinSearcher.Advance("3", nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(martyOrDustinSearcher.DocumentMatchPoolSize()), + } + match, err := martyOrDustinSearcher.Advance(ctx, index.IndexInternalID("3")) if err != nil { t.Errorf("unexpected error: %v", err) } diff --git a/search/searchers/search_docid.go b/search/searchers/search_docid.go index 262583d0..33b9b4c9 100644 --- a/search/searchers/search_docid.go +++ b/search/searchers/search_docid.go @@ -10,8 +10,6 @@ package searchers import ( - "sort" - "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search/scorers" @@ -19,54 +17,28 @@ import ( // DocIDSearcher returns documents matching a predefined set of identifiers. type DocIDSearcher struct { - ids []string - current int - scorer *scorers.ConstantScorer + reader index.DocIDReader + scorer *scorers.ConstantScorer + count int } func NewDocIDSearcher(indexReader index.IndexReader, ids []string, boost float64, explain bool) (searcher *DocIDSearcher, err error) { - kept := make([]string, len(ids)) - copy(kept, ids) - sort.Strings(kept) - - if len(ids) > 0 { - var idReader index.DocIDReader - endTerm := string(incrementBytes([]byte(kept[len(kept)-1]))) - idReader, err = indexReader.DocIDReader(kept[0], endTerm) - if err != nil { - return nil, err - } - defer func() { - if cerr := idReader.Close(); err == nil && cerr != nil { - err = cerr - } - }() - j := 0 - for _, id := range kept { - doc, err := idReader.Advance(id) - if err != nil { - return nil, err - } - // Non-duplicate match - if doc == id && (j == 0 || kept[j-1] != id) { - kept[j] = id - j++ - } - } - kept = kept[:j] + reader, err := indexReader.DocIDReaderOnly(ids) + if err != nil { + return nil, err } - scorer := scorers.NewConstantScorer(1.0, boost, explain) return &DocIDSearcher{ - ids: kept, scorer: scorer, + reader: reader, + count: len(ids), }, nil } func (s *DocIDSearcher) Count() uint64 { - return uint64(len(s.ids)) + return uint64(s.count) } func (s *DocIDSearcher) Weight() float64 { @@ -77,20 +49,30 @@ func (s *DocIDSearcher) SetQueryNorm(qnorm float64) { s.scorer.SetQueryNorm(qnorm) } -func (s *DocIDSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - if s.current >= len(s.ids) { +func (s *DocIDSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + docidMatch, err := s.reader.Next() + if err != nil { + return nil, err + } + if docidMatch == nil { return nil, nil } - id := s.ids[s.current] - s.current++ - docMatch := s.scorer.Score(id) - return docMatch, nil + docMatch := s.scorer.Score(ctx, docidMatch) + return docMatch, nil } -func (s *DocIDSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - s.current = sort.SearchStrings(s.ids, ID) - return s.Next(preAllocated) +func (s *DocIDSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + docidMatch, err := s.reader.Advance(ID) + if err != nil { + return nil, err + } + if docidMatch == nil { + return nil, nil + } + + docMatch := s.scorer.Score(ctx, docidMatch) + return docMatch, nil } func (s *DocIDSearcher) Close() error { @@ -100,3 +82,7 @@ func (s *DocIDSearcher) Close() error { func (s *DocIDSearcher) Min() int { return 0 } + +func (s *DocIDSearcher) DocumentMatchPoolSize() int { + return 1 +} diff --git a/search/searchers/search_docid_test.go b/search/searchers/search_docid_test.go index ad89288b..00fe3db9 100644 --- a/search/searchers/search_docid_test.go +++ b/search/searchers/search_docid_test.go @@ -16,6 +16,7 @@ import ( "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index/store/gtreap" "github.com/blevesearch/bleve/index/upside_down" + "github.com/blevesearch/bleve/search" ) func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { @@ -62,27 +63,29 @@ func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { } }() - if searcher.Count() != uint64(len(wanted)) { - t.Fatalf("expected count %v got %v", len(wanted), searcher.Count()) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(searcher.DocumentMatchPoolSize()), } // Check the sequence for i, id := range wanted { - m, err := searcher.Next(nil) + m, err := searcher.Next(ctx) if err != nil { t.Fatal(err) } - if id != m.ID { - t.Fatalf("expected %v at position %v, got %v", id, i, m.ID) + if !index.IndexInternalID(id).Equals(m.IndexInternalID) { + t.Fatalf("expected %v at position %v, got %v", id, i, m.IndexInternalID) } + ctx.DocumentMatchPool.Put(m) } - m, err := searcher.Next(nil) + m, err := searcher.Next(ctx) if err != nil { t.Fatal(err) } if m != nil { - t.Fatalf("expected nil past the end of the sequence, got %v", m.ID) + t.Fatalf("expected nil past the end of the sequence, got %v", m.IndexInternalID) } + ctx.DocumentMatchPool.Put(m) // Check seeking for _, id := range wanted { @@ -91,24 +94,26 @@ func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { } before := id[:1] for _, target := range []string{before, id} { - m, err := searcher.Advance(target, nil) + m, err := searcher.Advance(ctx, index.IndexInternalID(target)) if err != nil { t.Fatal(err) } - if m == nil || m.ID != id { + if m == nil || !m.IndexInternalID.Equals(index.IndexInternalID(id)) { t.Fatalf("advancing to %v returned %v instead of %v", before, m, id) } + ctx.DocumentMatchPool.Put(m) } } // Seek after the end of the sequence after := "zzz" - m, err = searcher.Advance(after, nil) + m, err = searcher.Advance(ctx, index.IndexInternalID(after)) if err != nil { t.Fatal(err) } if m != nil { t.Fatalf("advancing past the end of the sequence should return nil, got %v", m) } + ctx.DocumentMatchPool.Put(m) } func TestDocIDSearcherEmptySearchEmptyIndex(t *testing.T) { diff --git a/search/searchers/search_fuzzy.go b/search/searchers/search_fuzzy.go index da328a7e..2cd44ebf 100644 --- a/search/searchers/search_fuzzy.go +++ b/search/searchers/search_fuzzy.go @@ -107,13 +107,13 @@ func (s *FuzzySearcher) SetQueryNorm(qnorm float64) { s.searcher.SetQueryNorm(qnorm) } -func (s *FuzzySearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - return s.searcher.Next(preAllocated) +func (s *FuzzySearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + return s.searcher.Next(ctx) } -func (s *FuzzySearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - return s.searcher.Advance(ID, preAllocated) +func (s *FuzzySearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + return s.searcher.Advance(ctx, ID) } func (s *FuzzySearcher) Close() error { @@ -123,3 +123,7 @@ func (s *FuzzySearcher) Close() error { func (s *FuzzySearcher) Min() int { return 0 } + +func (s *FuzzySearcher) DocumentMatchPoolSize() int { + return s.searcher.DocumentMatchPoolSize() +} diff --git a/search/searchers/search_fuzzy_test.go b/search/searchers/search_fuzzy_test.go index 610367de..b4469666 100644 --- a/search/searchers/search_fuzzy_test.go +++ b/search/searchers/search_fuzzy_test.go @@ -12,6 +12,7 @@ package searchers import ( "testing" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -56,20 +57,20 @@ func TestFuzzySearch(t *testing.T) { searcher: fuzzySearcherbeet, results: []*search.DocumentMatch{ { - ID: "1", - Score: 1.0, + IndexInternalID: index.IndexInternalID("1"), + Score: 1.0, }, { - ID: "2", - Score: 0.5, + IndexInternalID: index.IndexInternalID("2"), + Score: 0.5, }, { - ID: "3", - Score: 0.5, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.5, }, { - ID: "4", - Score: 0.9999999838027345, + IndexInternalID: index.IndexInternalID("4"), + Score: 0.9999999838027345, }, }, }, @@ -81,8 +82,8 @@ func TestFuzzySearch(t *testing.T) { searcher: fuzzySearcheraplee, results: []*search.DocumentMatch{ { - ID: "3", - Score: 0.9581453659370776, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.9581453659370776, }, }, }, @@ -90,8 +91,8 @@ func TestFuzzySearch(t *testing.T) { searcher: fuzzySearcherprefix, results: []*search.DocumentMatch{ { - ID: "5", - Score: 1.916290731874155, + IndexInternalID: index.IndexInternalID("5"), + Score: 1.916290731874155, }, }, }, @@ -105,19 +106,23 @@ func TestFuzzySearch(t *testing.T) { } }() - next, err := test.searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()), + } + next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { - if next.ID != test.results[i].ID { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if next.Score != test.results[i].Score { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) t.Logf("scoring explanation: %s", next.Expl) } } - next, err = test.searcher.Next(nil) + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) i++ } if err != nil { diff --git a/search/searchers/search_match_all.go b/search/searchers/search_match_all.go index bd26ba13..9834d31a 100644 --- a/search/searchers/search_match_all.go +++ b/search/searchers/search_match_all.go @@ -46,35 +46,35 @@ func (s *MatchAllSearcher) SetQueryNorm(qnorm float64) { s.scorer.SetQueryNorm(qnorm) } -func (s *MatchAllSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *MatchAllSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { id, err := s.reader.Next() if err != nil { return nil, err } - if id == "" { + if id == nil { return nil, nil } // score match - docMatch := s.scorer.Score(id) + docMatch := s.scorer.Score(ctx, id) // return doc match return docMatch, nil } -func (s *MatchAllSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *MatchAllSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { id, err := s.reader.Advance(ID) if err != nil { return nil, err } - if id == "" { + if id == nil { return nil, nil } // score match - docMatch := s.scorer.Score(id) + docMatch := s.scorer.Score(ctx, id) // return doc match return docMatch, nil @@ -87,3 +87,7 @@ func (s *MatchAllSearcher) Close() error { func (s *MatchAllSearcher) Min() int { return 0 } + +func (s *MatchAllSearcher) DocumentMatchPoolSize() int { + return 1 +} diff --git a/search/searchers/search_match_all_test.go b/search/searchers/search_match_all_test.go index 5bcd1b51..26e8a3c2 100644 --- a/search/searchers/search_match_all_test.go +++ b/search/searchers/search_match_all_test.go @@ -12,6 +12,7 @@ package searchers import ( "testing" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -48,24 +49,24 @@ func TestMatchAllSearch(t *testing.T) { queryNorm: 1.0, results: []*search.DocumentMatch{ { - ID: "1", - Score: 1.0, + IndexInternalID: index.IndexInternalID("1"), + Score: 1.0, }, { - ID: "2", - Score: 1.0, + IndexInternalID: index.IndexInternalID("2"), + Score: 1.0, }, { - ID: "3", - Score: 1.0, + IndexInternalID: index.IndexInternalID("3"), + Score: 1.0, }, { - ID: "4", - Score: 1.0, + IndexInternalID: index.IndexInternalID("4"), + Score: 1.0, }, { - ID: "5", - Score: 1.0, + IndexInternalID: index.IndexInternalID("5"), + Score: 1.0, }, }, }, @@ -74,24 +75,24 @@ func TestMatchAllSearch(t *testing.T) { queryNorm: 0.8333333, results: []*search.DocumentMatch{ { - ID: "1", - Score: 1.0, + IndexInternalID: index.IndexInternalID("1"), + Score: 1.0, }, { - ID: "2", - Score: 1.0, + IndexInternalID: index.IndexInternalID("2"), + Score: 1.0, }, { - ID: "3", - Score: 1.0, + IndexInternalID: index.IndexInternalID("3"), + Score: 1.0, }, { - ID: "4", - Score: 1.0, + IndexInternalID: index.IndexInternalID("4"), + Score: 1.0, }, { - ID: "5", - Score: 1.0, + IndexInternalID: index.IndexInternalID("5"), + Score: 1.0, }, }, }, @@ -109,19 +110,23 @@ func TestMatchAllSearch(t *testing.T) { } }() - next, err := test.searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()), + } + next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { - if next.ID != test.results[i].ID { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) t.Logf("scoring explanation: %s", next.Expl) } } - next, err = test.searcher.Next(nil) + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) i++ } if err != nil { diff --git a/search/searchers/search_match_none.go b/search/searchers/search_match_none.go index 0d4f5a9a..c3159dcf 100644 --- a/search/searchers/search_match_none.go +++ b/search/searchers/search_match_none.go @@ -36,11 +36,11 @@ func (s *MatchNoneSearcher) SetQueryNorm(qnorm float64) { } -func (s *MatchNoneSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *MatchNoneSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { return nil, nil } -func (s *MatchNoneSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *MatchNoneSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { return nil, nil } @@ -51,3 +51,7 @@ func (s *MatchNoneSearcher) Close() error { func (s *MatchNoneSearcher) Min() int { return 0 } + +func (s *MatchNoneSearcher) DocumentMatchPoolSize() int { + return 0 +} diff --git a/search/searchers/search_match_none_test.go b/search/searchers/search_match_none_test.go index 0d470358..90ec526c 100644 --- a/search/searchers/search_match_none_test.go +++ b/search/searchers/search_match_none_test.go @@ -51,19 +51,23 @@ func TestMatchNoneSearch(t *testing.T) { } }() - next, err := test.searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()), + } + next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { - if next.ID != test.results[i].ID { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) t.Logf("scoring explanation: %s", next.Expl) } } - next, err = test.searcher.Next(nil) + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) i++ } if err != nil { diff --git a/search/searchers/search_numeric_range.go b/search/searchers/search_numeric_range.go index fdf3c4c0..03cd4959 100644 --- a/search/searchers/search_numeric_range.go +++ b/search/searchers/search_numeric_range.go @@ -96,12 +96,12 @@ func (s *NumericRangeSearcher) SetQueryNorm(qnorm float64) { s.searcher.SetQueryNorm(qnorm) } -func (s *NumericRangeSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - return s.searcher.Next(preAllocated) +func (s *NumericRangeSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + return s.searcher.Next(ctx) } -func (s *NumericRangeSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - return s.searcher.Advance(ID, preAllocated) +func (s *NumericRangeSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + return s.searcher.Advance(ctx, ID) } func (s *NumericRangeSearcher) Close() error { @@ -215,3 +215,7 @@ func newRangeBytes(minBytes, maxBytes []byte) *termRange { func (s *NumericRangeSearcher) Min() int { return 0 } + +func (s *NumericRangeSearcher) DocumentMatchPoolSize() int { + return s.searcher.DocumentMatchPoolSize() +} diff --git a/search/searchers/search_phrase.go b/search/searchers/search_phrase.go index 11e8c9c8..6456420f 100644 --- a/search/searchers/search_phrase.go +++ b/search/searchers/search_phrase.go @@ -52,11 +52,11 @@ func (s *PhraseSearcher) computeQueryNorm() { } } -func (s *PhraseSearcher) initSearchers() error { +func (s *PhraseSearcher) initSearchers(ctx *search.SearchContext) error { var err error // get all searchers pointing at their first match if s.mustSearcher != nil { - s.currMust, err = s.mustSearcher.Next(nil) + s.currMust, err = s.mustSearcher.Next(ctx) if err != nil { return err } @@ -66,11 +66,11 @@ func (s *PhraseSearcher) initSearchers() error { return nil } -func (s *PhraseSearcher) advanceNextMust() error { +func (s *PhraseSearcher) advanceNextMust(ctx *search.SearchContext) error { var err error if s.mustSearcher != nil { - s.currMust, err = s.mustSearcher.Next(nil) + s.currMust, err = s.mustSearcher.Next(ctx) if err != nil { return err } @@ -90,9 +90,9 @@ func (s *PhraseSearcher) SetQueryNorm(qnorm float64) { s.mustSearcher.SetQueryNorm(qnorm) } -func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *PhraseSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { if !s.initialized { - err := s.initSearchers() + err := s.initSearchers(ctx) if err != nil { return nil, err } @@ -144,14 +144,14 @@ func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docum // return match rv = s.currMust rv.Locations = rvftlm - err := s.advanceNextMust() + err := s.advanceNextMust(ctx) if err != nil { return nil, err } return rv, nil } - err := s.advanceNextMust() + err := s.advanceNextMust(ctx) if err != nil { return nil, err } @@ -160,19 +160,19 @@ func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docum return nil, nil } -func (s *PhraseSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *PhraseSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { if !s.initialized { - err := s.initSearchers() + err := s.initSearchers(ctx) if err != nil { return nil, err } } var err error - s.currMust, err = s.mustSearcher.Advance(ID, nil) + s.currMust, err = s.mustSearcher.Advance(ctx, ID) if err != nil { return nil, err } - return s.Next(preAllocated) + return s.Next(ctx) } func (s *PhraseSearcher) Count() uint64 { @@ -195,3 +195,7 @@ func (s *PhraseSearcher) Close() error { func (s *PhraseSearcher) Min() int { return 0 } + +func (s *PhraseSearcher) DocumentMatchPoolSize() int { + return s.mustSearcher.DocumentMatchPoolSize() + 1 +} diff --git a/search/searchers/search_phrase_test.go b/search/searchers/search_phrase_test.go index a3c14b70..8c9a76d2 100644 --- a/search/searchers/search_phrase_test.go +++ b/search/searchers/search_phrase_test.go @@ -12,6 +12,7 @@ package searchers import ( "testing" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -53,8 +54,8 @@ func TestPhraseSearch(t *testing.T) { searcher: phraseSearcher, results: []*search.DocumentMatch{ { - ID: "2", - Score: 1.0807601687084403, + IndexInternalID: index.IndexInternalID("2"), + Score: 1.0807601687084403, }, }, }, @@ -68,19 +69,23 @@ func TestPhraseSearch(t *testing.T) { } }() - next, err := test.searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()), + } + next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { - if next.ID != test.results[i].ID { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if next.Score != test.results[i].Score { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) t.Logf("scoring explanation: %s", next.Expl) } } - next, err = test.searcher.Next(nil) + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) i++ } if err != nil { diff --git a/search/searchers/search_regexp.go b/search/searchers/search_regexp.go index d92822f1..3ce61d82 100644 --- a/search/searchers/search_regexp.go +++ b/search/searchers/search_regexp.go @@ -106,13 +106,13 @@ func (s *RegexpSearcher) SetQueryNorm(qnorm float64) { s.searcher.SetQueryNorm(qnorm) } -func (s *RegexpSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - return s.searcher.Next(preAllocated) +func (s *RegexpSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + return s.searcher.Next(ctx) } -func (s *RegexpSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - return s.searcher.Advance(ID, preAllocated) +func (s *RegexpSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + return s.searcher.Advance(ctx, ID) } func (s *RegexpSearcher) Close() error { @@ -122,3 +122,7 @@ func (s *RegexpSearcher) Close() error { func (s *RegexpSearcher) Min() int { return 0 } + +func (s *RegexpSearcher) DocumentMatchPoolSize() int { + return s.searcher.DocumentMatchPoolSize() +} diff --git a/search/searchers/search_regexp_test.go b/search/searchers/search_regexp_test.go index cb4d01ee..f2859cb9 100644 --- a/search/searchers/search_regexp_test.go +++ b/search/searchers/search_regexp_test.go @@ -13,6 +13,7 @@ import ( "regexp" "testing" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -57,8 +58,8 @@ func TestRegexpSearch(t *testing.T) { searcher: regexpSearcher, results: []*search.DocumentMatch{ { - ID: "1", - Score: 1.916290731874155, + IndexInternalID: index.IndexInternalID("1"), + Score: 1.916290731874155, }, }, }, @@ -66,12 +67,12 @@ func TestRegexpSearch(t *testing.T) { searcher: regexpSearcherCo, results: []*search.DocumentMatch{ { - ID: "2", - Score: 0.33875554280828685, + IndexInternalID: index.IndexInternalID("2"), + Score: 0.33875554280828685, }, { - ID: "3", - Score: 0.33875554280828685, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.33875554280828685, }, }, }, @@ -85,19 +86,23 @@ func TestRegexpSearch(t *testing.T) { } }() - next, err := test.searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()), + } + next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { - if next.ID != test.results[i].ID { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if next.Score != test.results[i].Score { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) t.Logf("scoring explanation: %s", next.Expl) } } - next, err = test.searcher.Next(nil) + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) i++ } if err != nil { diff --git a/search/searchers/search_term.go b/search/searchers/search_term.go index ff034112..3879723c 100644 --- a/search/searchers/search_term.go +++ b/search/searchers/search_term.go @@ -26,7 +26,7 @@ type TermSearcher struct { } func NewTermSearcher(indexReader index.IndexReader, term string, field string, boost float64, explain bool) (*TermSearcher, error) { - reader, err := indexReader.TermFieldReader([]byte(term), field) + reader, err := indexReader.TermFieldReader([]byte(term), field, true, true, true) if err != nil { return nil, err } @@ -53,7 +53,7 @@ func (s *TermSearcher) SetQueryNorm(qnorm float64) { s.scorer.SetQueryNorm(qnorm) } -func (s *TermSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *TermSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { termMatch, err := s.reader.Next(s.tfd.Reset()) if err != nil { return nil, err @@ -64,13 +64,13 @@ func (s *TermSearcher) Next(preAllocated *search.DocumentMatch) (*search.Documen } // score match - docMatch := s.scorer.Score(termMatch, preAllocated) + docMatch := s.scorer.Score(ctx, termMatch) // return doc match return docMatch, nil } -func (s *TermSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *TermSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { termMatch, err := s.reader.Advance(ID, s.tfd.Reset()) if err != nil { return nil, err @@ -81,7 +81,7 @@ func (s *TermSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (* } // score match - docMatch := s.scorer.Score(termMatch, preAllocated) + docMatch := s.scorer.Score(ctx, termMatch) // return doc match return docMatch, nil @@ -94,3 +94,7 @@ func (s *TermSearcher) Close() error { func (s *TermSearcher) Min() int { return 0 } + +func (s *TermSearcher) DocumentMatchPoolSize() int { + return 1 +} diff --git a/search/searchers/search_term_prefix.go b/search/searchers/search_term_prefix.go index 35f34722..35caf717 100644 --- a/search/searchers/search_term_prefix.go +++ b/search/searchers/search_term_prefix.go @@ -70,13 +70,13 @@ func (s *TermPrefixSearcher) SetQueryNorm(qnorm float64) { s.searcher.SetQueryNorm(qnorm) } -func (s *TermPrefixSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - return s.searcher.Next(preAllocated) +func (s *TermPrefixSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + return s.searcher.Next(ctx) } -func (s *TermPrefixSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - return s.searcher.Advance(ID, preAllocated) +func (s *TermPrefixSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + return s.searcher.Advance(ctx, ID) } func (s *TermPrefixSearcher) Close() error { @@ -86,3 +86,7 @@ func (s *TermPrefixSearcher) Close() error { func (s *TermPrefixSearcher) Min() int { return 0 } + +func (s *TermPrefixSearcher) DocumentMatchPoolSize() int { + return s.searcher.DocumentMatchPoolSize() +} diff --git a/search/searchers/search_term_test.go b/search/searchers/search_term_test.go index ef3b927f..c94803e1 100644 --- a/search/searchers/search_term_test.go +++ b/search/searchers/search_term_test.go @@ -17,6 +17,7 @@ import ( "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index/store/gtreap" "github.com/blevesearch/bleve/index/upside_down" + "github.com/blevesearch/bleve/search" ) func TestTermSearcher(t *testing.T) { @@ -163,23 +164,28 @@ func TestTermSearcher(t *testing.T) { t.Errorf("expected count of 9, got %d", searcher.Count()) } - docMatch, err := searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(1), + } + docMatch, err := searcher.Next(ctx) if err != nil { t.Errorf("expected result, got %v", err) } - if docMatch.ID != "a" { - t.Errorf("expected result ID to be 'a', got '%s", docMatch.ID) + if !docMatch.IndexInternalID.Equals(index.IndexInternalID("a")) { + t.Errorf("expected result ID to be 'a', got '%s", docMatch.IndexInternalID) } - docMatch, err = searcher.Advance("c", nil) + ctx.DocumentMatchPool.Put(docMatch) + docMatch, err = searcher.Advance(ctx, index.IndexInternalID("c")) if err != nil { t.Errorf("expected result, got %v", err) } - if docMatch.ID != "c" { - t.Errorf("expected result ID to be 'c' got '%s'", docMatch.ID) + if !docMatch.IndexInternalID.Equals(index.IndexInternalID("c")) { + t.Errorf("expected result ID to be 'c' got '%s'", docMatch.IndexInternalID) } // try advancing past end - docMatch, err = searcher.Advance("z", nil) + ctx.DocumentMatchPool.Put(docMatch) + docMatch, err = searcher.Advance(ctx, index.IndexInternalID("z")) if err != nil { t.Fatal(err) } @@ -188,7 +194,8 @@ func TestTermSearcher(t *testing.T) { } // try pushing next past end - docMatch, err = searcher.Next(nil) + ctx.DocumentMatchPool.Put(docMatch) + docMatch, err = searcher.Next(ctx) if err != nil { t.Fatal(err) }