From 47ee69ae82fdd239529a2bad8ba8380d35b6b1f8 Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Sat, 30 Jul 2016 10:26:42 -0400 Subject: [PATCH 01/13] term field reader supports optionally omitting 3 details at the time you create the term field reader, you can specify that you don't need the term freq, the norm, or the term vectors in that case, the index implementation can choose to not return them in its subsequently returned values this is advisory only, some simple implementations may ignore this and continue to return the values anyway (as the current impl of upside_down does today) this change will allow future index implementations the opportunity to do less work when it isn't required --- index/index.go | 2 +- index/upside_down/index_reader.go | 6 +++--- index/upside_down/reader.go | 2 +- index/upside_down/reader_test.go | 10 +++++----- index/upside_down/upside_down_test.go | 2 +- search/searchers/search_term.go | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/index/index.go b/index/index.go index bf9f394a..93b72929 100644 --- a/index/index.go +++ b/index/index.go @@ -60,7 +60,7 @@ type AsyncIndex interface { } type IndexReader interface { - TermFieldReader(term []byte, field string) (TermFieldReader, error) + TermFieldReader(term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (TermFieldReader, error) // DocIDReader returns an iterator over documents which identifiers are // greater than or equal to start and smaller than end. Set start to the diff --git a/index/upside_down/index_reader.go b/index/upside_down/index_reader.go index 49655adf..e8047144 100644 --- a/index/upside_down/index_reader.go +++ b/index/upside_down/index_reader.go @@ -23,12 +23,12 @@ type IndexReader struct { docCount uint64 } -func (i *IndexReader) TermFieldReader(term []byte, fieldName string) (index.TermFieldReader, error) { +func (i *IndexReader) TermFieldReader(term []byte, fieldName string, includeFreq, includeNorm, includeTermVectors bool) (index.TermFieldReader, error) { fieldIndex, fieldExists := i.index.fieldCache.FieldNamed(fieldName, false) if fieldExists { - return newUpsideDownCouchTermFieldReader(i, term, uint16(fieldIndex)) + return newUpsideDownCouchTermFieldReader(i, term, uint16(fieldIndex), includeFreq, includeNorm, includeTermVectors) } - return newUpsideDownCouchTermFieldReader(i, []byte{ByteSeparator}, ^uint16(0)) + return newUpsideDownCouchTermFieldReader(i, []byte{ByteSeparator}, ^uint16(0), includeFreq, includeNorm, includeTermVectors) } func (i *IndexReader) FieldDict(fieldName string) (index.FieldDict, error) { diff --git a/index/upside_down/reader.go b/index/upside_down/reader.go index 53007ea0..a973329e 100644 --- a/index/upside_down/reader.go +++ b/index/upside_down/reader.go @@ -25,7 +25,7 @@ type UpsideDownCouchTermFieldReader struct { field uint16 } -func newUpsideDownCouchTermFieldReader(indexReader *IndexReader, term []byte, field uint16) (*UpsideDownCouchTermFieldReader, error) { +func newUpsideDownCouchTermFieldReader(indexReader *IndexReader, term []byte, field uint16, includeFreq, includeNorm, includeTermVectors bool) (*UpsideDownCouchTermFieldReader, error) { dictionaryRow := NewDictionaryRow(term, field, 0) val, err := indexReader.kvreader.Get(dictionaryRow.Key()) if err != nil { diff --git a/index/upside_down/reader_test.go b/index/upside_down/reader_test.go index 124b1462..62e81398 100644 --- a/index/upside_down/reader_test.go +++ b/index/upside_down/reader_test.go @@ -72,7 +72,7 @@ func TestIndexReader(t *testing.T) { }() // first look for a term that doesn't exist - reader, err := indexReader.TermFieldReader([]byte("nope"), "name") + reader, err := indexReader.TermFieldReader([]byte("nope"), "name", true, true, true) if err != nil { t.Errorf("Error accessing term field reader: %v", err) } @@ -85,7 +85,7 @@ func TestIndexReader(t *testing.T) { t.Fatal(err) } - reader, err = indexReader.TermFieldReader([]byte("test"), "name") + reader, err = indexReader.TermFieldReader([]byte("test"), "name", true, true, true) if err != nil { t.Errorf("Error accessing term field reader: %v", err) } @@ -123,7 +123,7 @@ func TestIndexReader(t *testing.T) { }, }, } - tfr, err := indexReader.TermFieldReader([]byte("rice"), "desc") + tfr, err := indexReader.TermFieldReader([]byte("rice"), "desc", true, true, true) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -140,7 +140,7 @@ func TestIndexReader(t *testing.T) { } // now test usage of advance - reader, err = indexReader.TermFieldReader([]byte("test"), "name") + reader, err = indexReader.TermFieldReader([]byte("test"), "name", true, true, true) if err != nil { t.Errorf("Error accessing term field reader: %v", err) } @@ -168,7 +168,7 @@ func TestIndexReader(t *testing.T) { } // now test creating a reader for a field that doesn't exist - reader, err = indexReader.TermFieldReader([]byte("water"), "doesnotexist") + reader, err = indexReader.TermFieldReader([]byte("water"), "doesnotexist", true, true, true) if err != nil { t.Errorf("Error accessing term field reader: %v", err) } diff --git a/index/upside_down/upside_down_test.go b/index/upside_down/upside_down_test.go index 85a62bc0..3a1fc004 100644 --- a/index/upside_down/upside_down_test.go +++ b/index/upside_down/upside_down_test.go @@ -1119,7 +1119,7 @@ func TestIndexTermReaderCompositeFields(t *testing.T) { } }() - termFieldReader, err := indexReader.TermFieldReader([]byte("mister"), "_all") + termFieldReader, err := indexReader.TermFieldReader([]byte("mister"), "_all", true, true, true) if err != nil { t.Error(err) } diff --git a/search/searchers/search_term.go b/search/searchers/search_term.go index ff034112..94c3eba9 100644 --- a/search/searchers/search_term.go +++ b/search/searchers/search_term.go @@ -26,7 +26,7 @@ type TermSearcher struct { } func NewTermSearcher(indexReader index.IndexReader, term string, field string, boost float64, explain bool) (*TermSearcher, error) { - reader, err := indexReader.TermFieldReader([]byte(term), field) + reader, err := indexReader.TermFieldReader([]byte(term), field, true, true, true) if err != nil { return nil, err } From 5aa9e954687e82541ee37ed70f7fdcd709ca6f69 Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Sun, 31 Jul 2016 13:46:18 -0400 Subject: [PATCH 02/13] major refactor of index/search API index id's are now opaque (until finally returned to top-level user) - the TermFieldDoc's returned by TermFieldReader no longer contain doc id - instead they return an opaque IndexInternalID - items returned are still in the "natural index order" - but that is no longer guaranteed to be "doc id order" - correct behavior requires that they all follow the same order - but not any particular order - new API FinalizeDocID which converts index internal ID's to public string ID - APIs used internally which previously took doc id now take IndexInternalID - that is DocumentFieldTerms() and DocumentFieldTermsForFields() - however, APIs that are used externally do not reflect this change - that is Document() - DocumentIDReader follows the same changes, but this is less obvious - behavior clarified, used to iterate doc ids, BUT NOT in doc id order - method STILL available to iterate doc ids in range - but again, you won't get them in any meaningful order - new method to iterate actual doc ids from list of possible ids - this was introduced to make the DocIDSearcher continue working searchers now work with the new opaque index internal doc ids - they return new DocumentMatchInternal (which does not have string ID) scorerers also work with these opaque index internal doc ids - they return DocumentMatchInternal (which does not have string ID) collectors now also perform a final step of converting the final result - they STILL return traditional DocumentMatch (with string ID) - but they now also require an IndexReader (so that they can do the conversion) --- index/index.go | 43 ++-- index/upside_down/dump.go | 2 +- index/upside_down/index_reader.go | 33 ++- index/upside_down/reader.go | 157 ++++++++++-- index/upside_down/reader_test.go | 20 +- index/upside_down/upside_down.go | 10 +- index/upside_down/upside_down_test.go | 12 +- index_impl.go | 2 +- search/collector.go | 4 +- search/collectors/bench_test.go | 8 +- search/collectors/collector_top_score.go | 38 ++- search/collectors/collector_top_score_test.go | 240 +++++++++--------- search/collectors/search_test.go | 80 +++++- search/facets_builder.go | 2 +- search/scorers/scorer_conjunction.go | 4 +- search/scorers/scorer_constant.go | 5 +- search/scorers/scorer_constant_test.go | 16 +- search/scorers/scorer_disjunction.go | 4 +- search/scorers/scorer_term.go | 4 +- search/scorers/scorer_term_test.go | 39 ++- search/search.go | 40 ++- search/searchers/search_boolean.go | 50 ++-- search/searchers/search_boolean_test.go | 45 ++-- search/searchers/search_conjunction.go | 28 +- search/searchers/search_conjunction_test.go | 29 ++- search/searchers/search_disjunction.go | 28 +- search/searchers/search_disjunction_test.go | 21 +- search/searchers/search_docid.go | 116 +++++---- search/searchers/search_docid_test.go | 12 +- search/searchers/search_fuzzy.go | 4 +- search/searchers/search_fuzzy_test.go | 25 +- search/searchers/search_match_all.go | 8 +- search/searchers/search_match_all_test.go | 29 ++- search/searchers/search_match_none.go | 4 +- search/searchers/search_match_none_test.go | 4 +- search/searchers/search_numeric_range.go | 4 +- search/searchers/search_phrase.go | 8 +- search/searchers/search_phrase_test.go | 9 +- search/searchers/search_regexp.go | 4 +- search/searchers/search_regexp_test.go | 15 +- search/searchers/search_term.go | 4 +- search/searchers/search_term_prefix.go | 4 +- search/searchers/search_term_test.go | 8 +- 43 files changed, 766 insertions(+), 456 deletions(-) diff --git a/index/index.go b/index/index.go index 93b72929..6804bb3a 100644 --- a/index/index.go +++ b/index/index.go @@ -69,6 +69,8 @@ type IndexReader interface { // The caller must close returned instance to release associated resources. DocIDReader(start, end string) (DocIDReader, error) + DocIDReaderOnly(ids []string) (DocIDReader, error) + FieldDict(field string) (FieldDict, error) // FieldDictRange is currently defined to include the start and end terms @@ -76,8 +78,8 @@ type IndexReader interface { FieldDictPrefix(field string, termPrefix []byte) (FieldDict, error) Document(id string) (*document.Document, error) - DocumentFieldTerms(id string) (FieldTerms, error) - DocumentFieldTermsForFields(id string, fields []string) (FieldTerms, error) + DocumentFieldTerms(id IndexInternalID) (FieldTerms, error) + DocumentFieldTermsForFields(id IndexInternalID, fields []string) (FieldTerms, error) Fields() ([]string, error) @@ -85,6 +87,8 @@ type IndexReader interface { DocCount() uint64 + FinalizeDocID(id IndexInternalID) (string, error) + Close() error } @@ -98,14 +102,28 @@ type TermFieldVector struct { End uint64 } +// IndexInternalID is an opaque document identifier interal to the index impl +// This allows us to delay the conversion to public identifier (string) and +// avoid it completely in other cases. It also servces to hide the underlying +// representation of a document identifer, allow more flexibility. +type IndexInternalID interface { + Equals(other IndexInternalID) bool + Compare(other IndexInternalID) int +} + type TermFieldDoc struct { Term string - ID string + ID IndexInternalID Freq uint64 Norm float64 Vectors []*TermFieldVector } +func (tfd *TermFieldDoc) Reset() *TermFieldDoc { + *tfd = TermFieldDoc{} + return tfd +} + // TermFieldReader is the interface exposing the enumeration of documents // containing a given term in a given field. Documents are returned in byte // lexicographic order over their identifiers. @@ -117,7 +135,7 @@ type TermFieldReader interface { // Advance resets the enumeration at specified document or its immediate // follower. - Advance(ID string, preAlloced *TermFieldDoc) (*TermFieldDoc, error) + Advance(ID IndexInternalID, preAlloced *TermFieldDoc) (*TermFieldDoc, error) // Count returns the number of documents contains the term in this field. Count() uint64 @@ -137,15 +155,15 @@ type FieldDict interface { // DocIDReader is the interface exposing enumeration of documents identifiers. // Close the reader to release associated resources. type DocIDReader interface { - // Next returns the next document identifier in ascending lexicographic - // byte order, or io.EOF when the end of the sequence is reached. - Next() (string, error) + // Next returns the next document internal identifier in the natural + // index order, or io.EOF when the end of the sequence is reached. + Next() (IndexInternalID, error) - // Advance resets the iteration to the first identifier greater than or - // equal to ID. If ID is smaller than the start of the range, the iteration + // Advance resets the iteration to the first internal identifier greater than + // or equal to ID. If ID is smaller than the start of the range, the iteration // will start there instead. If ID is greater than or equal to the end of // the range, Next() call will return io.EOF. - Advance(ID string) (string, error) + Advance(ID IndexInternalID) (IndexInternalID, error) Close() error } @@ -200,8 +218,3 @@ func (b *Batch) Reset() { b.IndexOps = make(map[string]*document.Document) b.InternalOps = make(map[string][]byte) } - -func (tfd *TermFieldDoc) Reset() *TermFieldDoc { - *tfd = TermFieldDoc{} - return tfd -} diff --git a/index/upside_down/dump.go b/index/upside_down/dump.go index 023ae458..fd2be837 100644 --- a/index/upside_down/dump.go +++ b/index/upside_down/dump.go @@ -151,7 +151,7 @@ func (udc *UpsideDownCouch) DumpDoc(id string) chan interface{} { } }() - back, err := udc.backIndexRowForDoc(kvreader, id) + back, err := udc.backIndexRowForDoc(kvreader, []byte(id)) if err != nil { rv <- err return diff --git a/index/upside_down/index_reader.go b/index/upside_down/index_reader.go index e8047144..ba09ed9d 100644 --- a/index/upside_down/index_reader.go +++ b/index/upside_down/index_reader.go @@ -10,6 +10,7 @@ package upside_down import ( + "bytes" "fmt" "github.com/blevesearch/bleve/document" @@ -17,6 +18,20 @@ import ( "github.com/blevesearch/bleve/index/store" ) +type InternalId []byte + +func (u InternalId) Compare(other index.IndexInternalID) int { + if other == nil { + // this internal ID is always greater than nil + return 1 + } + return bytes.Compare(u, other.(InternalId)) +} + +func (u InternalId) Equals(other index.IndexInternalID) bool { + return u.Compare(other.(InternalId)) == 0 +} + type IndexReader struct { index *UpsideDownCouch kvreader store.KVReader @@ -51,10 +66,14 @@ func (i *IndexReader) DocIDReader(start, end string) (index.DocIDReader, error) return newUpsideDownCouchDocIDReader(i, start, end) } +func (i *IndexReader) DocIDReaderOnly(ids []string) (index.DocIDReader, error) { + return newUpsideDownCouchDocIDReaderOnly(i, ids) +} + func (i *IndexReader) Document(id string) (doc *document.Document, err error) { // first hit the back index to confirm doc exists var backIndexRow *BackIndexRow - backIndexRow, err = i.index.backIndexRowForDoc(i.kvreader, id) + backIndexRow, err = i.index.backIndexRowForDoc(i.kvreader, []byte(id)) if err != nil { return } @@ -94,8 +113,8 @@ func (i *IndexReader) Document(id string) (doc *document.Document, err error) { return } -func (i *IndexReader) DocumentFieldTerms(id string) (index.FieldTerms, error) { - back, err := i.index.backIndexRowForDoc(i.kvreader, id) +func (i *IndexReader) DocumentFieldTerms(id index.IndexInternalID) (index.FieldTerms, error) { + back, err := i.index.backIndexRowForDoc(i.kvreader, id.(InternalId)) if err != nil { return nil, err } @@ -112,8 +131,8 @@ func (i *IndexReader) DocumentFieldTerms(id string) (index.FieldTerms, error) { return rv, nil } -func (i *IndexReader) DocumentFieldTermsForFields(id string, fields []string) (index.FieldTerms, error) { - back, err := i.index.backIndexRowForDoc(i.kvreader, id) +func (i *IndexReader) DocumentFieldTermsForFields(id index.IndexInternalID, fields []string) (index.FieldTerms, error) { + back, err := i.index.backIndexRowForDoc(i.kvreader, id.(InternalId)) if err != nil { return nil, err } @@ -181,6 +200,10 @@ func (i *IndexReader) Close() error { return i.kvreader.Close() } +func (i *IndexReader) FinalizeDocID(id index.IndexInternalID) (string, error) { + return string(id.(InternalId)), nil +} + func incrementBytes(in []byte) []byte { rv := make([]byte, len(in)) copy(rv, in) diff --git a/index/upside_down/reader.go b/index/upside_down/reader.go index a973329e..1764dbbc 100644 --- a/index/upside_down/reader.go +++ b/index/upside_down/reader.go @@ -10,6 +10,8 @@ package upside_down import ( + "bytes" + "sort" "sync/atomic" "github.com/blevesearch/bleve/index" @@ -81,7 +83,7 @@ func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (* if rv == nil { rv = &index.TermFieldDoc{} } - rv.ID = string(tfr.doc) + rv.ID = InternalId(tfr.doc) rv.Freq = tfr.freq rv.Norm = float64(tfr.norm) if tfr.vectors != nil { @@ -94,9 +96,10 @@ func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (* return nil, nil } -func (r *UpsideDownCouchTermFieldReader) Advance(docID string, preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) { +func (r *UpsideDownCouchTermFieldReader) Advance(docIDInternal index.IndexInternalID, preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) { + docID := docIDInternal.(InternalId) if r.iterator != nil { - tfr := NewTermFrequencyRow(r.term, r.field, []byte(docID), 0, 0) + tfr := NewTermFrequencyRow(r.term, r.field, docID, 0, 0) r.iterator.Seek(tfr.Key()) key, val, valid := r.iterator.Current() if valid { @@ -108,7 +111,7 @@ func (r *UpsideDownCouchTermFieldReader) Advance(docID string, preAlloced *index if rv == nil { rv = &index.TermFieldDoc{} } - rv.ID = string(tfr.doc) + rv.ID = InternalId(tfr.doc) rv.Freq = tfr.freq rv.Norm = float64(tfr.norm) if tfr.vectors != nil { @@ -131,6 +134,9 @@ func (r *UpsideDownCouchTermFieldReader) Close() error { type UpsideDownCouchDocIDReader struct { indexReader *IndexReader iterator store.KVIterator + only []string + onlyPos int + onlyMode bool } func newUpsideDownCouchDocIDReader(indexReader *IndexReader, start, end string) (*UpsideDownCouchDocIDReader, error) { @@ -152,37 +158,138 @@ func newUpsideDownCouchDocIDReader(indexReader *IndexReader, start, end string) }, nil } -func (r *UpsideDownCouchDocIDReader) Next() (string, error) { - key, val, valid := r.iterator.Current() - if valid { - br, err := NewBackIndexRowKV(key, val) - if err != nil { - return "", err - } - rv := string(br.doc) - r.iterator.Next() - return rv, nil +func newUpsideDownCouchDocIDReaderOnly(indexReader *IndexReader, ids []string) (*UpsideDownCouchDocIDReader, error) { + // ensure ids are sorted + sort.Strings(ids) + startBytes := []byte{0x0} + if len(ids) > 0 { + startBytes = []byte(ids[0]) } - return "", nil + endBytes := []byte{0xff} + if len(ids) > 0 { + endBytes = incrementBytes([]byte(ids[len(ids)-1])) + } + bisr := NewBackIndexRow(startBytes, nil, nil) + bier := NewBackIndexRow(endBytes, nil, nil) + it := indexReader.kvreader.RangeIterator(bisr.Key(), bier.Key()) + + return &UpsideDownCouchDocIDReader{ + indexReader: indexReader, + iterator: it, + only: ids, + onlyMode: true, + }, nil } -func (r *UpsideDownCouchDocIDReader) Advance(docID string) (string, error) { - bir := NewBackIndexRow([]byte(docID), nil, nil) +func (r *UpsideDownCouchDocIDReader) Next() (index.IndexInternalID, error) { + key, val, valid := r.iterator.Current() + + if r.onlyMode { + var rv InternalId + for valid && r.onlyPos < len(r.only) { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + if !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) { + ok := r.nextOnly() + if !ok { + return nil, nil + } + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + key, val, valid = r.iterator.Current() + continue + } else { + rv = InternalId(br.doc) + break + } + } + if valid && r.onlyPos < len(r.only) { + ok := r.nextOnly() + if ok { + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + } + return rv, nil + } + + } else { + if valid { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + rv := InternalId(br.doc) + r.iterator.Next() + return rv, nil + } + } + return nil, nil +} + +func (r *UpsideDownCouchDocIDReader) Advance(docID index.IndexInternalID) (index.IndexInternalID, error) { + docIDInternal := docID.(InternalId) + bir := NewBackIndexRow(docIDInternal, nil, nil) r.iterator.Seek(bir.Key()) key, val, valid := r.iterator.Current() - if valid { - br, err := NewBackIndexRowKV(key, val) - if err != nil { - return "", err + r.onlyPos = sort.SearchStrings(r.only, string(docIDInternal)) + + if r.onlyMode { + var rv InternalId + for valid && r.onlyPos < len(r.only) { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + if !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) { + ok := r.nextOnly() + if !ok { + return nil, nil + } + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + continue + } else { + rv = InternalId(br.doc) + break + } + } + if valid && r.onlyPos < len(r.only) { + ok := r.nextOnly() + if ok { + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + } + return rv, nil + } + } else { + if valid { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + rv := InternalId(br.doc) + r.iterator.Next() + return rv, nil } - rv := string(br.doc) - r.iterator.Next() - return rv, nil } - return "", nil + return nil, nil } func (r *UpsideDownCouchDocIDReader) Close() error { atomic.AddUint64(&r.indexReader.index.stats.termSearchersFinished, uint64(1)) return r.iterator.Close() } + +// move the r.only pos forward one, skipping duplicates +// return true if there is more data, or false if we got to the end of the list +func (r *UpsideDownCouchDocIDReader) nextOnly() bool { + + // advance 1 position, until we see a different key + // it's already sorted, so this skips duplicates + start := r.onlyPos + r.onlyPos++ + for r.onlyPos < len(r.only) && r.only[r.onlyPos] == r.only[start] { + start = r.onlyPos + r.onlyPos++ + } + // inidicate if we got to the end of the list + return r.onlyPos < len(r.only) +} diff --git a/index/upside_down/reader_test.go b/index/upside_down/reader_test.go index 62e81398..0a47e853 100644 --- a/index/upside_down/reader_test.go +++ b/index/upside_down/reader_test.go @@ -111,7 +111,7 @@ func TestIndexReader(t *testing.T) { } expectedMatch := &index.TermFieldDoc{ - ID: "2", + ID: InternalId("2"), Freq: 1, Norm: 0.5773502588272095, Vectors: []*index.TermFieldVector{ @@ -145,17 +145,17 @@ func TestIndexReader(t *testing.T) { t.Errorf("Error accessing term field reader: %v", err) } - match, err = reader.Advance("2", nil) + match, err = reader.Advance(InternalId("2"), nil) if err != nil { t.Errorf("unexpected error: %v", err) } if match == nil { t.Fatalf("Expected match, got nil") } - if match.ID != "2" { + if !match.ID.Equals(InternalId("2")) { t.Errorf("Expected ID '2', got '%s'", match.ID) } - match, err = reader.Advance("3", nil) + match, err = reader.Advance(InternalId("3"), nil) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -183,7 +183,7 @@ func TestIndexReader(t *testing.T) { if match != nil { t.Errorf("expected nil, got %v", match) } - match, err = reader.Advance("anywhere", nil) + match, err = reader.Advance(InternalId("anywhere"), nil) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -260,7 +260,7 @@ func TestIndexDocIdReader(t *testing.T) { id, err := reader.Next() count := uint64(0) - for id != "" { + for id != nil { count++ id, err = reader.Next() } @@ -280,19 +280,19 @@ func TestIndexDocIdReader(t *testing.T) { } }() - id, err = reader2.Advance("2") + id, err = reader2.Advance(InternalId("2")) if err != nil { t.Error(err) } - if id != "2" { + if !id.Equals(InternalId("2")) { t.Errorf("expected to find id '2', got '%s'", id) } - id, err = reader2.Advance("3") + id, err = reader2.Advance(InternalId("3")) if err != nil { t.Error(err) } - if id != "" { + if id != nil { t.Errorf("expected to find id '', got '%s'", id) } } diff --git a/index/upside_down/upside_down.go b/index/upside_down/upside_down.go index 7fec5724..2bde3e4f 100644 --- a/index/upside_down/upside_down.go +++ b/index/upside_down/upside_down.go @@ -439,7 +439,7 @@ func (udc *UpsideDownCouch) Update(doc *document.Document) (err error) { // first we lookup the backindex row for the doc id if it exists // lookup the back index row var backIndexRow *BackIndexRow - backIndexRow, err = udc.backIndexRowForDoc(kvreader, doc.ID) + backIndexRow, err = udc.backIndexRowForDoc(kvreader, InternalId(doc.ID)) if err != nil { _ = kvreader.Close() atomic.AddUint64(&udc.stats.errors, 1) @@ -627,7 +627,7 @@ func (udc *UpsideDownCouch) Delete(id string) (err error) { // first we lookup the backindex row for the doc id if it exists // lookup the back index row var backIndexRow *BackIndexRow - backIndexRow, err = udc.backIndexRowForDoc(kvreader, id) + backIndexRow, err = udc.backIndexRowForDoc(kvreader, InternalId(id)) if err != nil { _ = kvreader.Close() atomic.AddUint64(&udc.stats.errors, 1) @@ -695,10 +695,10 @@ func (udc *UpsideDownCouch) deleteSingle(id string, backIndexRow *BackIndexRow, return deleteRows } -func (udc *UpsideDownCouch) backIndexRowForDoc(kvreader store.KVReader, docID string) (*BackIndexRow, error) { +func (udc *UpsideDownCouch) backIndexRowForDoc(kvreader store.KVReader, docID InternalId) (*BackIndexRow, error) { // use a temporary row structure to build key tempRow := &BackIndexRow{ - doc: []byte(docID), + doc: docID, } keyBuf := GetRowBuffer() @@ -833,7 +833,7 @@ func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) { } for docID, doc := range batch.IndexOps { - backIndexRow, err := udc.backIndexRowForDoc(kvreader, docID) + backIndexRow, err := udc.backIndexRowForDoc(kvreader, InternalId(docID)) if err != nil { docBackIndexRowErr = err return diff --git a/index/upside_down/upside_down_test.go b/index/upside_down/upside_down_test.go index 3a1fc004..45b7a6f5 100644 --- a/index/upside_down/upside_down_test.go +++ b/index/upside_down/upside_down_test.go @@ -663,16 +663,16 @@ func TestIndexBatch(t *testing.T) { if err != nil { t.Error(err) } - docIds := make([]string, 0) + docIds := make([]InternalId, 0) docID, err := docIDReader.Next() - for docID != "" && err == nil { - docIds = append(docIds, docID) + for docID != nil && err == nil { + docIds = append(docIds, docID.(InternalId)) docID, err = docIDReader.Next() } if err != nil { t.Error(err) } - expectedDocIds := []string{"2", "3"} + expectedDocIds := []InternalId{InternalId("2"), InternalId("3")} if !reflect.DeepEqual(docIds, expectedDocIds) { t.Errorf("expected ids: %v, got ids: %v", expectedDocIds, docIds) } @@ -1126,7 +1126,7 @@ func TestIndexTermReaderCompositeFields(t *testing.T) { tfd, err := termFieldReader.Next(nil) for tfd != nil && err == nil { - if tfd.ID != "1" { + if !tfd.ID.Equals(InternalId("1")) { t.Errorf("expected to find document id 1") } tfd, err = termFieldReader.Next(nil) @@ -1179,7 +1179,7 @@ func TestIndexDocumentFieldTerms(t *testing.T) { } }() - fieldTerms, err := indexReader.DocumentFieldTerms("1") + fieldTerms, err := indexReader.DocumentFieldTerms(InternalId("1")) if err != nil { t.Error(err) } diff --git a/index_impl.go b/index_impl.go index f914ec94..66c33f8c 100644 --- a/index_impl.go +++ b/index_impl.go @@ -435,7 +435,7 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr collector.SetFacetsBuilder(facetsBuilder) } - err = collector.Collect(ctx, searcher) + err = collector.Collect(ctx, searcher, indexReader) if err != nil { return nil, err } diff --git a/search/collector.go b/search/collector.go index 773c8d55..a6d9148c 100644 --- a/search/collector.go +++ b/search/collector.go @@ -12,11 +12,13 @@ package search import ( "time" + "github.com/blevesearch/bleve/index" + "golang.org/x/net/context" ) type Collector interface { - Collect(ctx context.Context, searcher Searcher) error + Collect(ctx context.Context, searcher Searcher, reader index.IndexReader) error Results() DocumentMatchCollection Total() uint64 MaxScore() float64 diff --git a/search/collectors/bench_test.go b/search/collectors/bench_test.go index d8daeb9e..2ae7e02c 100644 --- a/search/collectors/bench_test.go +++ b/search/collectors/bench_test.go @@ -10,10 +10,10 @@ import ( ) func benchHelper(numOfMatches int, collector search.Collector, b *testing.B) { - matches := make(search.DocumentMatchCollection, 0, numOfMatches) + matches := make([]*search.DocumentMatchInternal, 0, numOfMatches) for i := 0; i < numOfMatches; i++ { - matches = append(matches, &search.DocumentMatch{ - ID: strconv.Itoa(i), + matches = append(matches, &search.DocumentMatchInternal{ + ID: testInternalId(strconv.Itoa(i)), Score: rand.Float64(), }) } @@ -24,7 +24,7 @@ func benchHelper(numOfMatches int, collector search.Collector, b *testing.B) { searcher := &stubSearcher{ matches: matches, } - err := collector.Collect(context.Background(), searcher) + err := collector.Collect(context.Background(), searcher, &stubReader{}) if err != nil { b.Fatal(err) } diff --git a/search/collectors/collector_top_score.go b/search/collectors/collector_top_score.go index 90b0c75e..b6e44f03 100644 --- a/search/collectors/collector_top_score.go +++ b/search/collectors/collector_top_score.go @@ -15,6 +15,7 @@ import ( "golang.org/x/net/context" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -27,6 +28,7 @@ type TopScoreCollector struct { minScore float64 total uint64 facetsBuilder *search.FacetsBuilder + actualResults search.DocumentMatchCollection } func NewTopScorerCollector(k int) *TopScoreCollector { @@ -59,11 +61,11 @@ func (tksc *TopScoreCollector) Took() time.Duration { var COLLECT_CHECK_DONE_EVERY = uint64(1024) -func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Searcher) error { +func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Searcher, reader index.IndexReader) error { startTime := time.Now() var err error - var pre search.DocumentMatch // A single pre-alloc'ed, reused instance. - var next *search.DocumentMatch + var pre search.DocumentMatchInternal // A single pre-alloc'ed, reused instance. + var next *search.DocumentMatchInternal select { case <-ctx.Done(): return ctx.Err() @@ -87,6 +89,12 @@ func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Sear } next, err = searcher.Next(pre.Reset()) } + // finalize actual results + tksc.actualResults, err = tksc.finalizeResults(reader) + if err != nil { + return err + } + // compute search duration tksc.took = time.Since(startTime) if err != nil { @@ -95,7 +103,7 @@ func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Sear return nil } -func (tksc *TopScoreCollector) collectSingle(dmIn *search.DocumentMatch) { +func (tksc *TopScoreCollector) collectSingle(dmIn *search.DocumentMatchInternal) { // increment total hits tksc.total++ @@ -111,18 +119,18 @@ func (tksc *TopScoreCollector) collectSingle(dmIn *search.DocumentMatch) { // Because the dmIn will be the single, pre-allocated, reused // instance, we need to copy the dmIn into a new, standalone // instance before inserting into our candidate results list. - dm := &search.DocumentMatch{} + dm := &search.DocumentMatchInternal{} *dm = *dmIn for e := tksc.results.Front(); e != nil; e = e.Next() { - curr := e.Value.(*search.DocumentMatch) + curr := e.Value.(*search.DocumentMatchInternal) if dm.Score <= curr.Score { tksc.results.InsertBefore(dm, e) // if we just made the list too long if tksc.results.Len() > (tksc.k + tksc.skip) { // remove the head - tksc.minScore = tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch).Score + tksc.minScore = tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatchInternal).Score } return } @@ -131,11 +139,15 @@ func (tksc *TopScoreCollector) collectSingle(dmIn *search.DocumentMatch) { tksc.results.PushBack(dm) if tksc.results.Len() > (tksc.k + tksc.skip) { // remove the head - tksc.minScore = tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch).Score + tksc.minScore = tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatchInternal).Score } } func (tksc *TopScoreCollector) Results() search.DocumentMatchCollection { + return tksc.actualResults +} + +func (tksc *TopScoreCollector) finalizeResults(r index.IndexReader) (search.DocumentMatchCollection, error) { if tksc.results.Len()-tksc.skip > 0 { rv := make(search.DocumentMatchCollection, tksc.results.Len()-tksc.skip) i := 0 @@ -145,12 +157,16 @@ func (tksc *TopScoreCollector) Results() search.DocumentMatchCollection { skipped++ continue } - rv[i] = e.Value.(*search.DocumentMatch) + var err error + rv[i], err = e.Value.(*search.DocumentMatchInternal).Finalize(r) + if err != nil { + return nil, err + } i++ } - return rv + return rv, nil } - return search.DocumentMatchCollection{} + return search.DocumentMatchCollection{}, nil } func (tksc *TopScoreCollector) SetFacetsBuilder(facetsBuilder *search.FacetsBuilder) { diff --git a/search/collectors/collector_top_score_test.go b/search/collectors/collector_top_score_test.go index 4bf76140..4498dbc8 100644 --- a/search/collectors/collector_top_score_test.go +++ b/search/collectors/collector_top_score_test.go @@ -23,68 +23,68 @@ func TestTop10Scores(t *testing.T) { // the top-10 scores are > 10 // everything else is less than 10 searcher := &stubSearcher{ - matches: search.DocumentMatchCollection{ - &search.DocumentMatch{ - ID: "a", + matches: []*search.DocumentMatchInternal{ + &search.DocumentMatchInternal{ + ID: testInternalId("a"), Score: 11, }, - &search.DocumentMatch{ - ID: "b", + &search.DocumentMatchInternal{ + ID: testInternalId("b"), Score: 9, }, - &search.DocumentMatch{ - ID: "c", + &search.DocumentMatchInternal{ + ID: testInternalId("c"), Score: 11, }, - &search.DocumentMatch{ - ID: "d", + &search.DocumentMatchInternal{ + ID: testInternalId("d"), Score: 9, }, - &search.DocumentMatch{ - ID: "e", + &search.DocumentMatchInternal{ + ID: testInternalId("e"), Score: 11, }, - &search.DocumentMatch{ - ID: "f", + &search.DocumentMatchInternal{ + ID: testInternalId("f"), Score: 9, }, - &search.DocumentMatch{ - ID: "g", + &search.DocumentMatchInternal{ + ID: testInternalId("g"), Score: 11, }, - &search.DocumentMatch{ - ID: "h", + &search.DocumentMatchInternal{ + ID: testInternalId("h"), Score: 9, }, - &search.DocumentMatch{ - ID: "i", + &search.DocumentMatchInternal{ + ID: testInternalId("i"), Score: 11, }, - &search.DocumentMatch{ - ID: "j", + &search.DocumentMatchInternal{ + ID: testInternalId("j"), Score: 11, }, - &search.DocumentMatch{ - ID: "k", + &search.DocumentMatchInternal{ + ID: testInternalId("k"), Score: 11, }, - &search.DocumentMatch{ - ID: "l", + &search.DocumentMatchInternal{ + ID: testInternalId("l"), Score: 99, }, - &search.DocumentMatch{ - ID: "m", + &search.DocumentMatchInternal{ + ID: testInternalId("m"), Score: 11, }, - &search.DocumentMatch{ - ID: "n", + &search.DocumentMatchInternal{ + ID: testInternalId("n"), Score: 11, }, }, } collector := NewTopScorerCollector(10) - err := collector.Collect(context.Background(), searcher) + err := collector.Collect(context.Background(), searcher, &stubReader{}) if err != nil { t.Fatal(err) } @@ -131,68 +131,68 @@ func TestTop10ScoresSkip10(t *testing.T) { // the top-10 scores are > 10 // everything else is less than 10 searcher := &stubSearcher{ - matches: search.DocumentMatchCollection{ - &search.DocumentMatch{ - ID: "a", + matches: []*search.DocumentMatchInternal{ + &search.DocumentMatchInternal{ + ID: testInternalId("a"), Score: 11, }, - &search.DocumentMatch{ - ID: "b", + &search.DocumentMatchInternal{ + ID: testInternalId("b"), Score: 9.5, }, - &search.DocumentMatch{ - ID: "c", + &search.DocumentMatchInternal{ + ID: testInternalId("c"), Score: 11, }, - &search.DocumentMatch{ - ID: "d", + &search.DocumentMatchInternal{ + ID: testInternalId("d"), Score: 9, }, - &search.DocumentMatch{ - ID: "e", + &search.DocumentMatchInternal{ + ID: testInternalId("e"), Score: 11, }, - &search.DocumentMatch{ - ID: "f", + &search.DocumentMatchInternal{ + ID: testInternalId("f"), Score: 9, }, - &search.DocumentMatch{ - ID: "g", + &search.DocumentMatchInternal{ + ID: testInternalId("g"), Score: 11, }, - &search.DocumentMatch{ - ID: "h", + &search.DocumentMatchInternal{ + ID: testInternalId("h"), Score: 9, }, - &search.DocumentMatch{ - ID: "i", + &search.DocumentMatchInternal{ + ID: testInternalId("i"), Score: 11, }, - &search.DocumentMatch{ - ID: "j", + &search.DocumentMatchInternal{ + ID: testInternalId("j"), Score: 11, }, - &search.DocumentMatch{ - ID: "k", + &search.DocumentMatchInternal{ + ID: testInternalId("k"), Score: 11, }, - &search.DocumentMatch{ - ID: "l", + &search.DocumentMatchInternal{ + ID: testInternalId("l"), Score: 99, }, - &search.DocumentMatch{ - ID: "m", + &search.DocumentMatchInternal{ + ID: testInternalId("m"), Score: 11, }, - &search.DocumentMatch{ - ID: "n", + &search.DocumentMatchInternal{ + ID: testInternalId("n"), Score: 11, }, }, } collector := NewTopScorerSkipCollector(10, 10) - err := collector.Collect(context.Background(), searcher) + err := collector.Collect(context.Background(), searcher, &stubReader{}) if err != nil { t.Fatal(err) } @@ -227,61 +227,61 @@ func TestPaginationSameScores(t *testing.T) { // a stub search with more than 10 matches // all documents have the same score searcher := &stubSearcher{ - matches: search.DocumentMatchCollection{ - &search.DocumentMatch{ - ID: "a", + matches: []*search.DocumentMatchInternal{ + &search.DocumentMatchInternal{ + ID: testInternalId("a"), Score: 5, }, - &search.DocumentMatch{ - ID: "b", + &search.DocumentMatchInternal{ + ID: testInternalId("b"), Score: 5, }, - &search.DocumentMatch{ - ID: "c", + &search.DocumentMatchInternal{ + ID: testInternalId("c"), Score: 5, }, - &search.DocumentMatch{ - ID: "d", + &search.DocumentMatchInternal{ + ID: testInternalId("d"), Score: 5, }, - &search.DocumentMatch{ - ID: "e", + &search.DocumentMatchInternal{ + ID: testInternalId("e"), Score: 5, }, - &search.DocumentMatch{ - ID: "f", + &search.DocumentMatchInternal{ + ID: testInternalId("f"), Score: 5, }, - &search.DocumentMatch{ - ID: "g", + &search.DocumentMatchInternal{ + ID: testInternalId("g"), Score: 5, }, - &search.DocumentMatch{ - ID: "h", + &search.DocumentMatchInternal{ + ID: testInternalId("h"), Score: 5, }, - &search.DocumentMatch{ - ID: "i", + &search.DocumentMatchInternal{ + ID: testInternalId("i"), Score: 5, }, - &search.DocumentMatch{ - ID: "j", + &search.DocumentMatchInternal{ + ID: testInternalId("j"), Score: 5, }, - &search.DocumentMatch{ - ID: "k", + &search.DocumentMatchInternal{ + ID: testInternalId("k"), Score: 5, }, - &search.DocumentMatch{ - ID: "l", + &search.DocumentMatchInternal{ + ID: testInternalId("l"), Score: 5, }, - &search.DocumentMatch{ - ID: "m", + &search.DocumentMatchInternal{ + ID: testInternalId("m"), Score: 5, }, - &search.DocumentMatch{ - ID: "n", + &search.DocumentMatchInternal{ + ID: testInternalId("n"), Score: 5, }, }, @@ -289,7 +289,7 @@ func TestPaginationSameScores(t *testing.T) { // first get first 5 hits collector := NewTopScorerSkipCollector(5, 0) - err := collector.Collect(context.Background(), searcher) + err := collector.Collect(context.Background(), searcher, &stubReader{}) if err != nil { t.Fatal(err) } @@ -313,61 +313,61 @@ func TestPaginationSameScores(t *testing.T) { // a stub search with more than 10 matches // all documents have the same score searcher = &stubSearcher{ - matches: search.DocumentMatchCollection{ - &search.DocumentMatch{ - ID: "a", + matches: []*search.DocumentMatchInternal{ + &search.DocumentMatchInternal{ + ID: testInternalId("a"), Score: 5, }, - &search.DocumentMatch{ - ID: "b", + &search.DocumentMatchInternal{ + ID: testInternalId("b"), Score: 5, }, - &search.DocumentMatch{ - ID: "c", + &search.DocumentMatchInternal{ + ID: testInternalId("c"), Score: 5, }, - &search.DocumentMatch{ - ID: "d", + &search.DocumentMatchInternal{ + ID: testInternalId("d"), Score: 5, }, - &search.DocumentMatch{ - ID: "e", + &search.DocumentMatchInternal{ + ID: testInternalId("e"), Score: 5, }, - &search.DocumentMatch{ - ID: "f", + &search.DocumentMatchInternal{ + ID: testInternalId("f"), Score: 5, }, - &search.DocumentMatch{ - ID: "g", + &search.DocumentMatchInternal{ + ID: testInternalId("g"), Score: 5, }, - &search.DocumentMatch{ - ID: "h", + &search.DocumentMatchInternal{ + ID: testInternalId("h"), Score: 5, }, - &search.DocumentMatch{ - ID: "i", + &search.DocumentMatchInternal{ + ID: testInternalId("i"), Score: 5, }, - &search.DocumentMatch{ - ID: "j", + &search.DocumentMatchInternal{ + ID: testInternalId("j"), Score: 5, }, - &search.DocumentMatch{ - ID: "k", + &search.DocumentMatchInternal{ + ID: testInternalId("k"), Score: 5, }, - &search.DocumentMatch{ - ID: "l", + &search.DocumentMatchInternal{ + ID: testInternalId("l"), Score: 5, }, - &search.DocumentMatch{ - ID: "m", + &search.DocumentMatchInternal{ + ID: testInternalId("m"), Score: 5, }, - &search.DocumentMatch{ - ID: "n", + &search.DocumentMatchInternal{ + ID: testInternalId("n"), Score: 5, }, }, @@ -375,7 +375,7 @@ func TestPaginationSameScores(t *testing.T) { // now get next 5 hits collector = NewTopScorerSkipCollector(5, 5) - err = collector.Collect(context.Background(), searcher) + err = collector.Collect(context.Background(), searcher, &stubReader{}) if err != nil { t.Fatal(err) } diff --git a/search/collectors/search_test.go b/search/collectors/search_test.go index 629a1eca..acb58957 100644 --- a/search/collectors/search_test.go +++ b/search/collectors/search_test.go @@ -10,15 +10,19 @@ package collectors import ( + "bytes" + + "github.com/blevesearch/bleve/document" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) type stubSearcher struct { index int - matches search.DocumentMatchCollection + matches []*search.DocumentMatchInternal } -func (ss *stubSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (ss *stubSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { if ss.index < len(ss.matches) { rv := ss.matches[ss.index] ss.index++ @@ -27,9 +31,9 @@ func (ss *stubSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docume return nil, nil } -func (ss *stubSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (ss *stubSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { - for ss.index < len(ss.matches) && ss.matches[ss.index].ID < ID { + for ss.index < len(ss.matches) && ss.matches[ss.index].ID.Compare(ID) < 0 { ss.index++ } if ss.index < len(ss.matches) { @@ -58,3 +62,71 @@ func (ss *stubSearcher) Count() uint64 { func (ss *stubSearcher) Min() int { return 0 } + +type testInternalId []byte + +func (t testInternalId) Compare(other index.IndexInternalID) int { + return bytes.Compare(t, other.(testInternalId)) +} + +func (t testInternalId) Equals(other index.IndexInternalID) bool { + return t.Compare(other.(testInternalId)) == 0 +} + +type stubReader struct{} + +func (sr *stubReader) TermFieldReader(term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (index.TermFieldReader, error) { + return nil, nil +} + +func (sr *stubReader) DocIDReader(start, end string) (index.DocIDReader, error) { + return nil, nil +} + +func (sr *stubReader) DocIDReaderOnly(ids []string) (index.DocIDReader, error) { + return nil, nil +} + +func (sr *stubReader) FieldDict(field string) (index.FieldDict, error) { + return nil, nil +} + +func (sr *stubReader) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) { + return nil, nil +} + +func (sr *stubReader) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) { + return nil, nil +} + +func (sr *stubReader) Document(id string) (*document.Document, error) { + return nil, nil +} + +func (sr *stubReader) DocumentFieldTerms(id index.IndexInternalID) (index.FieldTerms, error) { + return nil, nil +} + +func (sr *stubReader) DocumentFieldTermsForFields(id index.IndexInternalID, fields []string) (index.FieldTerms, error) { + return nil, nil +} + +func (sr *stubReader) Fields() ([]string, error) { + return nil, nil +} + +func (sr *stubReader) GetInternal(key []byte) ([]byte, error) { + return nil, nil +} + +func (sr *stubReader) DocCount() uint64 { + return 0 +} + +func (sr *stubReader) FinalizeDocID(id index.IndexInternalID) (string, error) { + return string(id.(testInternalId)), nil +} + +func (sr *stubReader) Close() error { + return nil +} diff --git a/search/facets_builder.go b/search/facets_builder.go index 4d52ec2b..c0c203cc 100644 --- a/search/facets_builder.go +++ b/search/facets_builder.go @@ -37,7 +37,7 @@ func (fb *FacetsBuilder) Add(name string, facetBuilder FacetBuilder) { fb.facets[name] = facetBuilder } -func (fb *FacetsBuilder) Update(docMatch *DocumentMatch) error { +func (fb *FacetsBuilder) Update(docMatch *DocumentMatchInternal) error { var fields []string for _, facetBuilder := range fb.facets { fields = append(fields, facetBuilder.Field()) diff --git a/search/scorers/scorer_conjunction.go b/search/scorers/scorer_conjunction.go index 422f282c..94a5eb62 100644 --- a/search/scorers/scorer_conjunction.go +++ b/search/scorers/scorer_conjunction.go @@ -23,8 +23,8 @@ func NewConjunctionQueryScorer(explain bool) *ConjunctionQueryScorer { } } -func (s *ConjunctionQueryScorer) Score(constituents []*search.DocumentMatch) *search.DocumentMatch { - rv := search.DocumentMatch{ +func (s *ConjunctionQueryScorer) Score(constituents []*search.DocumentMatchInternal) *search.DocumentMatchInternal { + rv := search.DocumentMatchInternal{ ID: constituents[0].ID, } diff --git a/search/scorers/scorer_constant.go b/search/scorers/scorer_constant.go index 1434bd5e..86717e9f 100644 --- a/search/scorers/scorer_constant.go +++ b/search/scorers/scorer_constant.go @@ -12,6 +12,7 @@ package scorers import ( "fmt" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -64,7 +65,7 @@ func (s *ConstantScorer) SetQueryNorm(qnorm float64) { } } -func (s *ConstantScorer) Score(id string) *search.DocumentMatch { +func (s *ConstantScorer) Score(id index.IndexInternalID) *search.DocumentMatchInternal { var scoreExplanation *search.Explanation score := s.constant @@ -91,7 +92,7 @@ func (s *ConstantScorer) Score(id string) *search.DocumentMatch { } } - rv := search.DocumentMatch{ + rv := search.DocumentMatchInternal{ ID: id, Score: score, } diff --git a/search/scorers/scorer_constant_test.go b/search/scorers/scorer_constant_test.go index 4c8740e0..8f50c5d3 100644 --- a/search/scorers/scorer_constant_test.go +++ b/search/scorers/scorer_constant_test.go @@ -23,12 +23,12 @@ func TestConstantScorer(t *testing.T) { tests := []struct { termMatch *index.TermFieldDoc - result *search.DocumentMatch + result *search.DocumentMatchInternal }{ // test some simple math { termMatch: &index.TermFieldDoc{ - ID: "one", + ID: testInternalId("one"), Freq: 1, Norm: 1.0, Vectors: []*index.TermFieldVector{ @@ -40,8 +40,8 @@ func TestConstantScorer(t *testing.T) { }, }, }, - result: &search.DocumentMatch{ - ID: "one", + result: &search.DocumentMatchInternal{ + ID: testInternalId("one"), Score: 1.0, Expl: &search.Explanation{ Value: 1.0, @@ -68,16 +68,16 @@ func TestConstantScorerWithQueryNorm(t *testing.T) { tests := []struct { termMatch *index.TermFieldDoc - result *search.DocumentMatch + result *search.DocumentMatchInternal }{ { termMatch: &index.TermFieldDoc{ - ID: "one", + ID: testInternalId("one"), Freq: 1, Norm: 1.0, }, - result: &search.DocumentMatch{ - ID: "one", + result: &search.DocumentMatchInternal{ + ID: testInternalId("one"), Score: 2.0, Expl: &search.Explanation{ Value: 2.0, diff --git a/search/scorers/scorer_disjunction.go b/search/scorers/scorer_disjunction.go index 00bc8cd0..0e5dc3e5 100644 --- a/search/scorers/scorer_disjunction.go +++ b/search/scorers/scorer_disjunction.go @@ -25,8 +25,8 @@ func NewDisjunctionQueryScorer(explain bool) *DisjunctionQueryScorer { } } -func (s *DisjunctionQueryScorer) Score(constituents []*search.DocumentMatch, countMatch, countTotal int) *search.DocumentMatch { - rv := search.DocumentMatch{ +func (s *DisjunctionQueryScorer) Score(constituents []*search.DocumentMatchInternal, countMatch, countTotal int) *search.DocumentMatchInternal { + rv := search.DocumentMatchInternal{ ID: constituents[0].ID, } diff --git a/search/scorers/scorer_term.go b/search/scorers/scorer_term.go index ce926221..ee6b6a9b 100644 --- a/search/scorers/scorer_term.go +++ b/search/scorers/scorer_term.go @@ -83,7 +83,7 @@ func (s *TermQueryScorer) SetQueryNorm(qnorm float64) { } } -func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc, preAllocated *search.DocumentMatch) *search.DocumentMatch { +func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc, preAllocated *search.DocumentMatchInternal) *search.DocumentMatchInternal { var scoreExplanation *search.Explanation // need to compute score @@ -130,7 +130,7 @@ func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc, preAllocated *sea rv := preAllocated if rv == nil { - rv = &search.DocumentMatch{} + rv = &search.DocumentMatchInternal{} } rv.ID = termMatch.ID rv.Score = score diff --git a/search/scorers/scorer_term_test.go b/search/scorers/scorer_term_test.go index 612b02d9..43950fc6 100644 --- a/search/scorers/scorer_term_test.go +++ b/search/scorers/scorer_term_test.go @@ -10,6 +10,7 @@ package scorers import ( + "bytes" "math" "reflect" "testing" @@ -18,6 +19,16 @@ import ( "github.com/blevesearch/bleve/search" ) +type testInternalId []byte + +func (t testInternalId) Compare(other index.IndexInternalID) int { + return bytes.Compare(t, other.(testInternalId)) +} + +func (t testInternalId) Equals(other index.IndexInternalID) bool { + return t.Compare(other.(testInternalId)) == 0 +} + func TestTermScorer(t *testing.T) { var docTotal uint64 = 100 @@ -30,12 +41,12 @@ func TestTermScorer(t *testing.T) { tests := []struct { termMatch *index.TermFieldDoc - result *search.DocumentMatch + result *search.DocumentMatchInternal }{ // test some simple math { termMatch: &index.TermFieldDoc{ - ID: "one", + ID: testInternalId("one"), Freq: 1, Norm: 1.0, Vectors: []*index.TermFieldVector{ @@ -47,8 +58,8 @@ func TestTermScorer(t *testing.T) { }, }, }, - result: &search.DocumentMatch{ - ID: "one", + result: &search.DocumentMatchInternal{ + ID: testInternalId("one"), Score: math.Sqrt(1.0) * idf, Expl: &search.Explanation{ Value: math.Sqrt(1.0) * idf, @@ -84,12 +95,12 @@ func TestTermScorer(t *testing.T) { // test the same thing again (score should be cached this time) { termMatch: &index.TermFieldDoc{ - ID: "one", + ID: testInternalId("one"), Freq: 1, Norm: 1.0, }, - result: &search.DocumentMatch{ - ID: "one", + result: &search.DocumentMatchInternal{ + ID: testInternalId("one"), Score: math.Sqrt(1.0) * idf, Expl: &search.Explanation{ Value: math.Sqrt(1.0) * idf, @@ -114,12 +125,12 @@ func TestTermScorer(t *testing.T) { // test a case where the sqrt isn't precalculated { termMatch: &index.TermFieldDoc{ - ID: "one", + ID: testInternalId("one"), Freq: 65, Norm: 1.0, }, - result: &search.DocumentMatch{ - ID: "one", + result: &search.DocumentMatchInternal{ + ID: testInternalId("one"), Score: math.Sqrt(65) * idf, Expl: &search.Explanation{ Value: math.Sqrt(65) * idf, @@ -173,16 +184,16 @@ func TestTermScorerWithQueryNorm(t *testing.T) { tests := []struct { termMatch *index.TermFieldDoc - result *search.DocumentMatch + result *search.DocumentMatchInternal }{ { termMatch: &index.TermFieldDoc{ - ID: "one", + ID: testInternalId("one"), Freq: 1, Norm: 1.0, }, - result: &search.DocumentMatch{ - ID: "one", + result: &search.DocumentMatchInternal{ + ID: testInternalId("one"), Score: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0, Expl: &search.Explanation{ Value: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0, diff --git a/search/search.go b/search/search.go index 4b3b988b..ca1726cb 100644 --- a/search/search.go +++ b/search/search.go @@ -9,6 +9,8 @@ package search +import "github.com/blevesearch/bleve/index" + type Location struct { Pos float64 `json:"pos"` Start float64 `json:"start"` @@ -50,6 +52,40 @@ type FieldTermLocationMap map[string]TermLocationMap type FieldFragmentMap map[string][]string +type DocumentMatchInternal struct { + Index string + ID index.IndexInternalID + Score float64 + Expl *Explanation + Locations FieldTermLocationMap + Fragments FieldFragmentMap + + // Fields contains the values for document fields listed in + // SearchRequest.Fields. Text fields are returned as strings, numeric + // fields as float64s and date fields as time.RFC3339 formatted strings. + Fields map[string]interface{} +} + +func (dm *DocumentMatchInternal) Reset() *DocumentMatchInternal { + *dm = DocumentMatchInternal{} + return dm +} + +func (dm *DocumentMatchInternal) Finalize(r index.IndexReader) (rv *DocumentMatch, err error) { + rv = &DocumentMatch{} + rv.ID, err = r.FinalizeDocID(dm.ID) + if err != nil { + return nil, err + } + rv.Index = dm.Index + rv.Expl = dm.Expl + rv.Fields = dm.Fields + rv.Fragments = dm.Fragments + rv.Locations = dm.Locations + rv.Score = dm.Score + return rv, nil +} + type DocumentMatch struct { Index string `json:"index,omitempty"` ID string `json:"id"` @@ -97,8 +133,8 @@ func (c DocumentMatchCollection) Swap(i, j int) { c[i], c[j] = c[j], c[i] } func (c DocumentMatchCollection) Less(i, j int) bool { return c[i].Score > c[j].Score } type Searcher interface { - Next(preAllocated *DocumentMatch) (*DocumentMatch, error) - Advance(ID string, preAllocated *DocumentMatch) (*DocumentMatch, error) + Next(preAllocated *DocumentMatchInternal) (*DocumentMatchInternal, error) + Advance(ID index.IndexInternalID, preAllocated *DocumentMatchInternal) (*DocumentMatchInternal, error) Close() error Weight() float64 SetQueryNorm(float64) diff --git a/search/searchers/search_boolean.go b/search/searchers/search_boolean.go index 0ae96483..b01e6f23 100644 --- a/search/searchers/search_boolean.go +++ b/search/searchers/search_boolean.go @@ -24,10 +24,10 @@ type BooleanSearcher struct { shouldSearcher search.Searcher mustNotSearcher search.Searcher queryNorm float64 - currMust *search.DocumentMatch - currShould *search.DocumentMatch - currMustNot *search.DocumentMatch - currentID string + currMust *search.DocumentMatchInternal + currShould *search.DocumentMatchInternal + currMustNot *search.DocumentMatchInternal + currentID index.IndexInternalID min uint64 scorer *scorers.ConjunctionQueryScorer } @@ -95,7 +95,7 @@ func (s *BooleanSearcher) initSearchers() error { } else if s.mustSearcher == nil && s.currShould != nil { s.currentID = s.currShould.ID } else { - s.currentID = "" + s.currentID = nil } s.initialized = true @@ -122,7 +122,7 @@ func (s *BooleanSearcher) advanceNextMust() error { } else if s.mustSearcher == nil && s.currShould != nil { s.currentID = s.currShould.ID } else { - s.currentID = "" + s.currentID = nil } return nil } @@ -148,7 +148,7 @@ func (s *BooleanSearcher) SetQueryNorm(qnorm float64) { } } -func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { if !s.initialized { err := s.initSearchers() @@ -158,16 +158,16 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu } var err error - var rv *search.DocumentMatch + var rv *search.DocumentMatchInternal - for s.currentID != "" { - if s.currMustNot != nil && s.currMustNot.ID < s.currentID { + for s.currentID != nil { + if s.currMustNot != nil && s.currMustNot.ID.Compare(s.currentID) < 0 { // advance must not searcher to our candidate entry s.currMustNot, err = s.mustNotSearcher.Advance(s.currentID, nil) if err != nil { return nil, err } - if s.currMustNot != nil && s.currMustNot.ID == s.currentID { + if s.currMustNot != nil && s.currMustNot.ID.Equals(s.currentID) { // the candidate is excluded err = s.advanceNextMust() if err != nil { @@ -175,7 +175,7 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu } continue } - } else if s.currMustNot != nil && s.currMustNot.ID == s.currentID { + } else if s.currMustNot != nil && s.currMustNot.ID.Equals(s.currentID) { // the candidate is excluded err = s.advanceNextMust() if err != nil { @@ -184,22 +184,22 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu continue } - if s.currShould != nil && s.currShould.ID < s.currentID { + if s.currShould != nil && s.currShould.ID.Compare(s.currentID) < 0 { // advance should searcher to our candidate entry s.currShould, err = s.shouldSearcher.Advance(s.currentID, nil) if err != nil { return nil, err } - if s.currShould != nil && s.currShould.ID == s.currentID { + if s.currShould != nil && s.currShould.ID.Equals(s.currentID) { // score bonus matches should - var cons []*search.DocumentMatch + var cons []*search.DocumentMatchInternal if s.currMust != nil { - cons = []*search.DocumentMatch{ + cons = []*search.DocumentMatchInternal{ s.currMust, s.currShould, } } else { - cons = []*search.DocumentMatch{ + cons = []*search.DocumentMatchInternal{ s.currShould, } } @@ -211,23 +211,23 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu break } else if s.shouldSearcher.Min() == 0 { // match is OK anyway - rv = s.scorer.Score([]*search.DocumentMatch{s.currMust}) + rv = s.scorer.Score([]*search.DocumentMatchInternal{s.currMust}) err = s.advanceNextMust() if err != nil { return nil, err } break } - } else if s.currShould != nil && s.currShould.ID == s.currentID { + } else if s.currShould != nil && s.currShould.ID.Equals(s.currentID) { // score bonus matches should - var cons []*search.DocumentMatch + var cons []*search.DocumentMatchInternal if s.currMust != nil { - cons = []*search.DocumentMatch{ + cons = []*search.DocumentMatchInternal{ s.currMust, s.currShould, } } else { - cons = []*search.DocumentMatch{ + cons = []*search.DocumentMatchInternal{ s.currShould, } } @@ -239,7 +239,7 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu break } else if s.shouldSearcher == nil || s.shouldSearcher.Min() == 0 { // match is OK anyway - rv = s.scorer.Score([]*search.DocumentMatch{s.currMust}) + rv = s.scorer.Score([]*search.DocumentMatchInternal{s.currMust}) err = s.advanceNextMust() if err != nil { return nil, err @@ -255,7 +255,7 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu return rv, nil } -func (s *BooleanSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *BooleanSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { if !s.initialized { err := s.initSearchers() @@ -289,7 +289,7 @@ func (s *BooleanSearcher) Advance(ID string, preAllocated *search.DocumentMatch) } else if s.mustSearcher == nil && s.currShould != nil { s.currentID = s.currShould.ID } else { - s.currentID = "" + s.currentID = nil } return s.Next(preAllocated) diff --git a/search/searchers/search_boolean_test.go b/search/searchers/search_boolean_test.go index ac6dc7ea..a09da69b 100644 --- a/search/searchers/search_boolean_test.go +++ b/search/searchers/search_boolean_test.go @@ -12,6 +12,7 @@ package searchers import ( "testing" + "github.com/blevesearch/bleve/index/upside_down" "github.com/blevesearch/bleve/search" ) @@ -242,34 +243,34 @@ func TestBooleanSearch(t *testing.T) { tests := []struct { searcher search.Searcher - results []*search.DocumentMatch + results []*search.DocumentMatchInternal }{ { searcher: booleanSearcher, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "1", + ID: upside_down.InternalId("1"), Score: 0.9818005051949021, }, { - ID: "3", + ID: upside_down.InternalId("3"), Score: 0.808709699395535, }, { - ID: "4", + ID: upside_down.InternalId("4"), Score: 0.34618161159873423, }, }, }, { searcher: booleanSearcher2, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "1", + ID: upside_down.InternalId("1"), Score: 0.6775110856165737, }, { - ID: "3", + ID: upside_down.InternalId("3"), Score: 0.6775110856165737, }, }, @@ -277,57 +278,57 @@ func TestBooleanSearch(t *testing.T) { // no MUST or SHOULD clauses yields no results { searcher: booleanSearcher3, - results: []*search.DocumentMatch{}, + results: []*search.DocumentMatchInternal{}, }, { searcher: booleanSearcher4, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "1", + ID: upside_down.InternalId("1"), Score: 1.0, }, { - ID: "3", + ID: upside_down.InternalId("3"), Score: 0.5, }, { - ID: "4", + ID: upside_down.InternalId("4"), Score: 1.0, }, }, }, { searcher: booleanSearcher5, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "3", + ID: upside_down.InternalId("3"), Score: 0.5, }, { - ID: "4", + ID: upside_down.InternalId("4"), Score: 1.0, }, }, }, { searcher: booleanSearcher6, - results: []*search.DocumentMatch{}, + results: []*search.DocumentMatchInternal{}, }, // test a conjunction query with a nested boolean { searcher: conjunctionSearcher7, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "1", + ID: upside_down.InternalId("1"), Score: 2.0097428702814377, }, }, }, { searcher: conjunctionSearcher8, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "3", + ID: upside_down.InternalId("3"), Score: 2.0681575785068107, }, }, @@ -346,7 +347,7 @@ func TestBooleanSearch(t *testing.T) { i := 0 for err == nil && next != nil { if i < len(test.results) { - if next.ID != test.results[i].ID { + if !next.ID.Equals(test.results[i].ID) { t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { diff --git a/search/searchers/search_conjunction.go b/search/searchers/search_conjunction.go index fe37b80b..031a977b 100644 --- a/search/searchers/search_conjunction.go +++ b/search/searchers/search_conjunction.go @@ -24,8 +24,8 @@ type ConjunctionSearcher struct { searchers OrderedSearcherList explain bool queryNorm float64 - currs []*search.DocumentMatch - currentID string + currs []*search.DocumentMatchInternal + currentID index.IndexInternalID scorer *scorers.ConjunctionQueryScorer } @@ -42,7 +42,7 @@ func NewConjunctionSearcher(indexReader index.IndexReader, qsearchers []search.S indexReader: indexReader, explain: explain, searchers: searchers, - currs: make([]*search.DocumentMatch, len(searchers)), + currs: make([]*search.DocumentMatchInternal, len(searchers)), scorer: scorers.NewConjunctionQueryScorer(explain), } rv.computeQueryNorm() @@ -77,7 +77,7 @@ func (s *ConjunctionSearcher) initSearchers() error { if s.currs[0] != nil { s.currentID = s.currs[0].ID } else { - s.currentID = "" + s.currentID = nil } } @@ -99,20 +99,20 @@ func (s *ConjunctionSearcher) SetQueryNorm(qnorm float64) { } } -func (s *ConjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *ConjunctionSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { if !s.initialized { err := s.initSearchers() if err != nil { return nil, err } } - var rv *search.DocumentMatch + var rv *search.DocumentMatchInternal var err error OUTER: - for s.currentID != "" { + for s.currentID != nil { for i, termSearcher := range s.searchers { - if s.currs[i] != nil && s.currs[i].ID != s.currentID { - if s.currentID < s.currs[i].ID { + if s.currs[i] != nil && !s.currs[i].ID.Equals(s.currentID) { + if s.currentID.Compare(s.currs[i].ID) < 0 { s.currentID = s.currs[i].ID continue OUTER } @@ -122,17 +122,17 @@ OUTER: return nil, err } if s.currs[i] == nil { - s.currentID = "" + s.currentID = nil continue OUTER } - if s.currs[i].ID != s.currentID { + if !s.currs[i].ID.Equals(s.currentID) { // we just advanced, so it doesn't match, it must be greater // no need to call next s.currentID = s.currs[i].ID continue OUTER } } else if s.currs[i] == nil { - s.currentID = "" + s.currentID = nil continue OUTER } } @@ -145,7 +145,7 @@ OUTER: return nil, err } if s.currs[0] == nil { - s.currentID = "" + s.currentID = nil } else { s.currentID = s.currs[0].ID } @@ -155,7 +155,7 @@ OUTER: return rv, nil } -func (s *ConjunctionSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *ConjunctionSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { if !s.initialized { err := s.initSearchers() if err != nil { diff --git a/search/searchers/search_conjunction_test.go b/search/searchers/search_conjunction_test.go index 1256059b..f4baa19d 100644 --- a/search/searchers/search_conjunction_test.go +++ b/search/searchers/search_conjunction_test.go @@ -12,6 +12,7 @@ package searchers import ( "testing" + "github.com/blevesearch/bleve/index/upside_down" "github.com/blevesearch/bleve/search" ) @@ -122,57 +123,57 @@ func TestConjunctionSearch(t *testing.T) { tests := []struct { searcher search.Searcher - results []*search.DocumentMatch + results []*search.DocumentMatchInternal }{ { searcher: beerAndMartySearcher, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "1", + ID: upside_down.InternalId("1"), Score: 2.0097428702814377, }, }, }, { searcher: angstAndBeerSearcher, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "2", + ID: upside_down.InternalId("2"), Score: 1.0807601687084403, }, }, }, { searcher: beerAndJackSearcher, - results: []*search.DocumentMatch{}, + results: []*search.DocumentMatchInternal{}, }, { searcher: beerAndMisterSearcher, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "2", + ID: upside_down.InternalId("2"), Score: 1.2877980334016337, }, { - ID: "3", + ID: upside_down.InternalId("3"), Score: 1.2877980334016337, }, }, }, { searcher: couchbaseAndMisterSearcher, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "2", + ID: upside_down.InternalId("2"), Score: 1.4436599157093672, }, }, }, { searcher: beerAndCouchbaseAndMisterSearcher, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "2", + ID: upside_down.InternalId("2"), Score: 1.441614953806971, }, }, @@ -191,7 +192,7 @@ func TestConjunctionSearch(t *testing.T) { i := 0 for err == nil && next != nil { if i < len(test.results) { - if next.ID != test.results[i].ID { + if !next.ID.Equals(test.results[i].ID) { t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { diff --git a/search/searchers/search_disjunction.go b/search/searchers/search_disjunction.go index d7cd9408..061ac9d2 100644 --- a/search/searchers/search_disjunction.go +++ b/search/searchers/search_disjunction.go @@ -29,8 +29,8 @@ type DisjunctionSearcher struct { indexReader index.IndexReader searchers OrderedSearcherList queryNorm float64 - currs []*search.DocumentMatch - currentID string + currs []*search.DocumentMatchInternal + currentID index.IndexInternalID scorer *scorers.DisjunctionQueryScorer min float64 } @@ -61,7 +61,7 @@ func NewDisjunctionSearcher(indexReader index.IndexReader, qsearchers []search.S rv := DisjunctionSearcher{ indexReader: indexReader, searchers: searchers, - currs: make([]*search.DocumentMatch, len(searchers)), + currs: make([]*search.DocumentMatchInternal, len(searchers)), scorer: scorers.NewDisjunctionQueryScorer(explain), min: min, } @@ -98,10 +98,10 @@ func (s *DisjunctionSearcher) initSearchers() error { return nil } -func (s *DisjunctionSearcher) nextSmallestID() string { - rv := "" +func (s *DisjunctionSearcher) nextSmallestID() index.IndexInternalID { + var rv index.IndexInternalID for _, curr := range s.currs { - if curr != nil && (curr.ID < rv || rv == "") { + if curr != nil && (curr.ID.Compare(rv) < 0 || rv == nil) { rv = curr.ID } } @@ -122,7 +122,7 @@ func (s *DisjunctionSearcher) SetQueryNorm(qnorm float64) { } } -func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { if !s.initialized { err := s.initSearchers() if err != nil { @@ -130,13 +130,13 @@ func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search. } } var err error - var rv *search.DocumentMatch - matching := make([]*search.DocumentMatch, 0, len(s.searchers)) + var rv *search.DocumentMatchInternal + matching := make([]*search.DocumentMatchInternal, 0, len(s.searchers)) found := false - for !found && s.currentID != "" { + for !found && s.currentID != nil { for _, curr := range s.currs { - if curr != nil && curr.ID == s.currentID { + if curr != nil && curr.ID.Equals(s.currentID) { matching = append(matching, curr) } } @@ -148,10 +148,10 @@ func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search. } // reset matching - matching = make([]*search.DocumentMatch, 0) + matching = make([]*search.DocumentMatchInternal, 0) // invoke next on all the matching searchers for i, curr := range s.currs { - if curr != nil && curr.ID == s.currentID { + if curr != nil && curr.ID.Equals(s.currentID) { searcher := s.searchers[i] s.currs[i], err = searcher.Next(nil) if err != nil { @@ -164,7 +164,7 @@ func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search. return rv, nil } -func (s *DisjunctionSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *DisjunctionSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { if !s.initialized { err := s.initSearchers() if err != nil { diff --git a/search/searchers/search_disjunction_test.go b/search/searchers/search_disjunction_test.go index 19b0dd42..231bfb0e 100644 --- a/search/searchers/search_disjunction_test.go +++ b/search/searchers/search_disjunction_test.go @@ -12,6 +12,7 @@ package searchers import ( "testing" + "github.com/blevesearch/bleve/index/upside_down" "github.com/blevesearch/bleve/search" ) @@ -65,17 +66,17 @@ func TestDisjunctionSearch(t *testing.T) { tests := []struct { searcher search.Searcher - results []*search.DocumentMatch + results []*search.DocumentMatchInternal }{ { searcher: martyOrDustinSearcher, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "1", + ID: upside_down.InternalId("1"), Score: 0.6775110856165737, }, { - ID: "3", + ID: upside_down.InternalId("3"), Score: 0.6775110856165737, }, }, @@ -83,17 +84,17 @@ func TestDisjunctionSearch(t *testing.T) { // test a nested disjunction { searcher: nestedRaviOrMartyOrDustinSearcher, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "1", + ID: upside_down.InternalId("1"), Score: 0.2765927424732821, }, { - ID: "3", + ID: upside_down.InternalId("3"), Score: 0.2765927424732821, }, { - ID: "4", + ID: upside_down.InternalId("4"), Score: 0.5531854849465642, }, }, @@ -112,7 +113,7 @@ func TestDisjunctionSearch(t *testing.T) { i := 0 for err == nil && next != nil { if i < len(test.results) { - if next.ID != test.results[i].ID { + if !next.ID.Equals(test.results[i].ID) { t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { @@ -158,7 +159,7 @@ func TestDisjunctionAdvance(t *testing.T) { t.Fatal(err) } - match, err := martyOrDustinSearcher.Advance("3", nil) + match, err := martyOrDustinSearcher.Advance(upside_down.InternalId("3"), nil) if err != nil { t.Errorf("unexpected error: %v", err) } diff --git a/search/searchers/search_docid.go b/search/searchers/search_docid.go index 262583d0..72c489cf 100644 --- a/search/searchers/search_docid.go +++ b/search/searchers/search_docid.go @@ -10,8 +10,6 @@ package searchers import ( - "sort" - "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search/scorers" @@ -19,54 +17,61 @@ import ( // DocIDSearcher returns documents matching a predefined set of identifiers. type DocIDSearcher struct { - ids []string - current int - scorer *scorers.ConstantScorer + reader index.DocIDReader + scorer *scorers.ConstantScorer + count int } func NewDocIDSearcher(indexReader index.IndexReader, ids []string, boost float64, explain bool) (searcher *DocIDSearcher, err error) { - kept := make([]string, len(ids)) - copy(kept, ids) - sort.Strings(kept) + // kept := make([]string, len(ids)) + // copy(kept, ids) + // sort.Strings(kept) + // + // if len(ids) > 0 { + // var idReader index.DocIDReader + // endTerm := string(incrementBytes([]byte(kept[len(kept)-1]))) + // idReader, err = indexReader.DocIDReader(kept[0], endTerm) + // if err != nil { + // return nil, err + // } + // defer func() { + // if cerr := idReader.Close(); err == nil && cerr != nil { + // err = cerr + // } + // }() + // j := 0 + // for _, id := range kept { + // doc, err := idReader.Next() + // if err != nil { + // return nil, err + // } + // // Non-duplicate match + // actualDocID := indexReader.FinalizeDocID(doc) + // if actualDocID == id && (j == 0 || kept[j-1] != id) { + // kept[j] = id + // j++ + // } + // } + // kept = kept[:j] + // } - if len(ids) > 0 { - var idReader index.DocIDReader - endTerm := string(incrementBytes([]byte(kept[len(kept)-1]))) - idReader, err = indexReader.DocIDReader(kept[0], endTerm) - if err != nil { - return nil, err - } - defer func() { - if cerr := idReader.Close(); err == nil && cerr != nil { - err = cerr - } - }() - j := 0 - for _, id := range kept { - doc, err := idReader.Advance(id) - if err != nil { - return nil, err - } - // Non-duplicate match - if doc == id && (j == 0 || kept[j-1] != id) { - kept[j] = id - j++ - } - } - kept = kept[:j] + reader, err := indexReader.DocIDReaderOnly(ids) + if err != nil { + return nil, err } - scorer := scorers.NewConstantScorer(1.0, boost, explain) return &DocIDSearcher{ - ids: kept, scorer: scorer, + reader: reader, + count: len(ids), }, nil } func (s *DocIDSearcher) Count() uint64 { - return uint64(len(s.ids)) + // return uint64(len(s.ids)) + return uint64(s.count) } func (s *DocIDSearcher) Weight() float64 { @@ -77,20 +82,41 @@ func (s *DocIDSearcher) SetQueryNorm(qnorm float64) { s.scorer.SetQueryNorm(qnorm) } -func (s *DocIDSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - if s.current >= len(s.ids) { +func (s *DocIDSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { + // if s.current >= len(s.ids) { + // return nil, nil + // } + // id := s.ids[s.current] + // s.current++ + // docMatch := s.scorer.Score(id) + // return docMatch, nil + + docidMatch, err := s.reader.Next() + if err != nil { + return nil, err + } + if docidMatch == nil { return nil, nil } - id := s.ids[s.current] - s.current++ - docMatch := s.scorer.Score(id) - return docMatch, nil + docMatch := s.scorer.Score(docidMatch) + return docMatch, nil } -func (s *DocIDSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - s.current = sort.SearchStrings(s.ids, ID) - return s.Next(preAllocated) +func (s *DocIDSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { + // s.current = sort.SearchStrings(s.ids, ID) + // return s.Next(preAllocated) + + docidMatch, err := s.reader.Advance(ID) + if err != nil { + return nil, err + } + if docidMatch == nil { + return nil, nil + } + + docMatch := s.scorer.Score(docidMatch) + return docMatch, nil } func (s *DocIDSearcher) Close() error { diff --git a/search/searchers/search_docid_test.go b/search/searchers/search_docid_test.go index ad89288b..271a25bc 100644 --- a/search/searchers/search_docid_test.go +++ b/search/searchers/search_docid_test.go @@ -62,17 +62,13 @@ func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { } }() - if searcher.Count() != uint64(len(wanted)) { - t.Fatalf("expected count %v got %v", len(wanted), searcher.Count()) - } - // Check the sequence for i, id := range wanted { m, err := searcher.Next(nil) if err != nil { t.Fatal(err) } - if id != m.ID { + if !upside_down.InternalId(id).Equals(m.ID) { t.Fatalf("expected %v at position %v, got %v", id, i, m.ID) } } @@ -91,18 +87,18 @@ func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { } before := id[:1] for _, target := range []string{before, id} { - m, err := searcher.Advance(target, nil) + m, err := searcher.Advance(upside_down.InternalId(target), nil) if err != nil { t.Fatal(err) } - if m == nil || m.ID != id { + if m == nil || !m.ID.Equals(upside_down.InternalId(id)) { t.Fatalf("advancing to %v returned %v instead of %v", before, m, id) } } } // Seek after the end of the sequence after := "zzz" - m, err = searcher.Advance(after, nil) + m, err = searcher.Advance(upside_down.InternalId(after), nil) if err != nil { t.Fatal(err) } diff --git a/search/searchers/search_fuzzy.go b/search/searchers/search_fuzzy.go index da328a7e..9f42995c 100644 --- a/search/searchers/search_fuzzy.go +++ b/search/searchers/search_fuzzy.go @@ -107,12 +107,12 @@ func (s *FuzzySearcher) SetQueryNorm(qnorm float64) { s.searcher.SetQueryNorm(qnorm) } -func (s *FuzzySearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *FuzzySearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { return s.searcher.Next(preAllocated) } -func (s *FuzzySearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *FuzzySearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { return s.searcher.Advance(ID, preAllocated) } diff --git a/search/searchers/search_fuzzy_test.go b/search/searchers/search_fuzzy_test.go index 610367de..52638044 100644 --- a/search/searchers/search_fuzzy_test.go +++ b/search/searchers/search_fuzzy_test.go @@ -12,6 +12,7 @@ package searchers import ( "testing" + "github.com/blevesearch/bleve/index/upside_down" "github.com/blevesearch/bleve/search" ) @@ -50,47 +51,47 @@ func TestFuzzySearch(t *testing.T) { tests := []struct { searcher search.Searcher - results []*search.DocumentMatch + results []*search.DocumentMatchInternal }{ { searcher: fuzzySearcherbeet, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "1", + ID: upside_down.InternalId("1"), Score: 1.0, }, { - ID: "2", + ID: upside_down.InternalId("2"), Score: 0.5, }, { - ID: "3", + ID: upside_down.InternalId("3"), Score: 0.5, }, { - ID: "4", + ID: upside_down.InternalId("4"), Score: 0.9999999838027345, }, }, }, { searcher: fuzzySearcherdouches, - results: []*search.DocumentMatch{}, + results: []*search.DocumentMatchInternal{}, }, { searcher: fuzzySearcheraplee, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "3", + ID: upside_down.InternalId("3"), Score: 0.9581453659370776, }, }, }, { searcher: fuzzySearcherprefix, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "5", + ID: upside_down.InternalId("5"), Score: 1.916290731874155, }, }, @@ -109,7 +110,7 @@ func TestFuzzySearch(t *testing.T) { i := 0 for err == nil && next != nil { if i < len(test.results) { - if next.ID != test.results[i].ID { + if !next.ID.Equals(test.results[i].ID) { t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) } if next.Score != test.results[i].Score { diff --git a/search/searchers/search_match_all.go b/search/searchers/search_match_all.go index bd26ba13..77cde2e1 100644 --- a/search/searchers/search_match_all.go +++ b/search/searchers/search_match_all.go @@ -46,13 +46,13 @@ func (s *MatchAllSearcher) SetQueryNorm(qnorm float64) { s.scorer.SetQueryNorm(qnorm) } -func (s *MatchAllSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *MatchAllSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { id, err := s.reader.Next() if err != nil { return nil, err } - if id == "" { + if id == nil { return nil, nil } @@ -63,13 +63,13 @@ func (s *MatchAllSearcher) Next(preAllocated *search.DocumentMatch) (*search.Doc } -func (s *MatchAllSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *MatchAllSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { id, err := s.reader.Advance(ID) if err != nil { return nil, err } - if id == "" { + if id == nil { return nil, nil } diff --git a/search/searchers/search_match_all_test.go b/search/searchers/search_match_all_test.go index 5bcd1b51..32b85514 100644 --- a/search/searchers/search_match_all_test.go +++ b/search/searchers/search_match_all_test.go @@ -12,6 +12,7 @@ package searchers import ( "testing" + "github.com/blevesearch/bleve/index/upside_down" "github.com/blevesearch/bleve/search" ) @@ -41,30 +42,30 @@ func TestMatchAllSearch(t *testing.T) { tests := []struct { searcher search.Searcher queryNorm float64 - results []*search.DocumentMatch + results []*search.DocumentMatchInternal }{ { searcher: allSearcher, queryNorm: 1.0, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "1", + ID: upside_down.InternalId("1"), Score: 1.0, }, { - ID: "2", + ID: upside_down.InternalId("2"), Score: 1.0, }, { - ID: "3", + ID: upside_down.InternalId("3"), Score: 1.0, }, { - ID: "4", + ID: upside_down.InternalId("4"), Score: 1.0, }, { - ID: "5", + ID: upside_down.InternalId("5"), Score: 1.0, }, }, @@ -72,25 +73,25 @@ func TestMatchAllSearch(t *testing.T) { { searcher: allSearcher2, queryNorm: 0.8333333, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "1", + ID: upside_down.InternalId("1"), Score: 1.0, }, { - ID: "2", + ID: upside_down.InternalId("2"), Score: 1.0, }, { - ID: "3", + ID: upside_down.InternalId("3"), Score: 1.0, }, { - ID: "4", + ID: upside_down.InternalId("4"), Score: 1.0, }, { - ID: "5", + ID: upside_down.InternalId("5"), Score: 1.0, }, }, @@ -113,7 +114,7 @@ func TestMatchAllSearch(t *testing.T) { i := 0 for err == nil && next != nil { if i < len(test.results) { - if next.ID != test.results[i].ID { + if !next.ID.Equals(test.results[i].ID) { t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { diff --git a/search/searchers/search_match_none.go b/search/searchers/search_match_none.go index 0d4f5a9a..08bae499 100644 --- a/search/searchers/search_match_none.go +++ b/search/searchers/search_match_none.go @@ -36,11 +36,11 @@ func (s *MatchNoneSearcher) SetQueryNorm(qnorm float64) { } -func (s *MatchNoneSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *MatchNoneSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { return nil, nil } -func (s *MatchNoneSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *MatchNoneSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { return nil, nil } diff --git a/search/searchers/search_match_none_test.go b/search/searchers/search_match_none_test.go index 0d470358..2752f998 100644 --- a/search/searchers/search_match_none_test.go +++ b/search/searchers/search_match_none_test.go @@ -35,11 +35,11 @@ func TestMatchNoneSearch(t *testing.T) { tests := []struct { searcher search.Searcher - results []*search.DocumentMatch + results []*search.DocumentMatchInternal }{ { searcher: noneSearcher, - results: []*search.DocumentMatch{}, + results: []*search.DocumentMatchInternal{}, }, } diff --git a/search/searchers/search_numeric_range.go b/search/searchers/search_numeric_range.go index fdf3c4c0..067ad9f6 100644 --- a/search/searchers/search_numeric_range.go +++ b/search/searchers/search_numeric_range.go @@ -96,11 +96,11 @@ func (s *NumericRangeSearcher) SetQueryNorm(qnorm float64) { s.searcher.SetQueryNorm(qnorm) } -func (s *NumericRangeSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *NumericRangeSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { return s.searcher.Next(preAllocated) } -func (s *NumericRangeSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *NumericRangeSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { return s.searcher.Advance(ID, preAllocated) } diff --git a/search/searchers/search_phrase.go b/search/searchers/search_phrase.go index 11e8c9c8..b7c63ef1 100644 --- a/search/searchers/search_phrase.go +++ b/search/searchers/search_phrase.go @@ -21,7 +21,7 @@ type PhraseSearcher struct { indexReader index.IndexReader mustSearcher *ConjunctionSearcher queryNorm float64 - currMust *search.DocumentMatch + currMust *search.DocumentMatchInternal slop int terms []string } @@ -90,7 +90,7 @@ func (s *PhraseSearcher) SetQueryNorm(qnorm float64) { s.mustSearcher.SetQueryNorm(qnorm) } -func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { if !s.initialized { err := s.initSearchers() if err != nil { @@ -98,7 +98,7 @@ func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docum } } - var rv *search.DocumentMatch + var rv *search.DocumentMatchInternal for s.currMust != nil { rvftlm := make(search.FieldTermLocationMap, 0) freq := 0 @@ -160,7 +160,7 @@ func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docum return nil, nil } -func (s *PhraseSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *PhraseSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { if !s.initialized { err := s.initSearchers() if err != nil { diff --git a/search/searchers/search_phrase_test.go b/search/searchers/search_phrase_test.go index a3c14b70..e9fcdb55 100644 --- a/search/searchers/search_phrase_test.go +++ b/search/searchers/search_phrase_test.go @@ -12,6 +12,7 @@ package searchers import ( "testing" + "github.com/blevesearch/bleve/index/upside_down" "github.com/blevesearch/bleve/search" ) @@ -47,13 +48,13 @@ func TestPhraseSearch(t *testing.T) { tests := []struct { searcher search.Searcher - results []*search.DocumentMatch + results []*search.DocumentMatchInternal }{ { searcher: phraseSearcher, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "2", + ID: upside_down.InternalId("2"), Score: 1.0807601687084403, }, }, @@ -72,7 +73,7 @@ func TestPhraseSearch(t *testing.T) { i := 0 for err == nil && next != nil { if i < len(test.results) { - if next.ID != test.results[i].ID { + if !next.ID.Equals(test.results[i].ID) { t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) } if next.Score != test.results[i].Score { diff --git a/search/searchers/search_regexp.go b/search/searchers/search_regexp.go index d92822f1..6173f07a 100644 --- a/search/searchers/search_regexp.go +++ b/search/searchers/search_regexp.go @@ -106,12 +106,12 @@ func (s *RegexpSearcher) SetQueryNorm(qnorm float64) { s.searcher.SetQueryNorm(qnorm) } -func (s *RegexpSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *RegexpSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { return s.searcher.Next(preAllocated) } -func (s *RegexpSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *RegexpSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { return s.searcher.Advance(ID, preAllocated) } diff --git a/search/searchers/search_regexp_test.go b/search/searchers/search_regexp_test.go index cb4d01ee..0552866f 100644 --- a/search/searchers/search_regexp_test.go +++ b/search/searchers/search_regexp_test.go @@ -13,6 +13,7 @@ import ( "regexp" "testing" + "github.com/blevesearch/bleve/index/upside_down" "github.com/blevesearch/bleve/search" ) @@ -51,26 +52,26 @@ func TestRegexpSearch(t *testing.T) { tests := []struct { searcher search.Searcher - results []*search.DocumentMatch + results []*search.DocumentMatchInternal }{ { searcher: regexpSearcher, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "1", + ID: upside_down.InternalId("1"), Score: 1.916290731874155, }, }, }, { searcher: regexpSearcherCo, - results: []*search.DocumentMatch{ + results: []*search.DocumentMatchInternal{ { - ID: "2", + ID: upside_down.InternalId("2"), Score: 0.33875554280828685, }, { - ID: "3", + ID: upside_down.InternalId("3"), Score: 0.33875554280828685, }, }, @@ -89,7 +90,7 @@ func TestRegexpSearch(t *testing.T) { i := 0 for err == nil && next != nil { if i < len(test.results) { - if next.ID != test.results[i].ID { + if !next.ID.Equals(test.results[i].ID) { t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) } if next.Score != test.results[i].Score { diff --git a/search/searchers/search_term.go b/search/searchers/search_term.go index 94c3eba9..aa95bc38 100644 --- a/search/searchers/search_term.go +++ b/search/searchers/search_term.go @@ -53,7 +53,7 @@ func (s *TermSearcher) SetQueryNorm(qnorm float64) { s.scorer.SetQueryNorm(qnorm) } -func (s *TermSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *TermSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { termMatch, err := s.reader.Next(s.tfd.Reset()) if err != nil { return nil, err @@ -70,7 +70,7 @@ func (s *TermSearcher) Next(preAllocated *search.DocumentMatch) (*search.Documen } -func (s *TermSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *TermSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { termMatch, err := s.reader.Advance(ID, s.tfd.Reset()) if err != nil { return nil, err diff --git a/search/searchers/search_term_prefix.go b/search/searchers/search_term_prefix.go index 35f34722..bbd8a78c 100644 --- a/search/searchers/search_term_prefix.go +++ b/search/searchers/search_term_prefix.go @@ -70,12 +70,12 @@ func (s *TermPrefixSearcher) SetQueryNorm(qnorm float64) { s.searcher.SetQueryNorm(qnorm) } -func (s *TermPrefixSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *TermPrefixSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { return s.searcher.Next(preAllocated) } -func (s *TermPrefixSearcher) Advance(ID string, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *TermPrefixSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { return s.searcher.Advance(ID, preAllocated) } diff --git a/search/searchers/search_term_test.go b/search/searchers/search_term_test.go index ef3b927f..83381e43 100644 --- a/search/searchers/search_term_test.go +++ b/search/searchers/search_term_test.go @@ -167,19 +167,19 @@ func TestTermSearcher(t *testing.T) { if err != nil { t.Errorf("expected result, got %v", err) } - if docMatch.ID != "a" { + if !docMatch.ID.Equals(upside_down.InternalId("a")) { t.Errorf("expected result ID to be 'a', got '%s", docMatch.ID) } - docMatch, err = searcher.Advance("c", nil) + docMatch, err = searcher.Advance(upside_down.InternalId("c"), nil) if err != nil { t.Errorf("expected result, got %v", err) } - if docMatch.ID != "c" { + if !docMatch.ID.Equals(upside_down.InternalId("c")) { t.Errorf("expected result ID to be 'c' got '%s'", docMatch.ID) } // try advancing past end - docMatch, err = searcher.Advance("z", nil) + docMatch, err = searcher.Advance(upside_down.InternalId("z"), nil) if err != nil { t.Fatal(err) } From 1aacd9bad5b0ff7b282c86526342c0518d8dcb6f Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Mon, 1 Aug 2016 14:26:50 -0400 Subject: [PATCH 03/13] changed approach IndexInternalID is now []byte this is still opaque, and should still work for any future index implementations as it is a least common denominator choice, all implementations must internally represent the id as []byte at some point for storage to disk --- index/index.go | 15 ++- index/upside_down/index_reader.go | 21 +--- index/upside_down/reader.go | 24 ++-- index/upside_down/reader_test.go | 16 +-- index/upside_down/upside_down.go | 8 +- index/upside_down/upside_down_test.go | 10 +- search/collectors/bench_test.go | 3 +- search/collectors/collector_top_score_test.go | 113 +++++++++--------- search/collectors/search_test.go | 14 +-- search/scorers/scorer_constant_test.go | 8 +- search/scorers/scorer_term_test.go | 27 ++--- search/searchers/search_boolean_test.go | 26 ++-- search/searchers/search_conjunction_test.go | 14 +-- search/searchers/search_disjunction_test.go | 14 +-- search/searchers/search_docid_test.go | 8 +- search/searchers/search_fuzzy_test.go | 14 +-- search/searchers/search_match_all_test.go | 22 ++-- search/searchers/search_match_none_test.go | 2 +- search/searchers/search_phrase_test.go | 4 +- search/searchers/search_regexp_test.go | 8 +- search/searchers/search_term_test.go | 8 +- 21 files changed, 172 insertions(+), 207 deletions(-) diff --git a/index/index.go b/index/index.go index 6804bb3a..880c35fe 100644 --- a/index/index.go +++ b/index/index.go @@ -10,6 +10,7 @@ package index import ( + "bytes" "encoding/json" "fmt" "time" @@ -103,12 +104,14 @@ type TermFieldVector struct { } // IndexInternalID is an opaque document identifier interal to the index impl -// This allows us to delay the conversion to public identifier (string) and -// avoid it completely in other cases. It also servces to hide the underlying -// representation of a document identifer, allow more flexibility. -type IndexInternalID interface { - Equals(other IndexInternalID) bool - Compare(other IndexInternalID) int +type IndexInternalID []byte + +func (id IndexInternalID) Equals(other IndexInternalID) bool { + return id.Compare(other) == 0 +} + +func (id IndexInternalID) Compare(other IndexInternalID) int { + return bytes.Compare(id, other) } type TermFieldDoc struct { diff --git a/index/upside_down/index_reader.go b/index/upside_down/index_reader.go index ba09ed9d..181b8b14 100644 --- a/index/upside_down/index_reader.go +++ b/index/upside_down/index_reader.go @@ -10,7 +10,6 @@ package upside_down import ( - "bytes" "fmt" "github.com/blevesearch/bleve/document" @@ -18,20 +17,6 @@ import ( "github.com/blevesearch/bleve/index/store" ) -type InternalId []byte - -func (u InternalId) Compare(other index.IndexInternalID) int { - if other == nil { - // this internal ID is always greater than nil - return 1 - } - return bytes.Compare(u, other.(InternalId)) -} - -func (u InternalId) Equals(other index.IndexInternalID) bool { - return u.Compare(other.(InternalId)) == 0 -} - type IndexReader struct { index *UpsideDownCouch kvreader store.KVReader @@ -114,7 +99,7 @@ func (i *IndexReader) Document(id string) (doc *document.Document, err error) { } func (i *IndexReader) DocumentFieldTerms(id index.IndexInternalID) (index.FieldTerms, error) { - back, err := i.index.backIndexRowForDoc(i.kvreader, id.(InternalId)) + back, err := i.index.backIndexRowForDoc(i.kvreader, id) if err != nil { return nil, err } @@ -132,7 +117,7 @@ func (i *IndexReader) DocumentFieldTerms(id index.IndexInternalID) (index.FieldT } func (i *IndexReader) DocumentFieldTermsForFields(id index.IndexInternalID, fields []string) (index.FieldTerms, error) { - back, err := i.index.backIndexRowForDoc(i.kvreader, id.(InternalId)) + back, err := i.index.backIndexRowForDoc(i.kvreader, id) if err != nil { return nil, err } @@ -201,7 +186,7 @@ func (i *IndexReader) Close() error { } func (i *IndexReader) FinalizeDocID(id index.IndexInternalID) (string, error) { - return string(id.(InternalId)), nil + return string(id), nil } func incrementBytes(in []byte) []byte { diff --git a/index/upside_down/reader.go b/index/upside_down/reader.go index 1764dbbc..92f77dbf 100644 --- a/index/upside_down/reader.go +++ b/index/upside_down/reader.go @@ -83,7 +83,7 @@ func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (* if rv == nil { rv = &index.TermFieldDoc{} } - rv.ID = InternalId(tfr.doc) + rv.ID = tfr.doc rv.Freq = tfr.freq rv.Norm = float64(tfr.norm) if tfr.vectors != nil { @@ -96,8 +96,7 @@ func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (* return nil, nil } -func (r *UpsideDownCouchTermFieldReader) Advance(docIDInternal index.IndexInternalID, preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) { - docID := docIDInternal.(InternalId) +func (r *UpsideDownCouchTermFieldReader) Advance(docID index.IndexInternalID, preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) { if r.iterator != nil { tfr := NewTermFrequencyRow(r.term, r.field, docID, 0, 0) r.iterator.Seek(tfr.Key()) @@ -111,7 +110,7 @@ func (r *UpsideDownCouchTermFieldReader) Advance(docIDInternal index.IndexIntern if rv == nil { rv = &index.TermFieldDoc{} } - rv.ID = InternalId(tfr.doc) + rv.ID = tfr.doc rv.Freq = tfr.freq rv.Norm = float64(tfr.norm) if tfr.vectors != nil { @@ -185,7 +184,7 @@ func (r *UpsideDownCouchDocIDReader) Next() (index.IndexInternalID, error) { key, val, valid := r.iterator.Current() if r.onlyMode { - var rv InternalId + var rv index.IndexInternalID for valid && r.onlyPos < len(r.only) { br, err := NewBackIndexRowKV(key, val) if err != nil { @@ -200,7 +199,7 @@ func (r *UpsideDownCouchDocIDReader) Next() (index.IndexInternalID, error) { key, val, valid = r.iterator.Current() continue } else { - rv = InternalId(br.doc) + rv = br.doc break } } @@ -218,7 +217,7 @@ func (r *UpsideDownCouchDocIDReader) Next() (index.IndexInternalID, error) { if err != nil { return nil, err } - rv := InternalId(br.doc) + rv := br.doc r.iterator.Next() return rv, nil } @@ -227,14 +226,13 @@ func (r *UpsideDownCouchDocIDReader) Next() (index.IndexInternalID, error) { } func (r *UpsideDownCouchDocIDReader) Advance(docID index.IndexInternalID) (index.IndexInternalID, error) { - docIDInternal := docID.(InternalId) - bir := NewBackIndexRow(docIDInternal, nil, nil) + bir := NewBackIndexRow(docID, nil, nil) r.iterator.Seek(bir.Key()) key, val, valid := r.iterator.Current() - r.onlyPos = sort.SearchStrings(r.only, string(docIDInternal)) + r.onlyPos = sort.SearchStrings(r.only, string(docID)) if r.onlyMode { - var rv InternalId + var rv index.IndexInternalID for valid && r.onlyPos < len(r.only) { br, err := NewBackIndexRowKV(key, val) if err != nil { @@ -248,7 +246,7 @@ func (r *UpsideDownCouchDocIDReader) Advance(docID index.IndexInternalID) (index r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) continue } else { - rv = InternalId(br.doc) + rv = br.doc break } } @@ -265,7 +263,7 @@ func (r *UpsideDownCouchDocIDReader) Advance(docID index.IndexInternalID) (index if err != nil { return nil, err } - rv := InternalId(br.doc) + rv := br.doc r.iterator.Next() return rv, nil } diff --git a/index/upside_down/reader_test.go b/index/upside_down/reader_test.go index 0a47e853..0e27e78f 100644 --- a/index/upside_down/reader_test.go +++ b/index/upside_down/reader_test.go @@ -111,7 +111,7 @@ func TestIndexReader(t *testing.T) { } expectedMatch := &index.TermFieldDoc{ - ID: InternalId("2"), + ID: index.IndexInternalID("2"), Freq: 1, Norm: 0.5773502588272095, Vectors: []*index.TermFieldVector{ @@ -145,17 +145,17 @@ func TestIndexReader(t *testing.T) { t.Errorf("Error accessing term field reader: %v", err) } - match, err = reader.Advance(InternalId("2"), nil) + match, err = reader.Advance(index.IndexInternalID("2"), nil) if err != nil { t.Errorf("unexpected error: %v", err) } if match == nil { t.Fatalf("Expected match, got nil") } - if !match.ID.Equals(InternalId("2")) { + if !match.ID.Equals(index.IndexInternalID("2")) { t.Errorf("Expected ID '2', got '%s'", match.ID) } - match, err = reader.Advance(InternalId("3"), nil) + match, err = reader.Advance(index.IndexInternalID("3"), nil) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -183,7 +183,7 @@ func TestIndexReader(t *testing.T) { if match != nil { t.Errorf("expected nil, got %v", match) } - match, err = reader.Advance(InternalId("anywhere"), nil) + match, err = reader.Advance(index.IndexInternalID("anywhere"), nil) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -280,15 +280,15 @@ func TestIndexDocIdReader(t *testing.T) { } }() - id, err = reader2.Advance(InternalId("2")) + id, err = reader2.Advance(index.IndexInternalID("2")) if err != nil { t.Error(err) } - if !id.Equals(InternalId("2")) { + if !id.Equals(index.IndexInternalID("2")) { t.Errorf("expected to find id '2', got '%s'", id) } - id, err = reader2.Advance(InternalId("3")) + id, err = reader2.Advance(index.IndexInternalID("3")) if err != nil { t.Error(err) } diff --git a/index/upside_down/upside_down.go b/index/upside_down/upside_down.go index 2bde3e4f..7c67e157 100644 --- a/index/upside_down/upside_down.go +++ b/index/upside_down/upside_down.go @@ -439,7 +439,7 @@ func (udc *UpsideDownCouch) Update(doc *document.Document) (err error) { // first we lookup the backindex row for the doc id if it exists // lookup the back index row var backIndexRow *BackIndexRow - backIndexRow, err = udc.backIndexRowForDoc(kvreader, InternalId(doc.ID)) + backIndexRow, err = udc.backIndexRowForDoc(kvreader, index.IndexInternalID(doc.ID)) if err != nil { _ = kvreader.Close() atomic.AddUint64(&udc.stats.errors, 1) @@ -627,7 +627,7 @@ func (udc *UpsideDownCouch) Delete(id string) (err error) { // first we lookup the backindex row for the doc id if it exists // lookup the back index row var backIndexRow *BackIndexRow - backIndexRow, err = udc.backIndexRowForDoc(kvreader, InternalId(id)) + backIndexRow, err = udc.backIndexRowForDoc(kvreader, index.IndexInternalID(id)) if err != nil { _ = kvreader.Close() atomic.AddUint64(&udc.stats.errors, 1) @@ -695,7 +695,7 @@ func (udc *UpsideDownCouch) deleteSingle(id string, backIndexRow *BackIndexRow, return deleteRows } -func (udc *UpsideDownCouch) backIndexRowForDoc(kvreader store.KVReader, docID InternalId) (*BackIndexRow, error) { +func (udc *UpsideDownCouch) backIndexRowForDoc(kvreader store.KVReader, docID index.IndexInternalID) (*BackIndexRow, error) { // use a temporary row structure to build key tempRow := &BackIndexRow{ doc: docID, @@ -833,7 +833,7 @@ func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) { } for docID, doc := range batch.IndexOps { - backIndexRow, err := udc.backIndexRowForDoc(kvreader, InternalId(docID)) + backIndexRow, err := udc.backIndexRowForDoc(kvreader, index.IndexInternalID(docID)) if err != nil { docBackIndexRowErr = err return diff --git a/index/upside_down/upside_down_test.go b/index/upside_down/upside_down_test.go index 45b7a6f5..099a2b0c 100644 --- a/index/upside_down/upside_down_test.go +++ b/index/upside_down/upside_down_test.go @@ -663,16 +663,16 @@ func TestIndexBatch(t *testing.T) { if err != nil { t.Error(err) } - docIds := make([]InternalId, 0) + docIds := make([]index.IndexInternalID, 0) docID, err := docIDReader.Next() for docID != nil && err == nil { - docIds = append(docIds, docID.(InternalId)) + docIds = append(docIds, docID) docID, err = docIDReader.Next() } if err != nil { t.Error(err) } - expectedDocIds := []InternalId{InternalId("2"), InternalId("3")} + expectedDocIds := []index.IndexInternalID{index.IndexInternalID("2"), index.IndexInternalID("3")} if !reflect.DeepEqual(docIds, expectedDocIds) { t.Errorf("expected ids: %v, got ids: %v", expectedDocIds, docIds) } @@ -1126,7 +1126,7 @@ func TestIndexTermReaderCompositeFields(t *testing.T) { tfd, err := termFieldReader.Next(nil) for tfd != nil && err == nil { - if !tfd.ID.Equals(InternalId("1")) { + if !tfd.ID.Equals(index.IndexInternalID("1")) { t.Errorf("expected to find document id 1") } tfd, err = termFieldReader.Next(nil) @@ -1179,7 +1179,7 @@ func TestIndexDocumentFieldTerms(t *testing.T) { } }() - fieldTerms, err := indexReader.DocumentFieldTerms(InternalId("1")) + fieldTerms, err := indexReader.DocumentFieldTerms(index.IndexInternalID("1")) if err != nil { t.Error(err) } diff --git a/search/collectors/bench_test.go b/search/collectors/bench_test.go index 2ae7e02c..8516b9f8 100644 --- a/search/collectors/bench_test.go +++ b/search/collectors/bench_test.go @@ -5,6 +5,7 @@ import ( "strconv" "testing" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" "golang.org/x/net/context" ) @@ -13,7 +14,7 @@ func benchHelper(numOfMatches int, collector search.Collector, b *testing.B) { matches := make([]*search.DocumentMatchInternal, 0, numOfMatches) for i := 0; i < numOfMatches; i++ { matches = append(matches, &search.DocumentMatchInternal{ - ID: testInternalId(strconv.Itoa(i)), + ID: index.IndexInternalID(strconv.Itoa(i)), Score: rand.Float64(), }) } diff --git a/search/collectors/collector_top_score_test.go b/search/collectors/collector_top_score_test.go index 4498dbc8..7a6997ee 100644 --- a/search/collectors/collector_top_score_test.go +++ b/search/collectors/collector_top_score_test.go @@ -14,6 +14,7 @@ import ( "golang.org/x/net/context" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -25,59 +26,59 @@ func TestTop10Scores(t *testing.T) { searcher := &stubSearcher{ matches: []*search.DocumentMatchInternal{ &search.DocumentMatchInternal{ - ID: testInternalId("a"), + ID: index.IndexInternalID("a"), Score: 11, }, &search.DocumentMatchInternal{ - ID: testInternalId("b"), + ID: index.IndexInternalID("b"), Score: 9, }, &search.DocumentMatchInternal{ - ID: testInternalId("c"), + ID: index.IndexInternalID("c"), Score: 11, }, &search.DocumentMatchInternal{ - ID: testInternalId("d"), + ID: index.IndexInternalID("d"), Score: 9, }, &search.DocumentMatchInternal{ - ID: testInternalId("e"), + ID: index.IndexInternalID("e"), Score: 11, }, &search.DocumentMatchInternal{ - ID: testInternalId("f"), + ID: index.IndexInternalID("f"), Score: 9, }, &search.DocumentMatchInternal{ - ID: testInternalId("g"), + ID: index.IndexInternalID("g"), Score: 11, }, &search.DocumentMatchInternal{ - ID: testInternalId("h"), + ID: index.IndexInternalID("h"), Score: 9, }, &search.DocumentMatchInternal{ - ID: testInternalId("i"), + ID: index.IndexInternalID("i"), Score: 11, }, &search.DocumentMatchInternal{ - ID: testInternalId("j"), + ID: index.IndexInternalID("j"), Score: 11, }, &search.DocumentMatchInternal{ - ID: testInternalId("k"), + ID: index.IndexInternalID("k"), Score: 11, }, &search.DocumentMatchInternal{ - ID: testInternalId("l"), + ID: index.IndexInternalID("l"), Score: 99, }, &search.DocumentMatchInternal{ - ID: testInternalId("m"), + ID: index.IndexInternalID("m"), Score: 11, }, &search.DocumentMatchInternal{ - ID: testInternalId("n"), + ID: index.IndexInternalID("n"), Score: 11, }, }, @@ -133,59 +134,59 @@ func TestTop10ScoresSkip10(t *testing.T) { searcher := &stubSearcher{ matches: []*search.DocumentMatchInternal{ &search.DocumentMatchInternal{ - ID: testInternalId("a"), + ID: index.IndexInternalID("a"), Score: 11, }, &search.DocumentMatchInternal{ - ID: testInternalId("b"), + ID: index.IndexInternalID("b"), Score: 9.5, }, &search.DocumentMatchInternal{ - ID: testInternalId("c"), + ID: index.IndexInternalID("c"), Score: 11, }, &search.DocumentMatchInternal{ - ID: testInternalId("d"), + ID: index.IndexInternalID("d"), Score: 9, }, &search.DocumentMatchInternal{ - ID: testInternalId("e"), + ID: index.IndexInternalID("e"), Score: 11, }, &search.DocumentMatchInternal{ - ID: testInternalId("f"), + ID: index.IndexInternalID("f"), Score: 9, }, &search.DocumentMatchInternal{ - ID: testInternalId("g"), + ID: index.IndexInternalID("g"), Score: 11, }, &search.DocumentMatchInternal{ - ID: testInternalId("h"), + ID: index.IndexInternalID("h"), Score: 9, }, &search.DocumentMatchInternal{ - ID: testInternalId("i"), + ID: index.IndexInternalID("i"), Score: 11, }, &search.DocumentMatchInternal{ - ID: testInternalId("j"), + ID: index.IndexInternalID("j"), Score: 11, }, &search.DocumentMatchInternal{ - ID: testInternalId("k"), + ID: index.IndexInternalID("k"), Score: 11, }, &search.DocumentMatchInternal{ - ID: testInternalId("l"), + ID: index.IndexInternalID("l"), Score: 99, }, &search.DocumentMatchInternal{ - ID: testInternalId("m"), + ID: index.IndexInternalID("m"), Score: 11, }, &search.DocumentMatchInternal{ - ID: testInternalId("n"), + ID: index.IndexInternalID("n"), Score: 11, }, }, @@ -229,59 +230,59 @@ func TestPaginationSameScores(t *testing.T) { searcher := &stubSearcher{ matches: []*search.DocumentMatchInternal{ &search.DocumentMatchInternal{ - ID: testInternalId("a"), + ID: index.IndexInternalID("a"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("b"), + ID: index.IndexInternalID("b"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("c"), + ID: index.IndexInternalID("c"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("d"), + ID: index.IndexInternalID("d"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("e"), + ID: index.IndexInternalID("e"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("f"), + ID: index.IndexInternalID("f"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("g"), + ID: index.IndexInternalID("g"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("h"), + ID: index.IndexInternalID("h"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("i"), + ID: index.IndexInternalID("i"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("j"), + ID: index.IndexInternalID("j"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("k"), + ID: index.IndexInternalID("k"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("l"), + ID: index.IndexInternalID("l"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("m"), + ID: index.IndexInternalID("m"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("n"), + ID: index.IndexInternalID("n"), Score: 5, }, }, @@ -315,59 +316,59 @@ func TestPaginationSameScores(t *testing.T) { searcher = &stubSearcher{ matches: []*search.DocumentMatchInternal{ &search.DocumentMatchInternal{ - ID: testInternalId("a"), + ID: index.IndexInternalID("a"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("b"), + ID: index.IndexInternalID("b"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("c"), + ID: index.IndexInternalID("c"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("d"), + ID: index.IndexInternalID("d"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("e"), + ID: index.IndexInternalID("e"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("f"), + ID: index.IndexInternalID("f"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("g"), + ID: index.IndexInternalID("g"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("h"), + ID: index.IndexInternalID("h"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("i"), + ID: index.IndexInternalID("i"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("j"), + ID: index.IndexInternalID("j"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("k"), + ID: index.IndexInternalID("k"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("l"), + ID: index.IndexInternalID("l"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("m"), + ID: index.IndexInternalID("m"), Score: 5, }, &search.DocumentMatchInternal{ - ID: testInternalId("n"), + ID: index.IndexInternalID("n"), Score: 5, }, }, diff --git a/search/collectors/search_test.go b/search/collectors/search_test.go index acb58957..40cec0d0 100644 --- a/search/collectors/search_test.go +++ b/search/collectors/search_test.go @@ -10,8 +10,6 @@ package collectors import ( - "bytes" - "github.com/blevesearch/bleve/document" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" @@ -63,16 +61,6 @@ func (ss *stubSearcher) Min() int { return 0 } -type testInternalId []byte - -func (t testInternalId) Compare(other index.IndexInternalID) int { - return bytes.Compare(t, other.(testInternalId)) -} - -func (t testInternalId) Equals(other index.IndexInternalID) bool { - return t.Compare(other.(testInternalId)) == 0 -} - type stubReader struct{} func (sr *stubReader) TermFieldReader(term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (index.TermFieldReader, error) { @@ -124,7 +112,7 @@ func (sr *stubReader) DocCount() uint64 { } func (sr *stubReader) FinalizeDocID(id index.IndexInternalID) (string, error) { - return string(id.(testInternalId)), nil + return string(id), nil } func (sr *stubReader) Close() error { diff --git a/search/scorers/scorer_constant_test.go b/search/scorers/scorer_constant_test.go index 8f50c5d3..9e02d256 100644 --- a/search/scorers/scorer_constant_test.go +++ b/search/scorers/scorer_constant_test.go @@ -28,7 +28,7 @@ func TestConstantScorer(t *testing.T) { // test some simple math { termMatch: &index.TermFieldDoc{ - ID: testInternalId("one"), + ID: index.IndexInternalID("one"), Freq: 1, Norm: 1.0, Vectors: []*index.TermFieldVector{ @@ -41,7 +41,7 @@ func TestConstantScorer(t *testing.T) { }, }, result: &search.DocumentMatchInternal{ - ID: testInternalId("one"), + ID: index.IndexInternalID("one"), Score: 1.0, Expl: &search.Explanation{ Value: 1.0, @@ -72,12 +72,12 @@ func TestConstantScorerWithQueryNorm(t *testing.T) { }{ { termMatch: &index.TermFieldDoc{ - ID: testInternalId("one"), + ID: index.IndexInternalID("one"), Freq: 1, Norm: 1.0, }, result: &search.DocumentMatchInternal{ - ID: testInternalId("one"), + ID: index.IndexInternalID("one"), Score: 2.0, Expl: &search.Explanation{ Value: 2.0, diff --git a/search/scorers/scorer_term_test.go b/search/scorers/scorer_term_test.go index 43950fc6..628e4e8f 100644 --- a/search/scorers/scorer_term_test.go +++ b/search/scorers/scorer_term_test.go @@ -10,7 +10,6 @@ package scorers import ( - "bytes" "math" "reflect" "testing" @@ -19,16 +18,6 @@ import ( "github.com/blevesearch/bleve/search" ) -type testInternalId []byte - -func (t testInternalId) Compare(other index.IndexInternalID) int { - return bytes.Compare(t, other.(testInternalId)) -} - -func (t testInternalId) Equals(other index.IndexInternalID) bool { - return t.Compare(other.(testInternalId)) == 0 -} - func TestTermScorer(t *testing.T) { var docTotal uint64 = 100 @@ -46,7 +35,7 @@ func TestTermScorer(t *testing.T) { // test some simple math { termMatch: &index.TermFieldDoc{ - ID: testInternalId("one"), + ID: index.IndexInternalID("one"), Freq: 1, Norm: 1.0, Vectors: []*index.TermFieldVector{ @@ -59,7 +48,7 @@ func TestTermScorer(t *testing.T) { }, }, result: &search.DocumentMatchInternal{ - ID: testInternalId("one"), + ID: index.IndexInternalID("one"), Score: math.Sqrt(1.0) * idf, Expl: &search.Explanation{ Value: math.Sqrt(1.0) * idf, @@ -95,12 +84,12 @@ func TestTermScorer(t *testing.T) { // test the same thing again (score should be cached this time) { termMatch: &index.TermFieldDoc{ - ID: testInternalId("one"), + ID: index.IndexInternalID("one"), Freq: 1, Norm: 1.0, }, result: &search.DocumentMatchInternal{ - ID: testInternalId("one"), + ID: index.IndexInternalID("one"), Score: math.Sqrt(1.0) * idf, Expl: &search.Explanation{ Value: math.Sqrt(1.0) * idf, @@ -125,12 +114,12 @@ func TestTermScorer(t *testing.T) { // test a case where the sqrt isn't precalculated { termMatch: &index.TermFieldDoc{ - ID: testInternalId("one"), + ID: index.IndexInternalID("one"), Freq: 65, Norm: 1.0, }, result: &search.DocumentMatchInternal{ - ID: testInternalId("one"), + ID: index.IndexInternalID("one"), Score: math.Sqrt(65) * idf, Expl: &search.Explanation{ Value: math.Sqrt(65) * idf, @@ -188,12 +177,12 @@ func TestTermScorerWithQueryNorm(t *testing.T) { }{ { termMatch: &index.TermFieldDoc{ - ID: testInternalId("one"), + ID: index.IndexInternalID("one"), Freq: 1, Norm: 1.0, }, result: &search.DocumentMatchInternal{ - ID: testInternalId("one"), + ID: index.IndexInternalID("one"), Score: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0, Expl: &search.Explanation{ Value: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0, diff --git a/search/searchers/search_boolean_test.go b/search/searchers/search_boolean_test.go index a09da69b..0e957673 100644 --- a/search/searchers/search_boolean_test.go +++ b/search/searchers/search_boolean_test.go @@ -12,7 +12,7 @@ package searchers import ( "testing" - "github.com/blevesearch/bleve/index/upside_down" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -249,15 +249,15 @@ func TestBooleanSearch(t *testing.T) { searcher: booleanSearcher, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("1"), + ID: index.IndexInternalID("1"), Score: 0.9818005051949021, }, { - ID: upside_down.InternalId("3"), + ID: index.IndexInternalID("3"), Score: 0.808709699395535, }, { - ID: upside_down.InternalId("4"), + ID: index.IndexInternalID("4"), Score: 0.34618161159873423, }, }, @@ -266,11 +266,11 @@ func TestBooleanSearch(t *testing.T) { searcher: booleanSearcher2, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("1"), + ID: index.IndexInternalID("1"), Score: 0.6775110856165737, }, { - ID: upside_down.InternalId("3"), + ID: index.IndexInternalID("3"), Score: 0.6775110856165737, }, }, @@ -284,15 +284,15 @@ func TestBooleanSearch(t *testing.T) { searcher: booleanSearcher4, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("1"), + ID: index.IndexInternalID("1"), Score: 1.0, }, { - ID: upside_down.InternalId("3"), + ID: index.IndexInternalID("3"), Score: 0.5, }, { - ID: upside_down.InternalId("4"), + ID: index.IndexInternalID("4"), Score: 1.0, }, }, @@ -301,11 +301,11 @@ func TestBooleanSearch(t *testing.T) { searcher: booleanSearcher5, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("3"), + ID: index.IndexInternalID("3"), Score: 0.5, }, { - ID: upside_down.InternalId("4"), + ID: index.IndexInternalID("4"), Score: 1.0, }, }, @@ -319,7 +319,7 @@ func TestBooleanSearch(t *testing.T) { searcher: conjunctionSearcher7, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("1"), + ID: index.IndexInternalID("1"), Score: 2.0097428702814377, }, }, @@ -328,7 +328,7 @@ func TestBooleanSearch(t *testing.T) { searcher: conjunctionSearcher8, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("3"), + ID: index.IndexInternalID("3"), Score: 2.0681575785068107, }, }, diff --git a/search/searchers/search_conjunction_test.go b/search/searchers/search_conjunction_test.go index f4baa19d..7d32547b 100644 --- a/search/searchers/search_conjunction_test.go +++ b/search/searchers/search_conjunction_test.go @@ -12,7 +12,7 @@ package searchers import ( "testing" - "github.com/blevesearch/bleve/index/upside_down" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -129,7 +129,7 @@ func TestConjunctionSearch(t *testing.T) { searcher: beerAndMartySearcher, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("1"), + ID: index.IndexInternalID("1"), Score: 2.0097428702814377, }, }, @@ -138,7 +138,7 @@ func TestConjunctionSearch(t *testing.T) { searcher: angstAndBeerSearcher, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("2"), + ID: index.IndexInternalID("2"), Score: 1.0807601687084403, }, }, @@ -151,11 +151,11 @@ func TestConjunctionSearch(t *testing.T) { searcher: beerAndMisterSearcher, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("2"), + ID: index.IndexInternalID("2"), Score: 1.2877980334016337, }, { - ID: upside_down.InternalId("3"), + ID: index.IndexInternalID("3"), Score: 1.2877980334016337, }, }, @@ -164,7 +164,7 @@ func TestConjunctionSearch(t *testing.T) { searcher: couchbaseAndMisterSearcher, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("2"), + ID: index.IndexInternalID("2"), Score: 1.4436599157093672, }, }, @@ -173,7 +173,7 @@ func TestConjunctionSearch(t *testing.T) { searcher: beerAndCouchbaseAndMisterSearcher, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("2"), + ID: index.IndexInternalID("2"), Score: 1.441614953806971, }, }, diff --git a/search/searchers/search_disjunction_test.go b/search/searchers/search_disjunction_test.go index 231bfb0e..bc9d6afa 100644 --- a/search/searchers/search_disjunction_test.go +++ b/search/searchers/search_disjunction_test.go @@ -12,7 +12,7 @@ package searchers import ( "testing" - "github.com/blevesearch/bleve/index/upside_down" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -72,11 +72,11 @@ func TestDisjunctionSearch(t *testing.T) { searcher: martyOrDustinSearcher, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("1"), + ID: index.IndexInternalID("1"), Score: 0.6775110856165737, }, { - ID: upside_down.InternalId("3"), + ID: index.IndexInternalID("3"), Score: 0.6775110856165737, }, }, @@ -86,15 +86,15 @@ func TestDisjunctionSearch(t *testing.T) { searcher: nestedRaviOrMartyOrDustinSearcher, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("1"), + ID: index.IndexInternalID("1"), Score: 0.2765927424732821, }, { - ID: upside_down.InternalId("3"), + ID: index.IndexInternalID("3"), Score: 0.2765927424732821, }, { - ID: upside_down.InternalId("4"), + ID: index.IndexInternalID("4"), Score: 0.5531854849465642, }, }, @@ -159,7 +159,7 @@ func TestDisjunctionAdvance(t *testing.T) { t.Fatal(err) } - match, err := martyOrDustinSearcher.Advance(upside_down.InternalId("3"), nil) + match, err := martyOrDustinSearcher.Advance(index.IndexInternalID("3"), nil) if err != nil { t.Errorf("unexpected error: %v", err) } diff --git a/search/searchers/search_docid_test.go b/search/searchers/search_docid_test.go index 271a25bc..4bd8c43c 100644 --- a/search/searchers/search_docid_test.go +++ b/search/searchers/search_docid_test.go @@ -68,7 +68,7 @@ func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { if err != nil { t.Fatal(err) } - if !upside_down.InternalId(id).Equals(m.ID) { + if !index.IndexInternalID(id).Equals(m.ID) { t.Fatalf("expected %v at position %v, got %v", id, i, m.ID) } } @@ -87,18 +87,18 @@ func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { } before := id[:1] for _, target := range []string{before, id} { - m, err := searcher.Advance(upside_down.InternalId(target), nil) + m, err := searcher.Advance(index.IndexInternalID(target), nil) if err != nil { t.Fatal(err) } - if m == nil || !m.ID.Equals(upside_down.InternalId(id)) { + if m == nil || !m.ID.Equals(index.IndexInternalID(id)) { t.Fatalf("advancing to %v returned %v instead of %v", before, m, id) } } } // Seek after the end of the sequence after := "zzz" - m, err = searcher.Advance(upside_down.InternalId(after), nil) + m, err = searcher.Advance(index.IndexInternalID(after), nil) if err != nil { t.Fatal(err) } diff --git a/search/searchers/search_fuzzy_test.go b/search/searchers/search_fuzzy_test.go index 52638044..461d8e31 100644 --- a/search/searchers/search_fuzzy_test.go +++ b/search/searchers/search_fuzzy_test.go @@ -12,7 +12,7 @@ package searchers import ( "testing" - "github.com/blevesearch/bleve/index/upside_down" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -57,19 +57,19 @@ func TestFuzzySearch(t *testing.T) { searcher: fuzzySearcherbeet, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("1"), + ID: index.IndexInternalID("1"), Score: 1.0, }, { - ID: upside_down.InternalId("2"), + ID: index.IndexInternalID("2"), Score: 0.5, }, { - ID: upside_down.InternalId("3"), + ID: index.IndexInternalID("3"), Score: 0.5, }, { - ID: upside_down.InternalId("4"), + ID: index.IndexInternalID("4"), Score: 0.9999999838027345, }, }, @@ -82,7 +82,7 @@ func TestFuzzySearch(t *testing.T) { searcher: fuzzySearcheraplee, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("3"), + ID: index.IndexInternalID("3"), Score: 0.9581453659370776, }, }, @@ -91,7 +91,7 @@ func TestFuzzySearch(t *testing.T) { searcher: fuzzySearcherprefix, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("5"), + ID: index.IndexInternalID("5"), Score: 1.916290731874155, }, }, diff --git a/search/searchers/search_match_all_test.go b/search/searchers/search_match_all_test.go index 32b85514..cba20d15 100644 --- a/search/searchers/search_match_all_test.go +++ b/search/searchers/search_match_all_test.go @@ -12,7 +12,7 @@ package searchers import ( "testing" - "github.com/blevesearch/bleve/index/upside_down" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -49,23 +49,23 @@ func TestMatchAllSearch(t *testing.T) { queryNorm: 1.0, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("1"), + ID: index.IndexInternalID("1"), Score: 1.0, }, { - ID: upside_down.InternalId("2"), + ID: index.IndexInternalID("2"), Score: 1.0, }, { - ID: upside_down.InternalId("3"), + ID: index.IndexInternalID("3"), Score: 1.0, }, { - ID: upside_down.InternalId("4"), + ID: index.IndexInternalID("4"), Score: 1.0, }, { - ID: upside_down.InternalId("5"), + ID: index.IndexInternalID("5"), Score: 1.0, }, }, @@ -75,23 +75,23 @@ func TestMatchAllSearch(t *testing.T) { queryNorm: 0.8333333, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("1"), + ID: index.IndexInternalID("1"), Score: 1.0, }, { - ID: upside_down.InternalId("2"), + ID: index.IndexInternalID("2"), Score: 1.0, }, { - ID: upside_down.InternalId("3"), + ID: index.IndexInternalID("3"), Score: 1.0, }, { - ID: upside_down.InternalId("4"), + ID: index.IndexInternalID("4"), Score: 1.0, }, { - ID: upside_down.InternalId("5"), + ID: index.IndexInternalID("5"), Score: 1.0, }, }, diff --git a/search/searchers/search_match_none_test.go b/search/searchers/search_match_none_test.go index 2752f998..dd730e38 100644 --- a/search/searchers/search_match_none_test.go +++ b/search/searchers/search_match_none_test.go @@ -55,7 +55,7 @@ func TestMatchNoneSearch(t *testing.T) { i := 0 for err == nil && next != nil { if i < len(test.results) { - if next.ID != test.results[i].ID { + if !next.ID.Equals(test.results[i].ID) { t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { diff --git a/search/searchers/search_phrase_test.go b/search/searchers/search_phrase_test.go index e9fcdb55..2386d975 100644 --- a/search/searchers/search_phrase_test.go +++ b/search/searchers/search_phrase_test.go @@ -12,7 +12,7 @@ package searchers import ( "testing" - "github.com/blevesearch/bleve/index/upside_down" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -54,7 +54,7 @@ func TestPhraseSearch(t *testing.T) { searcher: phraseSearcher, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("2"), + ID: index.IndexInternalID("2"), Score: 1.0807601687084403, }, }, diff --git a/search/searchers/search_regexp_test.go b/search/searchers/search_regexp_test.go index 0552866f..f1ce7c0e 100644 --- a/search/searchers/search_regexp_test.go +++ b/search/searchers/search_regexp_test.go @@ -13,7 +13,7 @@ import ( "regexp" "testing" - "github.com/blevesearch/bleve/index/upside_down" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -58,7 +58,7 @@ func TestRegexpSearch(t *testing.T) { searcher: regexpSearcher, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("1"), + ID: index.IndexInternalID("1"), Score: 1.916290731874155, }, }, @@ -67,11 +67,11 @@ func TestRegexpSearch(t *testing.T) { searcher: regexpSearcherCo, results: []*search.DocumentMatchInternal{ { - ID: upside_down.InternalId("2"), + ID: index.IndexInternalID("2"), Score: 0.33875554280828685, }, { - ID: upside_down.InternalId("3"), + ID: index.IndexInternalID("3"), Score: 0.33875554280828685, }, }, diff --git a/search/searchers/search_term_test.go b/search/searchers/search_term_test.go index 83381e43..d46274f6 100644 --- a/search/searchers/search_term_test.go +++ b/search/searchers/search_term_test.go @@ -167,19 +167,19 @@ func TestTermSearcher(t *testing.T) { if err != nil { t.Errorf("expected result, got %v", err) } - if !docMatch.ID.Equals(upside_down.InternalId("a")) { + if !docMatch.ID.Equals(index.IndexInternalID("a")) { t.Errorf("expected result ID to be 'a', got '%s", docMatch.ID) } - docMatch, err = searcher.Advance(upside_down.InternalId("c"), nil) + docMatch, err = searcher.Advance(index.IndexInternalID("c"), nil) if err != nil { t.Errorf("expected result, got %v", err) } - if !docMatch.ID.Equals(upside_down.InternalId("c")) { + if !docMatch.ID.Equals(index.IndexInternalID("c")) { t.Errorf("expected result ID to be 'c' got '%s'", docMatch.ID) } // try advancing past end - docMatch, err = searcher.Advance(upside_down.InternalId("z"), nil) + docMatch, err = searcher.Advance(index.IndexInternalID("z"), nil) if err != nil { t.Fatal(err) } From e188fe35f7ae334140c3ed2c51e1079066689665 Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Mon, 1 Aug 2016 14:58:02 -0400 Subject: [PATCH 04/13] switch back to single DocumentMatch struct instead of separate DocumentMatch/DocumentMatchInternal rules are simple, everything operates on the IndexInternalID field until the results are returned, then ID is set correctly the IndexInternalID field is not exported to JSON --- search/collectors/bench_test.go | 8 +- search/collectors/collector_top_score.go | 17 +- search/collectors/collector_top_score_test.go | 344 +++++++++--------- search/collectors/search_test.go | 8 +- search/facets_builder.go | 4 +- search/scorers/scorer_conjunction.go | 6 +- search/scorers/scorer_constant.go | 8 +- search/scorers/scorer_constant_test.go | 16 +- search/scorers/scorer_disjunction.go | 6 +- search/scorers/scorer_term.go | 6 +- search/scorers/scorer_term_test.go | 28 +- search/search.go | 51 +-- search/searchers/search_boolean.go | 52 +-- search/searchers/search_boolean_test.go | 70 ++-- search/searchers/search_conjunction.go | 24 +- search/searchers/search_conjunction_test.go | 42 +-- search/searchers/search_disjunction.go | 22 +- search/searchers/search_disjunction_test.go | 30 +- search/searchers/search_docid.go | 4 +- search/searchers/search_docid_test.go | 8 +- search/searchers/search_fuzzy.go | 4 +- search/searchers/search_fuzzy_test.go | 38 +- search/searchers/search_match_all.go | 4 +- search/searchers/search_match_all_test.go | 50 +-- search/searchers/search_match_none.go | 4 +- search/searchers/search_match_none_test.go | 8 +- search/searchers/search_numeric_range.go | 4 +- search/searchers/search_phrase.go | 8 +- search/searchers/search_phrase_test.go | 12 +- search/searchers/search_regexp.go | 4 +- search/searchers/search_regexp_test.go | 22 +- search/searchers/search_term.go | 4 +- search/searchers/search_term_prefix.go | 4 +- search/searchers/search_term_test.go | 8 +- 34 files changed, 448 insertions(+), 480 deletions(-) diff --git a/search/collectors/bench_test.go b/search/collectors/bench_test.go index 8516b9f8..cddf5e7f 100644 --- a/search/collectors/bench_test.go +++ b/search/collectors/bench_test.go @@ -11,11 +11,11 @@ import ( ) func benchHelper(numOfMatches int, collector search.Collector, b *testing.B) { - matches := make([]*search.DocumentMatchInternal, 0, numOfMatches) + matches := make([]*search.DocumentMatch, 0, numOfMatches) for i := 0; i < numOfMatches; i++ { - matches = append(matches, &search.DocumentMatchInternal{ - ID: index.IndexInternalID(strconv.Itoa(i)), - Score: rand.Float64(), + matches = append(matches, &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID(strconv.Itoa(i)), + Score: rand.Float64(), }) } diff --git a/search/collectors/collector_top_score.go b/search/collectors/collector_top_score.go index b6e44f03..a78eb2d5 100644 --- a/search/collectors/collector_top_score.go +++ b/search/collectors/collector_top_score.go @@ -64,8 +64,8 @@ var COLLECT_CHECK_DONE_EVERY = uint64(1024) func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Searcher, reader index.IndexReader) error { startTime := time.Now() var err error - var pre search.DocumentMatchInternal // A single pre-alloc'ed, reused instance. - var next *search.DocumentMatchInternal + var pre search.DocumentMatch // A single pre-alloc'ed, reused instance. + var next *search.DocumentMatch select { case <-ctx.Done(): return ctx.Err() @@ -103,7 +103,7 @@ func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Sear return nil } -func (tksc *TopScoreCollector) collectSingle(dmIn *search.DocumentMatchInternal) { +func (tksc *TopScoreCollector) collectSingle(dmIn *search.DocumentMatch) { // increment total hits tksc.total++ @@ -119,18 +119,18 @@ func (tksc *TopScoreCollector) collectSingle(dmIn *search.DocumentMatchInternal) // Because the dmIn will be the single, pre-allocated, reused // instance, we need to copy the dmIn into a new, standalone // instance before inserting into our candidate results list. - dm := &search.DocumentMatchInternal{} + dm := &search.DocumentMatch{} *dm = *dmIn for e := tksc.results.Front(); e != nil; e = e.Next() { - curr := e.Value.(*search.DocumentMatchInternal) + curr := e.Value.(*search.DocumentMatch) if dm.Score <= curr.Score { tksc.results.InsertBefore(dm, e) // if we just made the list too long if tksc.results.Len() > (tksc.k + tksc.skip) { // remove the head - tksc.minScore = tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatchInternal).Score + tksc.minScore = tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch).Score } return } @@ -139,7 +139,7 @@ func (tksc *TopScoreCollector) collectSingle(dmIn *search.DocumentMatchInternal) tksc.results.PushBack(dm) if tksc.results.Len() > (tksc.k + tksc.skip) { // remove the head - tksc.minScore = tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatchInternal).Score + tksc.minScore = tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch).Score } } @@ -158,7 +158,8 @@ func (tksc *TopScoreCollector) finalizeResults(r index.IndexReader) (search.Docu continue } var err error - rv[i], err = e.Value.(*search.DocumentMatchInternal).Finalize(r) + rv[i] = e.Value.(*search.DocumentMatch) + rv[i].ID, err = r.FinalizeDocID(rv[i].IndexInternalID) if err != nil { return nil, err } diff --git a/search/collectors/collector_top_score_test.go b/search/collectors/collector_top_score_test.go index 7a6997ee..66c8978c 100644 --- a/search/collectors/collector_top_score_test.go +++ b/search/collectors/collector_top_score_test.go @@ -24,62 +24,62 @@ func TestTop10Scores(t *testing.T) { // the top-10 scores are > 10 // everything else is less than 10 searcher := &stubSearcher{ - matches: []*search.DocumentMatchInternal{ - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("a"), - Score: 11, + matches: []*search.DocumentMatch{ + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("a"), + Score: 11, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("b"), - Score: 9, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("b"), + Score: 9, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("c"), - Score: 11, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("c"), + Score: 11, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("d"), - Score: 9, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("d"), + Score: 9, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("e"), - Score: 11, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("e"), + Score: 11, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("f"), - Score: 9, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("f"), + Score: 9, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("g"), - Score: 11, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("g"), + Score: 11, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("h"), - Score: 9, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("h"), + Score: 9, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("i"), - Score: 11, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("i"), + Score: 11, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("j"), - Score: 11, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("j"), + Score: 11, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("k"), - Score: 11, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("k"), + Score: 11, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("l"), - Score: 99, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("l"), + Score: 99, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("m"), - Score: 11, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("m"), + Score: 11, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("n"), - Score: 11, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("n"), + Score: 11, }, }, } @@ -132,62 +132,62 @@ func TestTop10ScoresSkip10(t *testing.T) { // the top-10 scores are > 10 // everything else is less than 10 searcher := &stubSearcher{ - matches: []*search.DocumentMatchInternal{ - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("a"), - Score: 11, + matches: []*search.DocumentMatch{ + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("a"), + Score: 11, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("b"), - Score: 9.5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("b"), + Score: 9.5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("c"), - Score: 11, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("c"), + Score: 11, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("d"), - Score: 9, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("d"), + Score: 9, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("e"), - Score: 11, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("e"), + Score: 11, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("f"), - Score: 9, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("f"), + Score: 9, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("g"), - Score: 11, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("g"), + Score: 11, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("h"), - Score: 9, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("h"), + Score: 9, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("i"), - Score: 11, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("i"), + Score: 11, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("j"), - Score: 11, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("j"), + Score: 11, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("k"), - Score: 11, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("k"), + Score: 11, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("l"), - Score: 99, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("l"), + Score: 99, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("m"), - Score: 11, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("m"), + Score: 11, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("n"), - Score: 11, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("n"), + Score: 11, }, }, } @@ -228,62 +228,62 @@ func TestPaginationSameScores(t *testing.T) { // a stub search with more than 10 matches // all documents have the same score searcher := &stubSearcher{ - matches: []*search.DocumentMatchInternal{ - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("a"), - Score: 5, + matches: []*search.DocumentMatch{ + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("a"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("b"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("b"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("c"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("c"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("d"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("d"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("e"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("e"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("f"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("f"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("g"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("g"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("h"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("h"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("i"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("i"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("j"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("j"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("k"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("k"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("l"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("l"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("m"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("m"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("n"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("n"), + Score: 5, }, }, } @@ -314,62 +314,62 @@ func TestPaginationSameScores(t *testing.T) { // a stub search with more than 10 matches // all documents have the same score searcher = &stubSearcher{ - matches: []*search.DocumentMatchInternal{ - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("a"), - Score: 5, + matches: []*search.DocumentMatch{ + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("a"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("b"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("b"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("c"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("c"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("d"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("d"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("e"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("e"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("f"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("f"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("g"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("g"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("h"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("h"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("i"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("i"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("j"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("j"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("k"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("k"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("l"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("l"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("m"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("m"), + Score: 5, }, - &search.DocumentMatchInternal{ - ID: index.IndexInternalID("n"), - Score: 5, + &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("n"), + Score: 5, }, }, } diff --git a/search/collectors/search_test.go b/search/collectors/search_test.go index 40cec0d0..0f803f1a 100644 --- a/search/collectors/search_test.go +++ b/search/collectors/search_test.go @@ -17,10 +17,10 @@ import ( type stubSearcher struct { index int - matches []*search.DocumentMatchInternal + matches []*search.DocumentMatch } -func (ss *stubSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (ss *stubSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { if ss.index < len(ss.matches) { rv := ss.matches[ss.index] ss.index++ @@ -29,9 +29,9 @@ func (ss *stubSearcher) Next(preAllocated *search.DocumentMatchInternal) (*searc return nil, nil } -func (ss *stubSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (ss *stubSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - for ss.index < len(ss.matches) && ss.matches[ss.index].ID.Compare(ID) < 0 { + for ss.index < len(ss.matches) && ss.matches[ss.index].IndexInternalID.Compare(ID) < 0 { ss.index++ } if ss.index < len(ss.matches) { diff --git a/search/facets_builder.go b/search/facets_builder.go index c0c203cc..55cb33ab 100644 --- a/search/facets_builder.go +++ b/search/facets_builder.go @@ -37,12 +37,12 @@ func (fb *FacetsBuilder) Add(name string, facetBuilder FacetBuilder) { fb.facets[name] = facetBuilder } -func (fb *FacetsBuilder) Update(docMatch *DocumentMatchInternal) error { +func (fb *FacetsBuilder) Update(docMatch *DocumentMatch) error { var fields []string for _, facetBuilder := range fb.facets { fields = append(fields, facetBuilder.Field()) } - fieldTerms, err := fb.indexReader.DocumentFieldTermsForFields(docMatch.ID, fields) + fieldTerms, err := fb.indexReader.DocumentFieldTermsForFields(docMatch.IndexInternalID, fields) if err != nil { return err } diff --git a/search/scorers/scorer_conjunction.go b/search/scorers/scorer_conjunction.go index 94a5eb62..b2c34ba0 100644 --- a/search/scorers/scorer_conjunction.go +++ b/search/scorers/scorer_conjunction.go @@ -23,9 +23,9 @@ func NewConjunctionQueryScorer(explain bool) *ConjunctionQueryScorer { } } -func (s *ConjunctionQueryScorer) Score(constituents []*search.DocumentMatchInternal) *search.DocumentMatchInternal { - rv := search.DocumentMatchInternal{ - ID: constituents[0].ID, +func (s *ConjunctionQueryScorer) Score(constituents []*search.DocumentMatch) *search.DocumentMatch { + rv := search.DocumentMatch{ + IndexInternalID: constituents[0].IndexInternalID, } var sum float64 diff --git a/search/scorers/scorer_constant.go b/search/scorers/scorer_constant.go index 86717e9f..af4eebf2 100644 --- a/search/scorers/scorer_constant.go +++ b/search/scorers/scorer_constant.go @@ -65,7 +65,7 @@ func (s *ConstantScorer) SetQueryNorm(qnorm float64) { } } -func (s *ConstantScorer) Score(id index.IndexInternalID) *search.DocumentMatchInternal { +func (s *ConstantScorer) Score(id index.IndexInternalID) *search.DocumentMatch { var scoreExplanation *search.Explanation score := s.constant @@ -92,9 +92,9 @@ func (s *ConstantScorer) Score(id index.IndexInternalID) *search.DocumentMatchIn } } - rv := search.DocumentMatchInternal{ - ID: id, - Score: score, + rv := search.DocumentMatch{ + IndexInternalID: id, + Score: score, } if s.explain { rv.Expl = scoreExplanation diff --git a/search/scorers/scorer_constant_test.go b/search/scorers/scorer_constant_test.go index 9e02d256..d701217b 100644 --- a/search/scorers/scorer_constant_test.go +++ b/search/scorers/scorer_constant_test.go @@ -23,7 +23,7 @@ func TestConstantScorer(t *testing.T) { tests := []struct { termMatch *index.TermFieldDoc - result *search.DocumentMatchInternal + result *search.DocumentMatch }{ // test some simple math { @@ -40,9 +40,9 @@ func TestConstantScorer(t *testing.T) { }, }, }, - result: &search.DocumentMatchInternal{ - ID: index.IndexInternalID("one"), - Score: 1.0, + result: &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("one"), + Score: 1.0, Expl: &search.Explanation{ Value: 1.0, Message: "ConstantScore()", @@ -68,7 +68,7 @@ func TestConstantScorerWithQueryNorm(t *testing.T) { tests := []struct { termMatch *index.TermFieldDoc - result *search.DocumentMatchInternal + result *search.DocumentMatch }{ { termMatch: &index.TermFieldDoc{ @@ -76,9 +76,9 @@ func TestConstantScorerWithQueryNorm(t *testing.T) { Freq: 1, Norm: 1.0, }, - result: &search.DocumentMatchInternal{ - ID: index.IndexInternalID("one"), - Score: 2.0, + result: &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("one"), + Score: 2.0, Expl: &search.Explanation{ Value: 2.0, Message: "weight(^1.000000), product of:", diff --git a/search/scorers/scorer_disjunction.go b/search/scorers/scorer_disjunction.go index 0e5dc3e5..e73aa578 100644 --- a/search/scorers/scorer_disjunction.go +++ b/search/scorers/scorer_disjunction.go @@ -25,9 +25,9 @@ func NewDisjunctionQueryScorer(explain bool) *DisjunctionQueryScorer { } } -func (s *DisjunctionQueryScorer) Score(constituents []*search.DocumentMatchInternal, countMatch, countTotal int) *search.DocumentMatchInternal { - rv := search.DocumentMatchInternal{ - ID: constituents[0].ID, +func (s *DisjunctionQueryScorer) Score(constituents []*search.DocumentMatch, countMatch, countTotal int) *search.DocumentMatch { + rv := search.DocumentMatch{ + IndexInternalID: constituents[0].IndexInternalID, } var sum float64 diff --git a/search/scorers/scorer_term.go b/search/scorers/scorer_term.go index ee6b6a9b..458c9312 100644 --- a/search/scorers/scorer_term.go +++ b/search/scorers/scorer_term.go @@ -83,7 +83,7 @@ func (s *TermQueryScorer) SetQueryNorm(qnorm float64) { } } -func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc, preAllocated *search.DocumentMatchInternal) *search.DocumentMatchInternal { +func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc, preAllocated *search.DocumentMatch) *search.DocumentMatch { var scoreExplanation *search.Explanation // need to compute score @@ -130,9 +130,9 @@ func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc, preAllocated *sea rv := preAllocated if rv == nil { - rv = &search.DocumentMatchInternal{} + rv = &search.DocumentMatch{} } - rv.ID = termMatch.ID + rv.IndexInternalID = termMatch.ID rv.Score = score if s.explain { rv.Expl = scoreExplanation diff --git a/search/scorers/scorer_term_test.go b/search/scorers/scorer_term_test.go index 628e4e8f..5fde2df3 100644 --- a/search/scorers/scorer_term_test.go +++ b/search/scorers/scorer_term_test.go @@ -30,7 +30,7 @@ func TestTermScorer(t *testing.T) { tests := []struct { termMatch *index.TermFieldDoc - result *search.DocumentMatchInternal + result *search.DocumentMatch }{ // test some simple math { @@ -47,9 +47,9 @@ func TestTermScorer(t *testing.T) { }, }, }, - result: &search.DocumentMatchInternal{ - ID: index.IndexInternalID("one"), - Score: math.Sqrt(1.0) * idf, + result: &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("one"), + Score: math.Sqrt(1.0) * idf, Expl: &search.Explanation{ Value: math.Sqrt(1.0) * idf, Message: "fieldWeight(desc:beer in one), product of:", @@ -88,9 +88,9 @@ func TestTermScorer(t *testing.T) { Freq: 1, Norm: 1.0, }, - result: &search.DocumentMatchInternal{ - ID: index.IndexInternalID("one"), - Score: math.Sqrt(1.0) * idf, + result: &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("one"), + Score: math.Sqrt(1.0) * idf, Expl: &search.Explanation{ Value: math.Sqrt(1.0) * idf, Message: "fieldWeight(desc:beer in one), product of:", @@ -118,9 +118,9 @@ func TestTermScorer(t *testing.T) { Freq: 65, Norm: 1.0, }, - result: &search.DocumentMatchInternal{ - ID: index.IndexInternalID("one"), - Score: math.Sqrt(65) * idf, + result: &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("one"), + Score: math.Sqrt(65) * idf, Expl: &search.Explanation{ Value: math.Sqrt(65) * idf, Message: "fieldWeight(desc:beer in one), product of:", @@ -173,7 +173,7 @@ func TestTermScorerWithQueryNorm(t *testing.T) { tests := []struct { termMatch *index.TermFieldDoc - result *search.DocumentMatchInternal + result *search.DocumentMatch }{ { termMatch: &index.TermFieldDoc{ @@ -181,9 +181,9 @@ func TestTermScorerWithQueryNorm(t *testing.T) { Freq: 1, Norm: 1.0, }, - result: &search.DocumentMatchInternal{ - ID: index.IndexInternalID("one"), - Score: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0, + result: &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("one"), + Score: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0, Expl: &search.Explanation{ Value: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0, Message: "weight(desc:beer^3.000000 in one), product of:", diff --git a/search/search.go b/search/search.go index ca1726cb..7a608744 100644 --- a/search/search.go +++ b/search/search.go @@ -52,47 +52,14 @@ type FieldTermLocationMap map[string]TermLocationMap type FieldFragmentMap map[string][]string -type DocumentMatchInternal struct { - Index string - ID index.IndexInternalID - Score float64 - Expl *Explanation - Locations FieldTermLocationMap - Fragments FieldFragmentMap - - // Fields contains the values for document fields listed in - // SearchRequest.Fields. Text fields are returned as strings, numeric - // fields as float64s and date fields as time.RFC3339 formatted strings. - Fields map[string]interface{} -} - -func (dm *DocumentMatchInternal) Reset() *DocumentMatchInternal { - *dm = DocumentMatchInternal{} - return dm -} - -func (dm *DocumentMatchInternal) Finalize(r index.IndexReader) (rv *DocumentMatch, err error) { - rv = &DocumentMatch{} - rv.ID, err = r.FinalizeDocID(dm.ID) - if err != nil { - return nil, err - } - rv.Index = dm.Index - rv.Expl = dm.Expl - rv.Fields = dm.Fields - rv.Fragments = dm.Fragments - rv.Locations = dm.Locations - rv.Score = dm.Score - return rv, nil -} - type DocumentMatch struct { - Index string `json:"index,omitempty"` - ID string `json:"id"` - Score float64 `json:"score"` - Expl *Explanation `json:"explanation,omitempty"` - Locations FieldTermLocationMap `json:"locations,omitempty"` - Fragments FieldFragmentMap `json:"fragments,omitempty"` + Index string `json:"index,omitempty"` + ID string `json:"id"` + IndexInternalID index.IndexInternalID `json:"-"` + Score float64 `json:"score"` + Expl *Explanation `json:"explanation,omitempty"` + Locations FieldTermLocationMap `json:"locations,omitempty"` + Fragments FieldFragmentMap `json:"fragments,omitempty"` // Fields contains the values for document fields listed in // SearchRequest.Fields. Text fields are returned as strings, numeric @@ -133,8 +100,8 @@ func (c DocumentMatchCollection) Swap(i, j int) { c[i], c[j] = c[j], c[i] } func (c DocumentMatchCollection) Less(i, j int) bool { return c[i].Score > c[j].Score } type Searcher interface { - Next(preAllocated *DocumentMatchInternal) (*DocumentMatchInternal, error) - Advance(ID index.IndexInternalID, preAllocated *DocumentMatchInternal) (*DocumentMatchInternal, error) + Next(preAllocated *DocumentMatch) (*DocumentMatch, error) + Advance(ID index.IndexInternalID, preAllocated *DocumentMatch) (*DocumentMatch, error) Close() error Weight() float64 SetQueryNorm(float64) diff --git a/search/searchers/search_boolean.go b/search/searchers/search_boolean.go index b01e6f23..dbd401ba 100644 --- a/search/searchers/search_boolean.go +++ b/search/searchers/search_boolean.go @@ -24,9 +24,9 @@ type BooleanSearcher struct { shouldSearcher search.Searcher mustNotSearcher search.Searcher queryNorm float64 - currMust *search.DocumentMatchInternal - currShould *search.DocumentMatchInternal - currMustNot *search.DocumentMatchInternal + currMust *search.DocumentMatch + currShould *search.DocumentMatch + currMustNot *search.DocumentMatch currentID index.IndexInternalID min uint64 scorer *scorers.ConjunctionQueryScorer @@ -91,9 +91,9 @@ func (s *BooleanSearcher) initSearchers() error { } if s.mustSearcher != nil && s.currMust != nil { - s.currentID = s.currMust.ID + s.currentID = s.currMust.IndexInternalID } else if s.mustSearcher == nil && s.currShould != nil { - s.currentID = s.currShould.ID + s.currentID = s.currShould.IndexInternalID } else { s.currentID = nil } @@ -118,9 +118,9 @@ func (s *BooleanSearcher) advanceNextMust() error { } if s.mustSearcher != nil && s.currMust != nil { - s.currentID = s.currMust.ID + s.currentID = s.currMust.IndexInternalID } else if s.mustSearcher == nil && s.currShould != nil { - s.currentID = s.currShould.ID + s.currentID = s.currShould.IndexInternalID } else { s.currentID = nil } @@ -148,7 +148,7 @@ func (s *BooleanSearcher) SetQueryNorm(qnorm float64) { } } -func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { if !s.initialized { err := s.initSearchers() @@ -158,16 +158,16 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatchInternal) (*sea } var err error - var rv *search.DocumentMatchInternal + var rv *search.DocumentMatch for s.currentID != nil { - if s.currMustNot != nil && s.currMustNot.ID.Compare(s.currentID) < 0 { + if s.currMustNot != nil && s.currMustNot.IndexInternalID.Compare(s.currentID) < 0 { // advance must not searcher to our candidate entry s.currMustNot, err = s.mustNotSearcher.Advance(s.currentID, nil) if err != nil { return nil, err } - if s.currMustNot != nil && s.currMustNot.ID.Equals(s.currentID) { + if s.currMustNot != nil && s.currMustNot.IndexInternalID.Equals(s.currentID) { // the candidate is excluded err = s.advanceNextMust() if err != nil { @@ -175,7 +175,7 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatchInternal) (*sea } continue } - } else if s.currMustNot != nil && s.currMustNot.ID.Equals(s.currentID) { + } else if s.currMustNot != nil && s.currMustNot.IndexInternalID.Equals(s.currentID) { // the candidate is excluded err = s.advanceNextMust() if err != nil { @@ -184,22 +184,22 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatchInternal) (*sea continue } - if s.currShould != nil && s.currShould.ID.Compare(s.currentID) < 0 { + if s.currShould != nil && s.currShould.IndexInternalID.Compare(s.currentID) < 0 { // advance should searcher to our candidate entry s.currShould, err = s.shouldSearcher.Advance(s.currentID, nil) if err != nil { return nil, err } - if s.currShould != nil && s.currShould.ID.Equals(s.currentID) { + if s.currShould != nil && s.currShould.IndexInternalID.Equals(s.currentID) { // score bonus matches should - var cons []*search.DocumentMatchInternal + var cons []*search.DocumentMatch if s.currMust != nil { - cons = []*search.DocumentMatchInternal{ + cons = []*search.DocumentMatch{ s.currMust, s.currShould, } } else { - cons = []*search.DocumentMatchInternal{ + cons = []*search.DocumentMatch{ s.currShould, } } @@ -211,23 +211,23 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatchInternal) (*sea break } else if s.shouldSearcher.Min() == 0 { // match is OK anyway - rv = s.scorer.Score([]*search.DocumentMatchInternal{s.currMust}) + rv = s.scorer.Score([]*search.DocumentMatch{s.currMust}) err = s.advanceNextMust() if err != nil { return nil, err } break } - } else if s.currShould != nil && s.currShould.ID.Equals(s.currentID) { + } else if s.currShould != nil && s.currShould.IndexInternalID.Equals(s.currentID) { // score bonus matches should - var cons []*search.DocumentMatchInternal + var cons []*search.DocumentMatch if s.currMust != nil { - cons = []*search.DocumentMatchInternal{ + cons = []*search.DocumentMatch{ s.currMust, s.currShould, } } else { - cons = []*search.DocumentMatchInternal{ + cons = []*search.DocumentMatch{ s.currShould, } } @@ -239,7 +239,7 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatchInternal) (*sea break } else if s.shouldSearcher == nil || s.shouldSearcher.Min() == 0 { // match is OK anyway - rv = s.scorer.Score([]*search.DocumentMatchInternal{s.currMust}) + rv = s.scorer.Score([]*search.DocumentMatch{s.currMust}) err = s.advanceNextMust() if err != nil { return nil, err @@ -255,7 +255,7 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatchInternal) (*sea return rv, nil } -func (s *BooleanSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *BooleanSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { if !s.initialized { err := s.initSearchers() @@ -285,9 +285,9 @@ func (s *BooleanSearcher) Advance(ID index.IndexInternalID, preAllocated *search } if s.mustSearcher != nil && s.currMust != nil { - s.currentID = s.currMust.ID + s.currentID = s.currMust.IndexInternalID } else if s.mustSearcher == nil && s.currShould != nil { - s.currentID = s.currShould.ID + s.currentID = s.currShould.IndexInternalID } else { s.currentID = nil } diff --git a/search/searchers/search_boolean_test.go b/search/searchers/search_boolean_test.go index 0e957673..7f9f6dd2 100644 --- a/search/searchers/search_boolean_test.go +++ b/search/searchers/search_boolean_test.go @@ -243,93 +243,93 @@ func TestBooleanSearch(t *testing.T) { tests := []struct { searcher search.Searcher - results []*search.DocumentMatchInternal + results []*search.DocumentMatch }{ { searcher: booleanSearcher, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("1"), - Score: 0.9818005051949021, + IndexInternalID: index.IndexInternalID("1"), + Score: 0.9818005051949021, }, { - ID: index.IndexInternalID("3"), - Score: 0.808709699395535, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.808709699395535, }, { - ID: index.IndexInternalID("4"), - Score: 0.34618161159873423, + IndexInternalID: index.IndexInternalID("4"), + Score: 0.34618161159873423, }, }, }, { searcher: booleanSearcher2, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("1"), - Score: 0.6775110856165737, + IndexInternalID: index.IndexInternalID("1"), + Score: 0.6775110856165737, }, { - ID: index.IndexInternalID("3"), - Score: 0.6775110856165737, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.6775110856165737, }, }, }, // no MUST or SHOULD clauses yields no results { searcher: booleanSearcher3, - results: []*search.DocumentMatchInternal{}, + results: []*search.DocumentMatch{}, }, { searcher: booleanSearcher4, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("1"), - Score: 1.0, + IndexInternalID: index.IndexInternalID("1"), + Score: 1.0, }, { - ID: index.IndexInternalID("3"), - Score: 0.5, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.5, }, { - ID: index.IndexInternalID("4"), - Score: 1.0, + IndexInternalID: index.IndexInternalID("4"), + Score: 1.0, }, }, }, { searcher: booleanSearcher5, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("3"), - Score: 0.5, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.5, }, { - ID: index.IndexInternalID("4"), - Score: 1.0, + IndexInternalID: index.IndexInternalID("4"), + Score: 1.0, }, }, }, { searcher: booleanSearcher6, - results: []*search.DocumentMatchInternal{}, + results: []*search.DocumentMatch{}, }, // test a conjunction query with a nested boolean { searcher: conjunctionSearcher7, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("1"), - Score: 2.0097428702814377, + IndexInternalID: index.IndexInternalID("1"), + Score: 2.0097428702814377, }, }, }, { searcher: conjunctionSearcher8, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("3"), - Score: 2.0681575785068107, + IndexInternalID: index.IndexInternalID("3"), + Score: 2.0681575785068107, }, }, }, @@ -347,8 +347,8 @@ func TestBooleanSearch(t *testing.T) { i := 0 for err == nil && next != nil { if i < len(test.results) { - if !next.ID.Equals(test.results[i].ID) { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) diff --git a/search/searchers/search_conjunction.go b/search/searchers/search_conjunction.go index 031a977b..8e8d967a 100644 --- a/search/searchers/search_conjunction.go +++ b/search/searchers/search_conjunction.go @@ -24,7 +24,7 @@ type ConjunctionSearcher struct { searchers OrderedSearcherList explain bool queryNorm float64 - currs []*search.DocumentMatchInternal + currs []*search.DocumentMatch currentID index.IndexInternalID scorer *scorers.ConjunctionQueryScorer } @@ -42,7 +42,7 @@ func NewConjunctionSearcher(indexReader index.IndexReader, qsearchers []search.S indexReader: indexReader, explain: explain, searchers: searchers, - currs: make([]*search.DocumentMatchInternal, len(searchers)), + currs: make([]*search.DocumentMatch, len(searchers)), scorer: scorers.NewConjunctionQueryScorer(explain), } rv.computeQueryNorm() @@ -75,7 +75,7 @@ func (s *ConjunctionSearcher) initSearchers() error { if len(s.currs) > 0 { if s.currs[0] != nil { - s.currentID = s.currs[0].ID + s.currentID = s.currs[0].IndexInternalID } else { s.currentID = nil } @@ -99,21 +99,21 @@ func (s *ConjunctionSearcher) SetQueryNorm(qnorm float64) { } } -func (s *ConjunctionSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *ConjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { if !s.initialized { err := s.initSearchers() if err != nil { return nil, err } } - var rv *search.DocumentMatchInternal + var rv *search.DocumentMatch var err error OUTER: for s.currentID != nil { for i, termSearcher := range s.searchers { - if s.currs[i] != nil && !s.currs[i].ID.Equals(s.currentID) { - if s.currentID.Compare(s.currs[i].ID) < 0 { - s.currentID = s.currs[i].ID + if s.currs[i] != nil && !s.currs[i].IndexInternalID.Equals(s.currentID) { + if s.currentID.Compare(s.currs[i].IndexInternalID) < 0 { + s.currentID = s.currs[i].IndexInternalID continue OUTER } // this reader doesn't have the currentID, try to advance @@ -125,10 +125,10 @@ OUTER: s.currentID = nil continue OUTER } - if !s.currs[i].ID.Equals(s.currentID) { + if !s.currs[i].IndexInternalID.Equals(s.currentID) { // we just advanced, so it doesn't match, it must be greater // no need to call next - s.currentID = s.currs[i].ID + s.currentID = s.currs[i].IndexInternalID continue OUTER } } else if s.currs[i] == nil { @@ -147,7 +147,7 @@ OUTER: if s.currs[0] == nil { s.currentID = nil } else { - s.currentID = s.currs[0].ID + s.currentID = s.currs[0].IndexInternalID } // don't continue now, wait for the next call to Next() break @@ -155,7 +155,7 @@ OUTER: return rv, nil } -func (s *ConjunctionSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *ConjunctionSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { if !s.initialized { err := s.initSearchers() if err != nil { diff --git a/search/searchers/search_conjunction_test.go b/search/searchers/search_conjunction_test.go index 7d32547b..5e9f80ee 100644 --- a/search/searchers/search_conjunction_test.go +++ b/search/searchers/search_conjunction_test.go @@ -123,58 +123,58 @@ func TestConjunctionSearch(t *testing.T) { tests := []struct { searcher search.Searcher - results []*search.DocumentMatchInternal + results []*search.DocumentMatch }{ { searcher: beerAndMartySearcher, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("1"), - Score: 2.0097428702814377, + IndexInternalID: index.IndexInternalID("1"), + Score: 2.0097428702814377, }, }, }, { searcher: angstAndBeerSearcher, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("2"), - Score: 1.0807601687084403, + IndexInternalID: index.IndexInternalID("2"), + Score: 1.0807601687084403, }, }, }, { searcher: beerAndJackSearcher, - results: []*search.DocumentMatchInternal{}, + results: []*search.DocumentMatch{}, }, { searcher: beerAndMisterSearcher, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("2"), - Score: 1.2877980334016337, + IndexInternalID: index.IndexInternalID("2"), + Score: 1.2877980334016337, }, { - ID: index.IndexInternalID("3"), - Score: 1.2877980334016337, + IndexInternalID: index.IndexInternalID("3"), + Score: 1.2877980334016337, }, }, }, { searcher: couchbaseAndMisterSearcher, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("2"), - Score: 1.4436599157093672, + IndexInternalID: index.IndexInternalID("2"), + Score: 1.4436599157093672, }, }, }, { searcher: beerAndCouchbaseAndMisterSearcher, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("2"), - Score: 1.441614953806971, + IndexInternalID: index.IndexInternalID("2"), + Score: 1.441614953806971, }, }, }, @@ -192,8 +192,8 @@ func TestConjunctionSearch(t *testing.T) { i := 0 for err == nil && next != nil { if i < len(test.results) { - if !next.ID.Equals(test.results[i].ID) { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) diff --git a/search/searchers/search_disjunction.go b/search/searchers/search_disjunction.go index 061ac9d2..f7286350 100644 --- a/search/searchers/search_disjunction.go +++ b/search/searchers/search_disjunction.go @@ -29,7 +29,7 @@ type DisjunctionSearcher struct { indexReader index.IndexReader searchers OrderedSearcherList queryNorm float64 - currs []*search.DocumentMatchInternal + currs []*search.DocumentMatch currentID index.IndexInternalID scorer *scorers.DisjunctionQueryScorer min float64 @@ -61,7 +61,7 @@ func NewDisjunctionSearcher(indexReader index.IndexReader, qsearchers []search.S rv := DisjunctionSearcher{ indexReader: indexReader, searchers: searchers, - currs: make([]*search.DocumentMatchInternal, len(searchers)), + currs: make([]*search.DocumentMatch, len(searchers)), scorer: scorers.NewDisjunctionQueryScorer(explain), min: min, } @@ -101,8 +101,8 @@ func (s *DisjunctionSearcher) initSearchers() error { func (s *DisjunctionSearcher) nextSmallestID() index.IndexInternalID { var rv index.IndexInternalID for _, curr := range s.currs { - if curr != nil && (curr.ID.Compare(rv) < 0 || rv == nil) { - rv = curr.ID + if curr != nil && (curr.IndexInternalID.Compare(rv) < 0 || rv == nil) { + rv = curr.IndexInternalID } } return rv @@ -122,7 +122,7 @@ func (s *DisjunctionSearcher) SetQueryNorm(qnorm float64) { } } -func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { if !s.initialized { err := s.initSearchers() if err != nil { @@ -130,13 +130,13 @@ func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatchInternal) ( } } var err error - var rv *search.DocumentMatchInternal - matching := make([]*search.DocumentMatchInternal, 0, len(s.searchers)) + var rv *search.DocumentMatch + matching := make([]*search.DocumentMatch, 0, len(s.searchers)) found := false for !found && s.currentID != nil { for _, curr := range s.currs { - if curr != nil && curr.ID.Equals(s.currentID) { + if curr != nil && curr.IndexInternalID.Equals(s.currentID) { matching = append(matching, curr) } } @@ -148,10 +148,10 @@ func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatchInternal) ( } // reset matching - matching = make([]*search.DocumentMatchInternal, 0) + matching = make([]*search.DocumentMatch, 0) // invoke next on all the matching searchers for i, curr := range s.currs { - if curr != nil && curr.ID.Equals(s.currentID) { + if curr != nil && curr.IndexInternalID.Equals(s.currentID) { searcher := s.searchers[i] s.currs[i], err = searcher.Next(nil) if err != nil { @@ -164,7 +164,7 @@ func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatchInternal) ( return rv, nil } -func (s *DisjunctionSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *DisjunctionSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { if !s.initialized { err := s.initSearchers() if err != nil { diff --git a/search/searchers/search_disjunction_test.go b/search/searchers/search_disjunction_test.go index bc9d6afa..83ba98fb 100644 --- a/search/searchers/search_disjunction_test.go +++ b/search/searchers/search_disjunction_test.go @@ -66,36 +66,36 @@ func TestDisjunctionSearch(t *testing.T) { tests := []struct { searcher search.Searcher - results []*search.DocumentMatchInternal + results []*search.DocumentMatch }{ { searcher: martyOrDustinSearcher, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("1"), - Score: 0.6775110856165737, + IndexInternalID: index.IndexInternalID("1"), + Score: 0.6775110856165737, }, { - ID: index.IndexInternalID("3"), - Score: 0.6775110856165737, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.6775110856165737, }, }, }, // test a nested disjunction { searcher: nestedRaviOrMartyOrDustinSearcher, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("1"), - Score: 0.2765927424732821, + IndexInternalID: index.IndexInternalID("1"), + Score: 0.2765927424732821, }, { - ID: index.IndexInternalID("3"), - Score: 0.2765927424732821, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.2765927424732821, }, { - ID: index.IndexInternalID("4"), - Score: 0.5531854849465642, + IndexInternalID: index.IndexInternalID("4"), + Score: 0.5531854849465642, }, }, }, @@ -113,8 +113,8 @@ func TestDisjunctionSearch(t *testing.T) { i := 0 for err == nil && next != nil { if i < len(test.results) { - if !next.ID.Equals(test.results[i].ID) { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) diff --git a/search/searchers/search_docid.go b/search/searchers/search_docid.go index 72c489cf..0ea378c8 100644 --- a/search/searchers/search_docid.go +++ b/search/searchers/search_docid.go @@ -82,7 +82,7 @@ func (s *DocIDSearcher) SetQueryNorm(qnorm float64) { s.scorer.SetQueryNorm(qnorm) } -func (s *DocIDSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *DocIDSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { // if s.current >= len(s.ids) { // return nil, nil // } @@ -103,7 +103,7 @@ func (s *DocIDSearcher) Next(preAllocated *search.DocumentMatchInternal) (*searc return docMatch, nil } -func (s *DocIDSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *DocIDSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { // s.current = sort.SearchStrings(s.ids, ID) // return s.Next(preAllocated) diff --git a/search/searchers/search_docid_test.go b/search/searchers/search_docid_test.go index 4bd8c43c..050ef2e2 100644 --- a/search/searchers/search_docid_test.go +++ b/search/searchers/search_docid_test.go @@ -68,8 +68,8 @@ func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { if err != nil { t.Fatal(err) } - if !index.IndexInternalID(id).Equals(m.ID) { - t.Fatalf("expected %v at position %v, got %v", id, i, m.ID) + if !index.IndexInternalID(id).Equals(m.IndexInternalID) { + t.Fatalf("expected %v at position %v, got %v", id, i, m.IndexInternalID) } } m, err := searcher.Next(nil) @@ -77,7 +77,7 @@ func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { t.Fatal(err) } if m != nil { - t.Fatalf("expected nil past the end of the sequence, got %v", m.ID) + t.Fatalf("expected nil past the end of the sequence, got %v", m.IndexInternalID) } // Check seeking @@ -91,7 +91,7 @@ func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { if err != nil { t.Fatal(err) } - if m == nil || !m.ID.Equals(index.IndexInternalID(id)) { + if m == nil || !m.IndexInternalID.Equals(index.IndexInternalID(id)) { t.Fatalf("advancing to %v returned %v instead of %v", before, m, id) } } diff --git a/search/searchers/search_fuzzy.go b/search/searchers/search_fuzzy.go index 9f42995c..0b26c540 100644 --- a/search/searchers/search_fuzzy.go +++ b/search/searchers/search_fuzzy.go @@ -107,12 +107,12 @@ func (s *FuzzySearcher) SetQueryNorm(qnorm float64) { s.searcher.SetQueryNorm(qnorm) } -func (s *FuzzySearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *FuzzySearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { return s.searcher.Next(preAllocated) } -func (s *FuzzySearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *FuzzySearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { return s.searcher.Advance(ID, preAllocated) } diff --git a/search/searchers/search_fuzzy_test.go b/search/searchers/search_fuzzy_test.go index 461d8e31..409a5069 100644 --- a/search/searchers/search_fuzzy_test.go +++ b/search/searchers/search_fuzzy_test.go @@ -51,48 +51,48 @@ func TestFuzzySearch(t *testing.T) { tests := []struct { searcher search.Searcher - results []*search.DocumentMatchInternal + results []*search.DocumentMatch }{ { searcher: fuzzySearcherbeet, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("1"), - Score: 1.0, + IndexInternalID: index.IndexInternalID("1"), + Score: 1.0, }, { - ID: index.IndexInternalID("2"), - Score: 0.5, + IndexInternalID: index.IndexInternalID("2"), + Score: 0.5, }, { - ID: index.IndexInternalID("3"), - Score: 0.5, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.5, }, { - ID: index.IndexInternalID("4"), - Score: 0.9999999838027345, + IndexInternalID: index.IndexInternalID("4"), + Score: 0.9999999838027345, }, }, }, { searcher: fuzzySearcherdouches, - results: []*search.DocumentMatchInternal{}, + results: []*search.DocumentMatch{}, }, { searcher: fuzzySearcheraplee, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("3"), - Score: 0.9581453659370776, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.9581453659370776, }, }, }, { searcher: fuzzySearcherprefix, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("5"), - Score: 1.916290731874155, + IndexInternalID: index.IndexInternalID("5"), + Score: 1.916290731874155, }, }, }, @@ -110,8 +110,8 @@ func TestFuzzySearch(t *testing.T) { i := 0 for err == nil && next != nil { if i < len(test.results) { - if !next.ID.Equals(test.results[i].ID) { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if next.Score != test.results[i].Score { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) diff --git a/search/searchers/search_match_all.go b/search/searchers/search_match_all.go index 77cde2e1..38ed6ab1 100644 --- a/search/searchers/search_match_all.go +++ b/search/searchers/search_match_all.go @@ -46,7 +46,7 @@ func (s *MatchAllSearcher) SetQueryNorm(qnorm float64) { s.scorer.SetQueryNorm(qnorm) } -func (s *MatchAllSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *MatchAllSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { id, err := s.reader.Next() if err != nil { return nil, err @@ -63,7 +63,7 @@ func (s *MatchAllSearcher) Next(preAllocated *search.DocumentMatchInternal) (*se } -func (s *MatchAllSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *MatchAllSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { id, err := s.reader.Advance(ID) if err != nil { return nil, err diff --git a/search/searchers/search_match_all_test.go b/search/searchers/search_match_all_test.go index cba20d15..f265e002 100644 --- a/search/searchers/search_match_all_test.go +++ b/search/searchers/search_match_all_test.go @@ -42,57 +42,57 @@ func TestMatchAllSearch(t *testing.T) { tests := []struct { searcher search.Searcher queryNorm float64 - results []*search.DocumentMatchInternal + results []*search.DocumentMatch }{ { searcher: allSearcher, queryNorm: 1.0, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("1"), - Score: 1.0, + IndexInternalID: index.IndexInternalID("1"), + Score: 1.0, }, { - ID: index.IndexInternalID("2"), - Score: 1.0, + IndexInternalID: index.IndexInternalID("2"), + Score: 1.0, }, { - ID: index.IndexInternalID("3"), - Score: 1.0, + IndexInternalID: index.IndexInternalID("3"), + Score: 1.0, }, { - ID: index.IndexInternalID("4"), - Score: 1.0, + IndexInternalID: index.IndexInternalID("4"), + Score: 1.0, }, { - ID: index.IndexInternalID("5"), - Score: 1.0, + IndexInternalID: index.IndexInternalID("5"), + Score: 1.0, }, }, }, { searcher: allSearcher2, queryNorm: 0.8333333, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("1"), - Score: 1.0, + IndexInternalID: index.IndexInternalID("1"), + Score: 1.0, }, { - ID: index.IndexInternalID("2"), - Score: 1.0, + IndexInternalID: index.IndexInternalID("2"), + Score: 1.0, }, { - ID: index.IndexInternalID("3"), - Score: 1.0, + IndexInternalID: index.IndexInternalID("3"), + Score: 1.0, }, { - ID: index.IndexInternalID("4"), - Score: 1.0, + IndexInternalID: index.IndexInternalID("4"), + Score: 1.0, }, { - ID: index.IndexInternalID("5"), - Score: 1.0, + IndexInternalID: index.IndexInternalID("5"), + Score: 1.0, }, }, }, @@ -114,8 +114,8 @@ func TestMatchAllSearch(t *testing.T) { i := 0 for err == nil && next != nil { if i < len(test.results) { - if !next.ID.Equals(test.results[i].ID) { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) diff --git a/search/searchers/search_match_none.go b/search/searchers/search_match_none.go index 08bae499..08eefb03 100644 --- a/search/searchers/search_match_none.go +++ b/search/searchers/search_match_none.go @@ -36,11 +36,11 @@ func (s *MatchNoneSearcher) SetQueryNorm(qnorm float64) { } -func (s *MatchNoneSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *MatchNoneSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { return nil, nil } -func (s *MatchNoneSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *MatchNoneSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { return nil, nil } diff --git a/search/searchers/search_match_none_test.go b/search/searchers/search_match_none_test.go index dd730e38..f704d29c 100644 --- a/search/searchers/search_match_none_test.go +++ b/search/searchers/search_match_none_test.go @@ -35,11 +35,11 @@ func TestMatchNoneSearch(t *testing.T) { tests := []struct { searcher search.Searcher - results []*search.DocumentMatchInternal + results []*search.DocumentMatch }{ { searcher: noneSearcher, - results: []*search.DocumentMatchInternal{}, + results: []*search.DocumentMatch{}, }, } @@ -55,8 +55,8 @@ func TestMatchNoneSearch(t *testing.T) { i := 0 for err == nil && next != nil { if i < len(test.results) { - if !next.ID.Equals(test.results[i].ID) { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if !scoresCloseEnough(next.Score, test.results[i].Score) { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) diff --git a/search/searchers/search_numeric_range.go b/search/searchers/search_numeric_range.go index 067ad9f6..6f236dd5 100644 --- a/search/searchers/search_numeric_range.go +++ b/search/searchers/search_numeric_range.go @@ -96,11 +96,11 @@ func (s *NumericRangeSearcher) SetQueryNorm(qnorm float64) { s.searcher.SetQueryNorm(qnorm) } -func (s *NumericRangeSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *NumericRangeSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { return s.searcher.Next(preAllocated) } -func (s *NumericRangeSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *NumericRangeSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { return s.searcher.Advance(ID, preAllocated) } diff --git a/search/searchers/search_phrase.go b/search/searchers/search_phrase.go index b7c63ef1..34a31ed9 100644 --- a/search/searchers/search_phrase.go +++ b/search/searchers/search_phrase.go @@ -21,7 +21,7 @@ type PhraseSearcher struct { indexReader index.IndexReader mustSearcher *ConjunctionSearcher queryNorm float64 - currMust *search.DocumentMatchInternal + currMust *search.DocumentMatch slop int terms []string } @@ -90,7 +90,7 @@ func (s *PhraseSearcher) SetQueryNorm(qnorm float64) { s.mustSearcher.SetQueryNorm(qnorm) } -func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { if !s.initialized { err := s.initSearchers() if err != nil { @@ -98,7 +98,7 @@ func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatchInternal) (*sear } } - var rv *search.DocumentMatchInternal + var rv *search.DocumentMatch for s.currMust != nil { rvftlm := make(search.FieldTermLocationMap, 0) freq := 0 @@ -160,7 +160,7 @@ func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatchInternal) (*sear return nil, nil } -func (s *PhraseSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *PhraseSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { if !s.initialized { err := s.initSearchers() if err != nil { diff --git a/search/searchers/search_phrase_test.go b/search/searchers/search_phrase_test.go index 2386d975..94380f6e 100644 --- a/search/searchers/search_phrase_test.go +++ b/search/searchers/search_phrase_test.go @@ -48,14 +48,14 @@ func TestPhraseSearch(t *testing.T) { tests := []struct { searcher search.Searcher - results []*search.DocumentMatchInternal + results []*search.DocumentMatch }{ { searcher: phraseSearcher, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("2"), - Score: 1.0807601687084403, + IndexInternalID: index.IndexInternalID("2"), + Score: 1.0807601687084403, }, }, }, @@ -73,8 +73,8 @@ func TestPhraseSearch(t *testing.T) { i := 0 for err == nil && next != nil { if i < len(test.results) { - if !next.ID.Equals(test.results[i].ID) { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if next.Score != test.results[i].Score { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) diff --git a/search/searchers/search_regexp.go b/search/searchers/search_regexp.go index 6173f07a..8e6ce52b 100644 --- a/search/searchers/search_regexp.go +++ b/search/searchers/search_regexp.go @@ -106,12 +106,12 @@ func (s *RegexpSearcher) SetQueryNorm(qnorm float64) { s.searcher.SetQueryNorm(qnorm) } -func (s *RegexpSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *RegexpSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { return s.searcher.Next(preAllocated) } -func (s *RegexpSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *RegexpSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { return s.searcher.Advance(ID, preAllocated) } diff --git a/search/searchers/search_regexp_test.go b/search/searchers/search_regexp_test.go index f1ce7c0e..489297e2 100644 --- a/search/searchers/search_regexp_test.go +++ b/search/searchers/search_regexp_test.go @@ -52,27 +52,27 @@ func TestRegexpSearch(t *testing.T) { tests := []struct { searcher search.Searcher - results []*search.DocumentMatchInternal + results []*search.DocumentMatch }{ { searcher: regexpSearcher, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("1"), - Score: 1.916290731874155, + IndexInternalID: index.IndexInternalID("1"), + Score: 1.916290731874155, }, }, }, { searcher: regexpSearcherCo, - results: []*search.DocumentMatchInternal{ + results: []*search.DocumentMatch{ { - ID: index.IndexInternalID("2"), - Score: 0.33875554280828685, + IndexInternalID: index.IndexInternalID("2"), + Score: 0.33875554280828685, }, { - ID: index.IndexInternalID("3"), - Score: 0.33875554280828685, + IndexInternalID: index.IndexInternalID("3"), + Score: 0.33875554280828685, }, }, }, @@ -90,8 +90,8 @@ func TestRegexpSearch(t *testing.T) { i := 0 for err == nil && next != nil { if i < len(test.results) { - if !next.ID.Equals(test.results[i].ID) { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) } if next.Score != test.results[i].Score { t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) diff --git a/search/searchers/search_term.go b/search/searchers/search_term.go index aa95bc38..ac9855a7 100644 --- a/search/searchers/search_term.go +++ b/search/searchers/search_term.go @@ -53,7 +53,7 @@ func (s *TermSearcher) SetQueryNorm(qnorm float64) { s.scorer.SetQueryNorm(qnorm) } -func (s *TermSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *TermSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { termMatch, err := s.reader.Next(s.tfd.Reset()) if err != nil { return nil, err @@ -70,7 +70,7 @@ func (s *TermSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search } -func (s *TermSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *TermSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { termMatch, err := s.reader.Advance(ID, s.tfd.Reset()) if err != nil { return nil, err diff --git a/search/searchers/search_term_prefix.go b/search/searchers/search_term_prefix.go index bbd8a78c..80f22e91 100644 --- a/search/searchers/search_term_prefix.go +++ b/search/searchers/search_term_prefix.go @@ -70,12 +70,12 @@ func (s *TermPrefixSearcher) SetQueryNorm(qnorm float64) { s.searcher.SetQueryNorm(qnorm) } -func (s *TermPrefixSearcher) Next(preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *TermPrefixSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { return s.searcher.Next(preAllocated) } -func (s *TermPrefixSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatchInternal) (*search.DocumentMatchInternal, error) { +func (s *TermPrefixSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { return s.searcher.Advance(ID, preAllocated) } diff --git a/search/searchers/search_term_test.go b/search/searchers/search_term_test.go index d46274f6..e60d9184 100644 --- a/search/searchers/search_term_test.go +++ b/search/searchers/search_term_test.go @@ -167,15 +167,15 @@ func TestTermSearcher(t *testing.T) { if err != nil { t.Errorf("expected result, got %v", err) } - if !docMatch.ID.Equals(index.IndexInternalID("a")) { - t.Errorf("expected result ID to be 'a', got '%s", docMatch.ID) + if !docMatch.IndexInternalID.Equals(index.IndexInternalID("a")) { + t.Errorf("expected result ID to be 'a', got '%s", docMatch.IndexInternalID) } docMatch, err = searcher.Advance(index.IndexInternalID("c"), nil) if err != nil { t.Errorf("expected result, got %v", err) } - if !docMatch.ID.Equals(index.IndexInternalID("c")) { - t.Errorf("expected result ID to be 'c' got '%s'", docMatch.ID) + if !docMatch.IndexInternalID.Equals(index.IndexInternalID("c")) { + t.Errorf("expected result ID to be 'c' got '%s'", docMatch.IndexInternalID) } // try advancing past end From 172ca7e69e973dbd0d949a261d8d4566e3292142 Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Mon, 1 Aug 2016 17:01:04 -0400 Subject: [PATCH 05/13] need to copy the doc ID for it to survive past next iteration --- index/upside_down/reader.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/index/upside_down/reader.go b/index/upside_down/reader.go index 92f77dbf..e3009b48 100644 --- a/index/upside_down/reader.go +++ b/index/upside_down/reader.go @@ -83,7 +83,7 @@ func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (* if rv == nil { rv = &index.TermFieldDoc{} } - rv.ID = tfr.doc + rv.ID = append([]byte(nil), tfr.doc...) rv.Freq = tfr.freq rv.Norm = float64(tfr.norm) if tfr.vectors != nil { @@ -110,7 +110,7 @@ func (r *UpsideDownCouchTermFieldReader) Advance(docID index.IndexInternalID, pr if rv == nil { rv = &index.TermFieldDoc{} } - rv.ID = tfr.doc + rv.ID = append([]byte(nil), tfr.doc...) rv.Freq = tfr.freq rv.Norm = float64(tfr.norm) if tfr.vectors != nil { @@ -199,7 +199,7 @@ func (r *UpsideDownCouchDocIDReader) Next() (index.IndexInternalID, error) { key, val, valid = r.iterator.Current() continue } else { - rv = br.doc + rv = append([]byte(nil), br.doc...) break } } @@ -217,7 +217,7 @@ func (r *UpsideDownCouchDocIDReader) Next() (index.IndexInternalID, error) { if err != nil { return nil, err } - rv := br.doc + rv := append([]byte(nil), br.doc...) r.iterator.Next() return rv, nil } @@ -246,7 +246,7 @@ func (r *UpsideDownCouchDocIDReader) Advance(docID index.IndexInternalID) (index r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) continue } else { - rv = br.doc + rv = append([]byte(nil), br.doc...) break } } @@ -263,7 +263,7 @@ func (r *UpsideDownCouchDocIDReader) Advance(docID index.IndexInternalID) (index if err != nil { return nil, err } - rv := br.doc + rv := append([]byte(nil), br.doc...) r.iterator.Next() return rv, nil } From cfce9c5fc5696c36ba80445e13f76abc27ad43ee Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Mon, 1 Aug 2016 17:01:34 -0400 Subject: [PATCH 06/13] initialize term vector list in parseV otherwise reusing previous term frequency row causes us to keep tacking on to one gigantic list --- index/upside_down/row.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/index/upside_down/row.go b/index/upside_down/row.go index 7327f5e3..a776e40b 100644 --- a/index/upside_down/row.go +++ b/index/upside_down/row.go @@ -551,9 +551,7 @@ func (tfr *TermFrequencyRow) parseV(value []byte) error { tv := TermVector{} tv.field = uint16(field) // at this point we expect at least one term vector - if tfr.vectors == nil { - tfr.vectors = make([]*TermVector, 0) - } + tfr.vectors = make([]*TermVector, 0) tv.pos, bytesRead = binary.Uvarint(value[currOffset:]) if bytesRead <= 0 { From 36de4a7097cb72fb083a6f2740b25efae31ffe01 Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Mon, 1 Aug 2016 17:17:29 -0400 Subject: [PATCH 07/13] cleaner fix for the TermFrequencyRow reuse bug reset to nil first, let remaining logic work as before --- index/upside_down/row.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/index/upside_down/row.go b/index/upside_down/row.go index a776e40b..f8cab8ab 100644 --- a/index/upside_down/row.go +++ b/index/upside_down/row.go @@ -544,6 +544,7 @@ func (tfr *TermFrequencyRow) parseV(value []byte) error { tfr.norm = math.Float32frombits(uint32(norm)) + tfr.vectors = nil var field uint64 field, bytesRead = binary.Uvarint(value[currOffset:]) for bytesRead > 0 { @@ -551,7 +552,9 @@ func (tfr *TermFrequencyRow) parseV(value []byte) error { tv := TermVector{} tv.field = uint16(field) // at this point we expect at least one term vector - tfr.vectors = make([]*TermVector, 0) + if tfr.vectors == nil { + tfr.vectors = make([]*TermVector, 0) + } tv.pos, bytesRead = binary.Uvarint(value[currOffset:]) if bytesRead <= 0 { From 4b1b866e0f677d47e50d335d01c2eaa4905f7299 Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Tue, 2 Aug 2016 16:48:00 -0400 Subject: [PATCH 08/13] remove commented out old code --- search/searchers/search_docid.go | 44 -------------------------------- 1 file changed, 44 deletions(-) diff --git a/search/searchers/search_docid.go b/search/searchers/search_docid.go index 0ea378c8..8061f25b 100644 --- a/search/searchers/search_docid.go +++ b/search/searchers/search_docid.go @@ -25,38 +25,6 @@ type DocIDSearcher struct { func NewDocIDSearcher(indexReader index.IndexReader, ids []string, boost float64, explain bool) (searcher *DocIDSearcher, err error) { - // kept := make([]string, len(ids)) - // copy(kept, ids) - // sort.Strings(kept) - // - // if len(ids) > 0 { - // var idReader index.DocIDReader - // endTerm := string(incrementBytes([]byte(kept[len(kept)-1]))) - // idReader, err = indexReader.DocIDReader(kept[0], endTerm) - // if err != nil { - // return nil, err - // } - // defer func() { - // if cerr := idReader.Close(); err == nil && cerr != nil { - // err = cerr - // } - // }() - // j := 0 - // for _, id := range kept { - // doc, err := idReader.Next() - // if err != nil { - // return nil, err - // } - // // Non-duplicate match - // actualDocID := indexReader.FinalizeDocID(doc) - // if actualDocID == id && (j == 0 || kept[j-1] != id) { - // kept[j] = id - // j++ - // } - // } - // kept = kept[:j] - // } - reader, err := indexReader.DocIDReaderOnly(ids) if err != nil { return nil, err @@ -70,7 +38,6 @@ func NewDocIDSearcher(indexReader index.IndexReader, ids []string, boost float64 } func (s *DocIDSearcher) Count() uint64 { - // return uint64(len(s.ids)) return uint64(s.count) } @@ -83,14 +50,6 @@ func (s *DocIDSearcher) SetQueryNorm(qnorm float64) { } func (s *DocIDSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - // if s.current >= len(s.ids) { - // return nil, nil - // } - // id := s.ids[s.current] - // s.current++ - // docMatch := s.scorer.Score(id) - // return docMatch, nil - docidMatch, err := s.reader.Next() if err != nil { return nil, err @@ -104,9 +63,6 @@ func (s *DocIDSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docume } func (s *DocIDSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - // s.current = sort.SearchStrings(s.ids, ID) - // return s.Next(preAllocated) - docidMatch, err := s.reader.Advance(ID) if err != nil { return nil, err From 89d83cb5a17bcf4351a4ae5ae87db9077ceb13d8 Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Wed, 3 Aug 2016 13:45:48 -0400 Subject: [PATCH 09/13] reuse memory already allocated for copies of docids when the term field reader is copying ID values out of the kv store's iterator, it is already attempting to reuse the term frequency row data structure. this change allows us to also attempt to reuse the []byte allocated for previous copies of the docid. we reset the slice length to zero then copy the data into the existing slice, avoiding new allocation and garbage collection in the cases where there is already enough space --- index/upside_down/reader.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/index/upside_down/reader.go b/index/upside_down/reader.go index e3009b48..e2521c32 100644 --- a/index/upside_down/reader.go +++ b/index/upside_down/reader.go @@ -83,7 +83,8 @@ func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (* if rv == nil { rv = &index.TermFieldDoc{} } - rv.ID = append([]byte(nil), tfr.doc...) + rv.ID = rv.ID[:0] + rv.ID = append(rv.ID, tfr.doc...) rv.Freq = tfr.freq rv.Norm = float64(tfr.norm) if tfr.vectors != nil { @@ -110,7 +111,8 @@ func (r *UpsideDownCouchTermFieldReader) Advance(docID index.IndexInternalID, pr if rv == nil { rv = &index.TermFieldDoc{} } - rv.ID = append([]byte(nil), tfr.doc...) + rv.ID = rv.ID[:0] + rv.ID = append(rv.ID, tfr.doc...) rv.Freq = tfr.freq rv.Norm = float64(tfr.norm) if tfr.vectors != nil { From d7405a4d79e9e74918a18ba1fd842fb822c2be6b Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Wed, 3 Aug 2016 17:01:27 -0400 Subject: [PATCH 10/13] updated attempt to reuse []byte previous attempt was flawed (but maked by Reset() method) new approach is to do this work in the Reset() method itself, logically this is where it belongs. but further we acknowledge that IndexInternalID []byte lifetime lives beyond the TermFieldDoc, so another copy is made into the DocumentMatch. Although this introduces yet another copy the theory being tested is that it allows each of these structuress to reuse memory without additional allocation. --- index/index.go | 2 ++ index/upside_down/reader.go | 2 -- search/scorers/scorer_term.go | 2 +- search/search.go | 2 ++ 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/index/index.go b/index/index.go index 880c35fe..77e4d468 100644 --- a/index/index.go +++ b/index/index.go @@ -123,7 +123,9 @@ type TermFieldDoc struct { } func (tfd *TermFieldDoc) Reset() *TermFieldDoc { + id := tfd.ID *tfd = TermFieldDoc{} + tfd.ID = id[:0] return tfd } diff --git a/index/upside_down/reader.go b/index/upside_down/reader.go index e2521c32..caca3fac 100644 --- a/index/upside_down/reader.go +++ b/index/upside_down/reader.go @@ -83,7 +83,6 @@ func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (* if rv == nil { rv = &index.TermFieldDoc{} } - rv.ID = rv.ID[:0] rv.ID = append(rv.ID, tfr.doc...) rv.Freq = tfr.freq rv.Norm = float64(tfr.norm) @@ -111,7 +110,6 @@ func (r *UpsideDownCouchTermFieldReader) Advance(docID index.IndexInternalID, pr if rv == nil { rv = &index.TermFieldDoc{} } - rv.ID = rv.ID[:0] rv.ID = append(rv.ID, tfr.doc...) rv.Freq = tfr.freq rv.Norm = float64(tfr.norm) diff --git a/search/scorers/scorer_term.go b/search/scorers/scorer_term.go index 458c9312..820842bf 100644 --- a/search/scorers/scorer_term.go +++ b/search/scorers/scorer_term.go @@ -132,7 +132,7 @@ func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc, preAllocated *sea if rv == nil { rv = &search.DocumentMatch{} } - rv.IndexInternalID = termMatch.ID + rv.IndexInternalID = append(rv.IndexInternalID, termMatch.ID...) rv.Score = score if s.explain { rv.Expl = scoreExplanation diff --git a/search/search.go b/search/search.go index 7a608744..a5739921 100644 --- a/search/search.go +++ b/search/search.go @@ -89,7 +89,9 @@ func (dm *DocumentMatch) AddFieldValue(name string, value interface{}) { } func (dm *DocumentMatch) Reset() *DocumentMatch { + indexInternalId := dm.IndexInternalID *dm = DocumentMatch{} + dm.IndexInternalID = indexInternalId[:0] return dm } From b857769217796ce48b0c38d42ca532251d750819 Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Wed, 3 Aug 2016 17:16:15 -0400 Subject: [PATCH 11/13] document Reset behavior as its non-obvious --- index/index.go | 4 ++++ search/search.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/index/index.go b/index/index.go index 77e4d468..ac86f20a 100644 --- a/index/index.go +++ b/index/index.go @@ -122,9 +122,13 @@ type TermFieldDoc struct { Vectors []*TermFieldVector } +// Reset allows an already allocated TermFieldDoc to be reused func (tfd *TermFieldDoc) Reset() *TermFieldDoc { + // remember the []byte used for the ID id := tfd.ID + // idiom to copy over from empty TermFieldDoc (0 allocations) *tfd = TermFieldDoc{} + // reuse the []byte already allocated (and reset len to 0) tfd.ID = id[:0] return tfd } diff --git a/search/search.go b/search/search.go index a5739921..984b1464 100644 --- a/search/search.go +++ b/search/search.go @@ -88,9 +88,13 @@ func (dm *DocumentMatch) AddFieldValue(name string, value interface{}) { dm.Fields[name] = valSlice } +// Reset allows an already allocated DocumentMatch to be reused func (dm *DocumentMatch) Reset() *DocumentMatch { + // remember the []byte used for the IndexInternalID indexInternalId := dm.IndexInternalID + // idiom to copy over from empty DocumentMatch (0 allocations) *dm = DocumentMatch{} + // reuse the []byte already allocated (and reset len to 0) dm.IndexInternalID = indexInternalId[:0] return dm } From 24a2b57e29508a6c315b495ff92d65747e9367e8 Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Mon, 8 Aug 2016 22:21:47 -0400 Subject: [PATCH 12/13] refactor search package to reuse DocumentMatch and ID []byte's the motivation for this commit is long and detailed and has been documented externally here: https://gist.github.com/mschoch/5cc5c9cf4669a5fe8512cb7770d3c1a2 the core of the changes are: 1. recognize that collector/searcher need only a fixed number of DocumentMatch instances, and this number can be determined from the structure of the query, not the size of the data 2. knowing this, instances can be allocated in bulk, up front and they can be reused without locking (since all search operations take place in a single goroutine 3. combined with previous commits which enabled reuse of the IndexInternalID []byte, this allows for no allocation/copy of these bytes as well (by using DocumentMatch Reset() method when returning entries to the pool --- search/collectors/collector_top_score.go | 43 +++++---- search/collectors/search_test.go | 8 +- search/pool.go | 70 ++++++++++++++ search/scorers/scorer_conjunction.go | 17 ++-- search/scorers/scorer_constant.go | 11 +-- search/scorers/scorer_constant_test.go | 10 +- search/scorers/scorer_disjunction.go | 17 ++-- search/scorers/scorer_term.go | 7 +- search/scorers/scorer_term_test.go | 10 +- search/search.go | 11 ++- search/searchers/search_boolean.go | 100 ++++++++++++++------ search/searchers/search_boolean_test.go | 8 +- search/searchers/search_conjunction.go | 53 ++++++++--- search/searchers/search_conjunction_test.go | 7 +- search/searchers/search_disjunction.go | 37 ++++++-- search/searchers/search_disjunction_test.go | 13 ++- search/searchers/search_docid.go | 12 ++- search/searchers/search_docid_test.go | 17 +++- search/searchers/search_fuzzy.go | 12 ++- search/searchers/search_fuzzy_test.go | 8 +- search/searchers/search_match_all.go | 12 ++- search/searchers/search_match_all_test.go | 8 +- search/searchers/search_match_none.go | 8 +- search/searchers/search_match_none_test.go | 8 +- search/searchers/search_numeric_range.go | 12 ++- search/searchers/search_phrase.go | 28 +++--- search/searchers/search_phrase_test.go | 8 +- search/searchers/search_regexp.go | 12 ++- search/searchers/search_regexp_test.go | 8 +- search/searchers/search_term.go | 12 ++- search/searchers/search_term_prefix.go | 12 ++- search/searchers/search_term_test.go | 15 ++- 32 files changed, 441 insertions(+), 173 deletions(-) create mode 100644 search/pool.go diff --git a/search/collectors/collector_top_score.go b/search/collectors/collector_top_score.go index a78eb2d5..7d7131ce 100644 --- a/search/collectors/collector_top_score.go +++ b/search/collectors/collector_top_score.go @@ -64,13 +64,18 @@ var COLLECT_CHECK_DONE_EVERY = uint64(1024) func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Searcher, reader index.IndexReader) error { startTime := time.Now() var err error - var pre search.DocumentMatch // A single pre-alloc'ed, reused instance. var next *search.DocumentMatch + + // search context with enough pre-allocated document matches + searchContext := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(tksc.k + tksc.skip + searcher.DocumentMatchPoolSize()), + } + select { case <-ctx.Done(): return ctx.Err() default: - next, err = searcher.Next(&pre) + next, err = searcher.Next(searchContext) } for err == nil && next != nil { if tksc.total%COLLECT_CHECK_DONE_EVERY == 0 { @@ -80,14 +85,15 @@ func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Sear default: } } - tksc.collectSingle(next) if tksc.facetsBuilder != nil { err = tksc.facetsBuilder.Update(next) if err != nil { break } } - next, err = searcher.Next(pre.Reset()) + tksc.collectSingle(searchContext, next) + + next, err = searcher.Next(searchContext) } // finalize actual results tksc.actualResults, err = tksc.finalizeResults(reader) @@ -103,43 +109,42 @@ func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Sear return nil } -func (tksc *TopScoreCollector) collectSingle(dmIn *search.DocumentMatch) { +func (tksc *TopScoreCollector) collectSingle(ctx *search.SearchContext, d *search.DocumentMatch) { // increment total hits tksc.total++ // update max score - if dmIn.Score > tksc.maxScore { - tksc.maxScore = dmIn.Score + if d.Score > tksc.maxScore { + tksc.maxScore = d.Score } - if dmIn.Score <= tksc.minScore { + if d.Score <= tksc.minScore { + ctx.DocumentMatchPool.Put(d) return } - // Because the dmIn will be the single, pre-allocated, reused - // instance, we need to copy the dmIn into a new, standalone - // instance before inserting into our candidate results list. - dm := &search.DocumentMatch{} - *dm = *dmIn - for e := tksc.results.Front(); e != nil; e = e.Next() { curr := e.Value.(*search.DocumentMatch) - if dm.Score <= curr.Score { + if d.Score <= curr.Score { - tksc.results.InsertBefore(dm, e) + tksc.results.InsertBefore(d, e) // if we just made the list too long if tksc.results.Len() > (tksc.k + tksc.skip) { // remove the head - tksc.minScore = tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch).Score + removed := tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch) + tksc.minScore = removed.Score + ctx.DocumentMatchPool.Put(removed) } return } } // if we got to the end, we still have to add it - tksc.results.PushBack(dm) + tksc.results.PushBack(d) if tksc.results.Len() > (tksc.k + tksc.skip) { // remove the head - tksc.minScore = tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch).Score + removed := tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch) + tksc.minScore = removed.Score + ctx.DocumentMatchPool.Put(removed) } } diff --git a/search/collectors/search_test.go b/search/collectors/search_test.go index 0f803f1a..c5faa243 100644 --- a/search/collectors/search_test.go +++ b/search/collectors/search_test.go @@ -20,7 +20,7 @@ type stubSearcher struct { matches []*search.DocumentMatch } -func (ss *stubSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (ss *stubSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { if ss.index < len(ss.matches) { rv := ss.matches[ss.index] ss.index++ @@ -29,7 +29,7 @@ func (ss *stubSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docume return nil, nil } -func (ss *stubSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (ss *stubSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { for ss.index < len(ss.matches) && ss.matches[ss.index].IndexInternalID.Compare(ID) < 0 { ss.index++ @@ -61,6 +61,10 @@ func (ss *stubSearcher) Min() int { return 0 } +func (ss *stubSearcher) DocumentMatchPoolSize() int { + return 0 +} + type stubReader struct{} func (sr *stubReader) TermFieldReader(term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (index.TermFieldReader, error) { diff --git a/search/pool.go b/search/pool.go new file mode 100644 index 00000000..108d494b --- /dev/null +++ b/search/pool.go @@ -0,0 +1,70 @@ +// Copyright (c) 2014 Couchbase, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file +// except in compliance with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed under the +// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +// either express or implied. See the License for the specific language governing permissions +// and limitations under the License. + +package search + +// DocumentMatchPoolTooSmall is a callback function that can be executed +// when the DocumentMatchPool does not have sufficient capacity +// By default we just perform just-in-time allocation, but you could log +// a message, or panic, etc. +type DocumentMatchPoolTooSmall func(p *DocumentMatchPool) *DocumentMatch + +// DocumentMatchPool manages use/re-use of DocumentMatch instances +// it pre-allocates space from a single large block with the expected +// number of instances. It is not thread-safe as currently all +// aspects of search take place in a single goroutine. +type DocumentMatchPool struct { + avail DocumentMatchCollection + TooSmall DocumentMatchPoolTooSmall +} + +func defaultDocumentMatchPoolTooSmall(p *DocumentMatchPool) *DocumentMatch { + return &DocumentMatch{} +} + +// NewDocumentMatchPool will build a DocumentMatchPool with memory +// pre-allocated to accomodate the requested number of DocumentMatch +// instances +func NewDocumentMatchPool(size int) *DocumentMatchPool { + avail := make(DocumentMatchCollection, 0, size) + // pre-allocate the expected number of instances + startBlock := make([]DocumentMatch, size) + // make these initial instances available + for i := range startBlock { + avail = append(avail, &startBlock[i]) + } + return &DocumentMatchPool{ + avail: avail, + TooSmall: defaultDocumentMatchPoolTooSmall, + } +} + +// Get returns an available DocumentMatch from the pool +// if the pool was not allocated with sufficient size, an allocation will +// occur to satisfy this request. As a side-effect this will grow the size +// of the pool. +func (p *DocumentMatchPool) Get() *DocumentMatch { + var rv *DocumentMatch + if len(p.avail) > 0 { + rv, p.avail = p.avail[len(p.avail)-1], p.avail[:len(p.avail)-1] + } else { + rv = p.TooSmall(p) + } + return rv +} + +// Put returns a DocumentMatch to the pool +func (p *DocumentMatchPool) Put(d *DocumentMatch) { + if d == nil { + return + } + // reset DocumentMatch before returning it to available pool + d.Reset() + p.avail = append(p.avail, d) +} diff --git a/search/scorers/scorer_conjunction.go b/search/scorers/scorer_conjunction.go index b2c34ba0..c941c69e 100644 --- a/search/scorers/scorer_conjunction.go +++ b/search/scorers/scorer_conjunction.go @@ -23,11 +23,7 @@ func NewConjunctionQueryScorer(explain bool) *ConjunctionQueryScorer { } } -func (s *ConjunctionQueryScorer) Score(constituents []*search.DocumentMatch) *search.DocumentMatch { - rv := search.DocumentMatch{ - IndexInternalID: constituents[0].IndexInternalID, - } - +func (s *ConjunctionQueryScorer) Score(ctx *search.SearchContext, constituents []*search.DocumentMatch) *search.DocumentMatch { var sum float64 var childrenExplanations []*search.Explanation if s.explain { @@ -44,16 +40,21 @@ func (s *ConjunctionQueryScorer) Score(constituents []*search.DocumentMatch) *se locations = append(locations, docMatch.Locations) } } - rv.Score = sum + newScore := sum + var newExpl *search.Explanation if s.explain { - rv.Expl = &search.Explanation{Value: sum, Message: "sum of:", Children: childrenExplanations} + newExpl = &search.Explanation{Value: sum, Message: "sum of:", Children: childrenExplanations} } + // reuse constituents[0] as the return value + rv := constituents[0] + rv.Score = newScore + rv.Expl = newExpl if len(locations) == 1 { rv.Locations = locations[0] } else if len(locations) > 1 { rv.Locations = search.MergeLocations(locations) } - return &rv + return rv } diff --git a/search/scorers/scorer_constant.go b/search/scorers/scorer_constant.go index af4eebf2..adbaf478 100644 --- a/search/scorers/scorer_constant.go +++ b/search/scorers/scorer_constant.go @@ -65,7 +65,7 @@ func (s *ConstantScorer) SetQueryNorm(qnorm float64) { } } -func (s *ConstantScorer) Score(id index.IndexInternalID) *search.DocumentMatch { +func (s *ConstantScorer) Score(ctx *search.SearchContext, id index.IndexInternalID) *search.DocumentMatch { var scoreExplanation *search.Explanation score := s.constant @@ -92,13 +92,12 @@ func (s *ConstantScorer) Score(id index.IndexInternalID) *search.DocumentMatch { } } - rv := search.DocumentMatch{ - IndexInternalID: id, - Score: score, - } + rv := ctx.DocumentMatchPool.Get() + rv.IndexInternalID = id + rv.Score = score if s.explain { rv.Expl = scoreExplanation } - return &rv + return rv } diff --git a/search/scorers/scorer_constant_test.go b/search/scorers/scorer_constant_test.go index d701217b..e1e6e9c6 100644 --- a/search/scorers/scorer_constant_test.go +++ b/search/scorers/scorer_constant_test.go @@ -52,7 +52,10 @@ func TestConstantScorer(t *testing.T) { } for _, test := range tests { - actual := scorer.Score(test.termMatch.ID) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(1), + } + actual := scorer.Score(ctx, test.termMatch.ID) if !reflect.DeepEqual(actual, test.result) { t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) @@ -108,7 +111,10 @@ func TestConstantScorerWithQueryNorm(t *testing.T) { } for _, test := range tests { - actual := scorer.Score(test.termMatch.ID) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(1), + } + actual := scorer.Score(ctx, test.termMatch.ID) if !reflect.DeepEqual(actual, test.result) { t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) diff --git a/search/scorers/scorer_disjunction.go b/search/scorers/scorer_disjunction.go index e73aa578..4868fc4f 100644 --- a/search/scorers/scorer_disjunction.go +++ b/search/scorers/scorer_disjunction.go @@ -25,11 +25,7 @@ func NewDisjunctionQueryScorer(explain bool) *DisjunctionQueryScorer { } } -func (s *DisjunctionQueryScorer) Score(constituents []*search.DocumentMatch, countMatch, countTotal int) *search.DocumentMatch { - rv := search.DocumentMatch{ - IndexInternalID: constituents[0].IndexInternalID, - } - +func (s *DisjunctionQueryScorer) Score(ctx *search.SearchContext, constituents []*search.DocumentMatch, countMatch, countTotal int) *search.DocumentMatch { var sum float64 var childrenExplanations []*search.Explanation if s.explain { @@ -53,19 +49,24 @@ func (s *DisjunctionQueryScorer) Score(constituents []*search.DocumentMatch, cou } coord := float64(countMatch) / float64(countTotal) - rv.Score = sum * coord + newScore := sum * coord + var newExpl *search.Explanation if s.explain { ce := make([]*search.Explanation, 2) ce[0] = rawExpl ce[1] = &search.Explanation{Value: coord, Message: fmt.Sprintf("coord(%d/%d)", countMatch, countTotal)} - rv.Expl = &search.Explanation{Value: rv.Score, Message: "product of:", Children: ce} + newExpl = &search.Explanation{Value: newScore, Message: "product of:", Children: ce} } + // reuse constituents[0] as the return value + rv := constituents[0] + rv.Score = newScore + rv.Expl = newExpl if len(locations) == 1 { rv.Locations = locations[0] } else if len(locations) > 1 { rv.Locations = search.MergeLocations(locations) } - return &rv + return rv } diff --git a/search/scorers/scorer_term.go b/search/scorers/scorer_term.go index 820842bf..4304d067 100644 --- a/search/scorers/scorer_term.go +++ b/search/scorers/scorer_term.go @@ -83,7 +83,7 @@ func (s *TermQueryScorer) SetQueryNorm(qnorm float64) { } } -func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc, preAllocated *search.DocumentMatch) *search.DocumentMatch { +func (s *TermQueryScorer) Score(ctx *search.SearchContext, termMatch *index.TermFieldDoc) *search.DocumentMatch { var scoreExplanation *search.Explanation // need to compute score @@ -128,10 +128,7 @@ func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc, preAllocated *sea } } - rv := preAllocated - if rv == nil { - rv = &search.DocumentMatch{} - } + rv := ctx.DocumentMatchPool.Get() rv.IndexInternalID = append(rv.IndexInternalID, termMatch.ID...) rv.Score = score if s.explain { diff --git a/search/scorers/scorer_term_test.go b/search/scorers/scorer_term_test.go index 5fde2df3..0241d163 100644 --- a/search/scorers/scorer_term_test.go +++ b/search/scorers/scorer_term_test.go @@ -144,7 +144,10 @@ func TestTermScorer(t *testing.T) { } for _, test := range tests { - actual := scorer.Score(test.termMatch, nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(1), + } + actual := scorer.Score(ctx, test.termMatch) if !reflect.DeepEqual(actual, test.result) { t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) @@ -231,7 +234,10 @@ func TestTermScorerWithQueryNorm(t *testing.T) { } for _, test := range tests { - actual := scorer.Score(test.termMatch, nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(1), + } + actual := scorer.Score(ctx, test.termMatch) if !reflect.DeepEqual(actual, test.result) { t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) diff --git a/search/search.go b/search/search.go index 984b1464..22abc064 100644 --- a/search/search.go +++ b/search/search.go @@ -106,11 +106,18 @@ func (c DocumentMatchCollection) Swap(i, j int) { c[i], c[j] = c[j], c[i] } func (c DocumentMatchCollection) Less(i, j int) bool { return c[i].Score > c[j].Score } type Searcher interface { - Next(preAllocated *DocumentMatch) (*DocumentMatch, error) - Advance(ID index.IndexInternalID, preAllocated *DocumentMatch) (*DocumentMatch, error) + Next(ctx *SearchContext) (*DocumentMatch, error) + Advance(ctx *SearchContext, ID index.IndexInternalID) (*DocumentMatch, error) Close() error Weight() float64 SetQueryNorm(float64) Count() uint64 Min() int + + DocumentMatchPoolSize() int +} + +// SearchContext represents the context around a single search +type SearchContext struct { + DocumentMatchPool *DocumentMatchPool } diff --git a/search/searchers/search_boolean.go b/search/searchers/search_boolean.go index dbd401ba..a66ba44f 100644 --- a/search/searchers/search_boolean.go +++ b/search/searchers/search_boolean.go @@ -66,25 +66,34 @@ func (s *BooleanSearcher) computeQueryNorm() { } } -func (s *BooleanSearcher) initSearchers() error { +func (s *BooleanSearcher) initSearchers(ctx *search.SearchContext) error { var err error // get all searchers pointing at their first match if s.mustSearcher != nil { - s.currMust, err = s.mustSearcher.Next(nil) + if s.currMust != nil { + ctx.DocumentMatchPool.Put(s.currMust) + } + s.currMust, err = s.mustSearcher.Next(ctx) if err != nil { return err } } if s.shouldSearcher != nil { - s.currShould, err = s.shouldSearcher.Next(nil) + if s.currShould != nil { + ctx.DocumentMatchPool.Put(s.currShould) + } + s.currShould, err = s.shouldSearcher.Next(ctx) if err != nil { return err } } if s.mustNotSearcher != nil { - s.currMustNot, err = s.mustNotSearcher.Next(nil) + if s.currMustNot != nil { + ctx.DocumentMatchPool.Put(s.currMustNot) + } + s.currMustNot, err = s.mustNotSearcher.Next(ctx) if err != nil { return err } @@ -102,16 +111,22 @@ func (s *BooleanSearcher) initSearchers() error { return nil } -func (s *BooleanSearcher) advanceNextMust() error { +func (s *BooleanSearcher) advanceNextMust(ctx *search.SearchContext, skipReturn *search.DocumentMatch) error { var err error if s.mustSearcher != nil { - s.currMust, err = s.mustSearcher.Next(nil) + if s.currMust != skipReturn { + ctx.DocumentMatchPool.Put(s.currMust) + } + s.currMust, err = s.mustSearcher.Next(ctx) if err != nil { return err } } else if s.mustSearcher == nil { - s.currShould, err = s.shouldSearcher.Next(nil) + if s.currShould != skipReturn { + ctx.DocumentMatchPool.Put(s.currShould) + } + s.currShould, err = s.shouldSearcher.Next(ctx) if err != nil { return err } @@ -148,10 +163,10 @@ func (s *BooleanSearcher) SetQueryNorm(qnorm float64) { } } -func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *BooleanSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { if !s.initialized { - err := s.initSearchers() + err := s.initSearchers(ctx) if err != nil { return nil, err } @@ -162,14 +177,17 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu for s.currentID != nil { if s.currMustNot != nil && s.currMustNot.IndexInternalID.Compare(s.currentID) < 0 { + if s.currMustNot != nil { + ctx.DocumentMatchPool.Put(s.currMustNot) + } // advance must not searcher to our candidate entry - s.currMustNot, err = s.mustNotSearcher.Advance(s.currentID, nil) + s.currMustNot, err = s.mustNotSearcher.Advance(ctx, s.currentID) if err != nil { return nil, err } if s.currMustNot != nil && s.currMustNot.IndexInternalID.Equals(s.currentID) { // the candidate is excluded - err = s.advanceNextMust() + err = s.advanceNextMust(ctx, nil) if err != nil { return nil, err } @@ -177,7 +195,7 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu } } else if s.currMustNot != nil && s.currMustNot.IndexInternalID.Equals(s.currentID) { // the candidate is excluded - err = s.advanceNextMust() + err = s.advanceNextMust(ctx, nil) if err != nil { return nil, err } @@ -186,7 +204,10 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu if s.currShould != nil && s.currShould.IndexInternalID.Compare(s.currentID) < 0 { // advance should searcher to our candidate entry - s.currShould, err = s.shouldSearcher.Advance(s.currentID, nil) + if s.currShould != nil { + ctx.DocumentMatchPool.Put(s.currShould) + } + s.currShould, err = s.shouldSearcher.Advance(ctx, s.currentID) if err != nil { return nil, err } @@ -203,16 +224,16 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu s.currShould, } } - rv = s.scorer.Score(cons) - err = s.advanceNextMust() + rv = s.scorer.Score(ctx, cons) + err = s.advanceNextMust(ctx, rv) if err != nil { return nil, err } break } else if s.shouldSearcher.Min() == 0 { // match is OK anyway - rv = s.scorer.Score([]*search.DocumentMatch{s.currMust}) - err = s.advanceNextMust() + rv = s.scorer.Score(ctx, []*search.DocumentMatch{s.currMust}) + err = s.advanceNextMust(ctx, rv) if err != nil { return nil, err } @@ -231,23 +252,23 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu s.currShould, } } - rv = s.scorer.Score(cons) - err = s.advanceNextMust() + rv = s.scorer.Score(ctx, cons) + err = s.advanceNextMust(ctx, rv) if err != nil { return nil, err } break } else if s.shouldSearcher == nil || s.shouldSearcher.Min() == 0 { // match is OK anyway - rv = s.scorer.Score([]*search.DocumentMatch{s.currMust}) - err = s.advanceNextMust() + rv = s.scorer.Score(ctx, []*search.DocumentMatch{s.currMust}) + err = s.advanceNextMust(ctx, rv) if err != nil { return nil, err } break } - err = s.advanceNextMust() + err = s.advanceNextMust(ctx, nil) if err != nil { return nil, err } @@ -255,10 +276,10 @@ func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docu return rv, nil } -func (s *BooleanSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *BooleanSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { if !s.initialized { - err := s.initSearchers() + err := s.initSearchers(ctx) if err != nil { return nil, err } @@ -266,19 +287,28 @@ func (s *BooleanSearcher) Advance(ID index.IndexInternalID, preAllocated *search var err error if s.mustSearcher != nil { - s.currMust, err = s.mustSearcher.Advance(ID, nil) + if s.currMust != nil { + ctx.DocumentMatchPool.Put(s.currMust) + } + s.currMust, err = s.mustSearcher.Advance(ctx, ID) if err != nil { return nil, err } } if s.shouldSearcher != nil { - s.currShould, err = s.shouldSearcher.Advance(ID, nil) + if s.currShould != nil { + ctx.DocumentMatchPool.Put(s.currShould) + } + s.currShould, err = s.shouldSearcher.Advance(ctx, ID) if err != nil { return nil, err } } if s.mustNotSearcher != nil { - s.currMustNot, err = s.mustNotSearcher.Advance(ID, nil) + if s.currMustNot != nil { + ctx.DocumentMatchPool.Put(s.currMustNot) + } + s.currMustNot, err = s.mustNotSearcher.Advance(ctx, ID) if err != nil { return nil, err } @@ -292,7 +322,7 @@ func (s *BooleanSearcher) Advance(ID index.IndexInternalID, preAllocated *search s.currentID = nil } - return s.Next(preAllocated) + return s.Next(ctx) } func (s *BooleanSearcher) Count() uint64 { @@ -333,3 +363,17 @@ func (s *BooleanSearcher) Close() error { func (s *BooleanSearcher) Min() int { return 0 } + +func (s *BooleanSearcher) DocumentMatchPoolSize() int { + rv := 3 + if s.mustSearcher != nil { + rv += s.mustSearcher.DocumentMatchPoolSize() + } + if s.shouldSearcher != nil { + rv += s.shouldSearcher.DocumentMatchPoolSize() + } + if s.mustNotSearcher != nil { + rv += s.mustNotSearcher.DocumentMatchPoolSize() + } + return rv +} diff --git a/search/searchers/search_boolean_test.go b/search/searchers/search_boolean_test.go index 7f9f6dd2..ddd31d2e 100644 --- a/search/searchers/search_boolean_test.go +++ b/search/searchers/search_boolean_test.go @@ -343,7 +343,10 @@ func TestBooleanSearch(t *testing.T) { } }() - next, err := test.searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()), + } + next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { @@ -355,7 +358,8 @@ func TestBooleanSearch(t *testing.T) { t.Logf("scoring explanation: %s", next.Expl) } } - next, err = test.searcher.Next(nil) + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) i++ } if err != nil { diff --git a/search/searchers/search_conjunction.go b/search/searchers/search_conjunction.go index 8e8d967a..c773f6a5 100644 --- a/search/searchers/search_conjunction.go +++ b/search/searchers/search_conjunction.go @@ -63,11 +63,14 @@ func (s *ConjunctionSearcher) computeQueryNorm() { } } -func (s *ConjunctionSearcher) initSearchers() error { +func (s *ConjunctionSearcher) initSearchers(ctx *search.SearchContext) error { var err error // get all searchers pointing at their first match for i, termSearcher := range s.searchers { - s.currs[i], err = termSearcher.Next(nil) + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = termSearcher.Next(ctx) if err != nil { return err } @@ -99,9 +102,9 @@ func (s *ConjunctionSearcher) SetQueryNorm(qnorm float64) { } } -func (s *ConjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *ConjunctionSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { if !s.initialized { - err := s.initSearchers() + err := s.initSearchers(ctx) if err != nil { return nil, err } @@ -117,7 +120,10 @@ OUTER: continue OUTER } // this reader doesn't have the currentID, try to advance - s.currs[i], err = termSearcher.Advance(s.currentID, nil) + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = termSearcher.Advance(ctx, s.currentID) if err != nil { return nil, err } @@ -137,40 +143,51 @@ OUTER: } } // if we get here, a doc matched all readers, sum the score and add it - rv = s.scorer.Score(s.currs) + rv = s.scorer.Score(ctx, s.currs) - // prepare for next entry - s.currs[0], err = s.searchers[0].Next(nil) - if err != nil { - return nil, err + // we know all the searchers are pointing at the same thing + // so they all need to be advanced + for i, termSearcher := range s.searchers { + if s.currs[i] != rv { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = termSearcher.Next(ctx) + if err != nil { + return nil, err + } } + if s.currs[0] == nil { s.currentID = nil } else { s.currentID = s.currs[0].IndexInternalID } + // don't continue now, wait for the next call to Next() break } return rv, nil } -func (s *ConjunctionSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *ConjunctionSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { if !s.initialized { - err := s.initSearchers() + err := s.initSearchers(ctx) if err != nil { return nil, err } } var err error for i, searcher := range s.searchers { - s.currs[i], err = searcher.Advance(ID, nil) + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = searcher.Advance(ctx, ID) if err != nil { return nil, err } } s.currentID = ID - return s.Next(preAllocated) + return s.Next(ctx) } func (s *ConjunctionSearcher) Count() uint64 { @@ -195,3 +212,11 @@ func (s *ConjunctionSearcher) Close() error { func (s *ConjunctionSearcher) Min() int { return 0 } + +func (s *ConjunctionSearcher) DocumentMatchPoolSize() int { + rv := len(s.currs) + for _, s := range s.searchers { + rv += s.DocumentMatchPoolSize() + } + return rv +} diff --git a/search/searchers/search_conjunction_test.go b/search/searchers/search_conjunction_test.go index 5e9f80ee..227a70ba 100644 --- a/search/searchers/search_conjunction_test.go +++ b/search/searchers/search_conjunction_test.go @@ -188,7 +188,10 @@ func TestConjunctionSearch(t *testing.T) { } }() - next, err := test.searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(10), + } + next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { @@ -200,7 +203,7 @@ func TestConjunctionSearch(t *testing.T) { t.Logf("scoring explanation: %s", next.Expl) } } - next, err = test.searcher.Next(nil) + next, err = test.searcher.Next(ctx) i++ } if err != nil { diff --git a/search/searchers/search_disjunction.go b/search/searchers/search_disjunction.go index f7286350..65a42d81 100644 --- a/search/searchers/search_disjunction.go +++ b/search/searchers/search_disjunction.go @@ -83,11 +83,14 @@ func (s *DisjunctionSearcher) computeQueryNorm() { } } -func (s *DisjunctionSearcher) initSearchers() error { +func (s *DisjunctionSearcher) initSearchers(ctx *search.SearchContext) error { var err error // get all searchers pointing at their first match for i, termSearcher := range s.searchers { - s.currs[i], err = termSearcher.Next(nil) + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = termSearcher.Next(ctx) if err != nil { return err } @@ -122,9 +125,9 @@ func (s *DisjunctionSearcher) SetQueryNorm(qnorm float64) { } } -func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *DisjunctionSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { if !s.initialized { - err := s.initSearchers() + err := s.initSearchers(ctx) if err != nil { return nil, err } @@ -144,7 +147,7 @@ func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search. if len(matching) >= int(s.min) { found = true // score this match - rv = s.scorer.Score(matching, len(matching), len(s.searchers)) + rv = s.scorer.Score(ctx, matching, len(matching), len(s.searchers)) } // reset matching @@ -153,7 +156,10 @@ func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search. for i, curr := range s.currs { if curr != nil && curr.IndexInternalID.Equals(s.currentID) { searcher := s.searchers[i] - s.currs[i], err = searcher.Next(nil) + if s.currs[i] != rv { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = searcher.Next(ctx) if err != nil { return nil, err } @@ -164,9 +170,9 @@ func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search. return rv, nil } -func (s *DisjunctionSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *DisjunctionSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { if !s.initialized { - err := s.initSearchers() + err := s.initSearchers(ctx) if err != nil { return nil, err } @@ -174,7 +180,10 @@ func (s *DisjunctionSearcher) Advance(ID index.IndexInternalID, preAllocated *se // get all searchers pointing at their first match var err error for i, termSearcher := range s.searchers { - s.currs[i], err = termSearcher.Advance(ID, nil) + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = termSearcher.Advance(ctx, ID) if err != nil { return nil, err } @@ -182,7 +191,7 @@ func (s *DisjunctionSearcher) Advance(ID index.IndexInternalID, preAllocated *se s.currentID = s.nextSmallestID() - return s.Next(preAllocated) + return s.Next(ctx) } func (s *DisjunctionSearcher) Count() uint64 { @@ -207,3 +216,11 @@ func (s *DisjunctionSearcher) Close() error { func (s *DisjunctionSearcher) Min() int { return int(s.min) // FIXME just make this an int } + +func (s *DisjunctionSearcher) DocumentMatchPoolSize() int { + rv := len(s.currs) + for _, s := range s.searchers { + rv += s.DocumentMatchPoolSize() + } + return rv +} diff --git a/search/searchers/search_disjunction_test.go b/search/searchers/search_disjunction_test.go index 83ba98fb..5f194065 100644 --- a/search/searchers/search_disjunction_test.go +++ b/search/searchers/search_disjunction_test.go @@ -109,7 +109,10 @@ func TestDisjunctionSearch(t *testing.T) { } }() - next, err := test.searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()), + } + next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { @@ -121,7 +124,8 @@ func TestDisjunctionSearch(t *testing.T) { t.Logf("scoring explanation: %s", next.Expl) } } - next, err = test.searcher.Next(nil) + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) i++ } if err != nil { @@ -159,7 +163,10 @@ func TestDisjunctionAdvance(t *testing.T) { t.Fatal(err) } - match, err := martyOrDustinSearcher.Advance(index.IndexInternalID("3"), nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(martyOrDustinSearcher.DocumentMatchPoolSize()), + } + match, err := martyOrDustinSearcher.Advance(ctx, index.IndexInternalID("3")) if err != nil { t.Errorf("unexpected error: %v", err) } diff --git a/search/searchers/search_docid.go b/search/searchers/search_docid.go index 8061f25b..33b9b4c9 100644 --- a/search/searchers/search_docid.go +++ b/search/searchers/search_docid.go @@ -49,7 +49,7 @@ func (s *DocIDSearcher) SetQueryNorm(qnorm float64) { s.scorer.SetQueryNorm(qnorm) } -func (s *DocIDSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *DocIDSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { docidMatch, err := s.reader.Next() if err != nil { return nil, err @@ -58,11 +58,11 @@ func (s *DocIDSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docume return nil, nil } - docMatch := s.scorer.Score(docidMatch) + docMatch := s.scorer.Score(ctx, docidMatch) return docMatch, nil } -func (s *DocIDSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *DocIDSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { docidMatch, err := s.reader.Advance(ID) if err != nil { return nil, err @@ -71,7 +71,7 @@ func (s *DocIDSearcher) Advance(ID index.IndexInternalID, preAllocated *search.D return nil, nil } - docMatch := s.scorer.Score(docidMatch) + docMatch := s.scorer.Score(ctx, docidMatch) return docMatch, nil } @@ -82,3 +82,7 @@ func (s *DocIDSearcher) Close() error { func (s *DocIDSearcher) Min() int { return 0 } + +func (s *DocIDSearcher) DocumentMatchPoolSize() int { + return 1 +} diff --git a/search/searchers/search_docid_test.go b/search/searchers/search_docid_test.go index 050ef2e2..00fe3db9 100644 --- a/search/searchers/search_docid_test.go +++ b/search/searchers/search_docid_test.go @@ -16,6 +16,7 @@ import ( "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index/store/gtreap" "github.com/blevesearch/bleve/index/upside_down" + "github.com/blevesearch/bleve/search" ) func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { @@ -62,23 +63,29 @@ func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { } }() + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(searcher.DocumentMatchPoolSize()), + } + // Check the sequence for i, id := range wanted { - m, err := searcher.Next(nil) + m, err := searcher.Next(ctx) if err != nil { t.Fatal(err) } if !index.IndexInternalID(id).Equals(m.IndexInternalID) { t.Fatalf("expected %v at position %v, got %v", id, i, m.IndexInternalID) } + ctx.DocumentMatchPool.Put(m) } - m, err := searcher.Next(nil) + m, err := searcher.Next(ctx) if err != nil { t.Fatal(err) } if m != nil { t.Fatalf("expected nil past the end of the sequence, got %v", m.IndexInternalID) } + ctx.DocumentMatchPool.Put(m) // Check seeking for _, id := range wanted { @@ -87,24 +94,26 @@ func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { } before := id[:1] for _, target := range []string{before, id} { - m, err := searcher.Advance(index.IndexInternalID(target), nil) + m, err := searcher.Advance(ctx, index.IndexInternalID(target)) if err != nil { t.Fatal(err) } if m == nil || !m.IndexInternalID.Equals(index.IndexInternalID(id)) { t.Fatalf("advancing to %v returned %v instead of %v", before, m, id) } + ctx.DocumentMatchPool.Put(m) } } // Seek after the end of the sequence after := "zzz" - m, err = searcher.Advance(index.IndexInternalID(after), nil) + m, err = searcher.Advance(ctx, index.IndexInternalID(after)) if err != nil { t.Fatal(err) } if m != nil { t.Fatalf("advancing past the end of the sequence should return nil, got %v", m) } + ctx.DocumentMatchPool.Put(m) } func TestDocIDSearcherEmptySearchEmptyIndex(t *testing.T) { diff --git a/search/searchers/search_fuzzy.go b/search/searchers/search_fuzzy.go index 0b26c540..2cd44ebf 100644 --- a/search/searchers/search_fuzzy.go +++ b/search/searchers/search_fuzzy.go @@ -107,13 +107,13 @@ func (s *FuzzySearcher) SetQueryNorm(qnorm float64) { s.searcher.SetQueryNorm(qnorm) } -func (s *FuzzySearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - return s.searcher.Next(preAllocated) +func (s *FuzzySearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + return s.searcher.Next(ctx) } -func (s *FuzzySearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - return s.searcher.Advance(ID, preAllocated) +func (s *FuzzySearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + return s.searcher.Advance(ctx, ID) } func (s *FuzzySearcher) Close() error { @@ -123,3 +123,7 @@ func (s *FuzzySearcher) Close() error { func (s *FuzzySearcher) Min() int { return 0 } + +func (s *FuzzySearcher) DocumentMatchPoolSize() int { + return s.searcher.DocumentMatchPoolSize() +} diff --git a/search/searchers/search_fuzzy_test.go b/search/searchers/search_fuzzy_test.go index 409a5069..b4469666 100644 --- a/search/searchers/search_fuzzy_test.go +++ b/search/searchers/search_fuzzy_test.go @@ -106,7 +106,10 @@ func TestFuzzySearch(t *testing.T) { } }() - next, err := test.searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()), + } + next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { @@ -118,7 +121,8 @@ func TestFuzzySearch(t *testing.T) { t.Logf("scoring explanation: %s", next.Expl) } } - next, err = test.searcher.Next(nil) + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) i++ } if err != nil { diff --git a/search/searchers/search_match_all.go b/search/searchers/search_match_all.go index 38ed6ab1..9834d31a 100644 --- a/search/searchers/search_match_all.go +++ b/search/searchers/search_match_all.go @@ -46,7 +46,7 @@ func (s *MatchAllSearcher) SetQueryNorm(qnorm float64) { s.scorer.SetQueryNorm(qnorm) } -func (s *MatchAllSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *MatchAllSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { id, err := s.reader.Next() if err != nil { return nil, err @@ -57,13 +57,13 @@ func (s *MatchAllSearcher) Next(preAllocated *search.DocumentMatch) (*search.Doc } // score match - docMatch := s.scorer.Score(id) + docMatch := s.scorer.Score(ctx, id) // return doc match return docMatch, nil } -func (s *MatchAllSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *MatchAllSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { id, err := s.reader.Advance(ID) if err != nil { return nil, err @@ -74,7 +74,7 @@ func (s *MatchAllSearcher) Advance(ID index.IndexInternalID, preAllocated *searc } // score match - docMatch := s.scorer.Score(id) + docMatch := s.scorer.Score(ctx, id) // return doc match return docMatch, nil @@ -87,3 +87,7 @@ func (s *MatchAllSearcher) Close() error { func (s *MatchAllSearcher) Min() int { return 0 } + +func (s *MatchAllSearcher) DocumentMatchPoolSize() int { + return 1 +} diff --git a/search/searchers/search_match_all_test.go b/search/searchers/search_match_all_test.go index f265e002..26e8a3c2 100644 --- a/search/searchers/search_match_all_test.go +++ b/search/searchers/search_match_all_test.go @@ -110,7 +110,10 @@ func TestMatchAllSearch(t *testing.T) { } }() - next, err := test.searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()), + } + next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { @@ -122,7 +125,8 @@ func TestMatchAllSearch(t *testing.T) { t.Logf("scoring explanation: %s", next.Expl) } } - next, err = test.searcher.Next(nil) + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) i++ } if err != nil { diff --git a/search/searchers/search_match_none.go b/search/searchers/search_match_none.go index 08eefb03..c3159dcf 100644 --- a/search/searchers/search_match_none.go +++ b/search/searchers/search_match_none.go @@ -36,11 +36,11 @@ func (s *MatchNoneSearcher) SetQueryNorm(qnorm float64) { } -func (s *MatchNoneSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *MatchNoneSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { return nil, nil } -func (s *MatchNoneSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *MatchNoneSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { return nil, nil } @@ -51,3 +51,7 @@ func (s *MatchNoneSearcher) Close() error { func (s *MatchNoneSearcher) Min() int { return 0 } + +func (s *MatchNoneSearcher) DocumentMatchPoolSize() int { + return 0 +} diff --git a/search/searchers/search_match_none_test.go b/search/searchers/search_match_none_test.go index f704d29c..90ec526c 100644 --- a/search/searchers/search_match_none_test.go +++ b/search/searchers/search_match_none_test.go @@ -51,7 +51,10 @@ func TestMatchNoneSearch(t *testing.T) { } }() - next, err := test.searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()), + } + next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { @@ -63,7 +66,8 @@ func TestMatchNoneSearch(t *testing.T) { t.Logf("scoring explanation: %s", next.Expl) } } - next, err = test.searcher.Next(nil) + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) i++ } if err != nil { diff --git a/search/searchers/search_numeric_range.go b/search/searchers/search_numeric_range.go index 6f236dd5..03cd4959 100644 --- a/search/searchers/search_numeric_range.go +++ b/search/searchers/search_numeric_range.go @@ -96,12 +96,12 @@ func (s *NumericRangeSearcher) SetQueryNorm(qnorm float64) { s.searcher.SetQueryNorm(qnorm) } -func (s *NumericRangeSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - return s.searcher.Next(preAllocated) +func (s *NumericRangeSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + return s.searcher.Next(ctx) } -func (s *NumericRangeSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - return s.searcher.Advance(ID, preAllocated) +func (s *NumericRangeSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + return s.searcher.Advance(ctx, ID) } func (s *NumericRangeSearcher) Close() error { @@ -215,3 +215,7 @@ func newRangeBytes(minBytes, maxBytes []byte) *termRange { func (s *NumericRangeSearcher) Min() int { return 0 } + +func (s *NumericRangeSearcher) DocumentMatchPoolSize() int { + return s.searcher.DocumentMatchPoolSize() +} diff --git a/search/searchers/search_phrase.go b/search/searchers/search_phrase.go index 34a31ed9..6456420f 100644 --- a/search/searchers/search_phrase.go +++ b/search/searchers/search_phrase.go @@ -52,11 +52,11 @@ func (s *PhraseSearcher) computeQueryNorm() { } } -func (s *PhraseSearcher) initSearchers() error { +func (s *PhraseSearcher) initSearchers(ctx *search.SearchContext) error { var err error // get all searchers pointing at their first match if s.mustSearcher != nil { - s.currMust, err = s.mustSearcher.Next(nil) + s.currMust, err = s.mustSearcher.Next(ctx) if err != nil { return err } @@ -66,11 +66,11 @@ func (s *PhraseSearcher) initSearchers() error { return nil } -func (s *PhraseSearcher) advanceNextMust() error { +func (s *PhraseSearcher) advanceNextMust(ctx *search.SearchContext) error { var err error if s.mustSearcher != nil { - s.currMust, err = s.mustSearcher.Next(nil) + s.currMust, err = s.mustSearcher.Next(ctx) if err != nil { return err } @@ -90,9 +90,9 @@ func (s *PhraseSearcher) SetQueryNorm(qnorm float64) { s.mustSearcher.SetQueryNorm(qnorm) } -func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *PhraseSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { if !s.initialized { - err := s.initSearchers() + err := s.initSearchers(ctx) if err != nil { return nil, err } @@ -144,14 +144,14 @@ func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docum // return match rv = s.currMust rv.Locations = rvftlm - err := s.advanceNextMust() + err := s.advanceNextMust(ctx) if err != nil { return nil, err } return rv, nil } - err := s.advanceNextMust() + err := s.advanceNextMust(ctx) if err != nil { return nil, err } @@ -160,19 +160,19 @@ func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatch) (*search.Docum return nil, nil } -func (s *PhraseSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *PhraseSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { if !s.initialized { - err := s.initSearchers() + err := s.initSearchers(ctx) if err != nil { return nil, err } } var err error - s.currMust, err = s.mustSearcher.Advance(ID, nil) + s.currMust, err = s.mustSearcher.Advance(ctx, ID) if err != nil { return nil, err } - return s.Next(preAllocated) + return s.Next(ctx) } func (s *PhraseSearcher) Count() uint64 { @@ -195,3 +195,7 @@ func (s *PhraseSearcher) Close() error { func (s *PhraseSearcher) Min() int { return 0 } + +func (s *PhraseSearcher) DocumentMatchPoolSize() int { + return s.mustSearcher.DocumentMatchPoolSize() + 1 +} diff --git a/search/searchers/search_phrase_test.go b/search/searchers/search_phrase_test.go index 94380f6e..8c9a76d2 100644 --- a/search/searchers/search_phrase_test.go +++ b/search/searchers/search_phrase_test.go @@ -69,7 +69,10 @@ func TestPhraseSearch(t *testing.T) { } }() - next, err := test.searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()), + } + next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { @@ -81,7 +84,8 @@ func TestPhraseSearch(t *testing.T) { t.Logf("scoring explanation: %s", next.Expl) } } - next, err = test.searcher.Next(nil) + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) i++ } if err != nil { diff --git a/search/searchers/search_regexp.go b/search/searchers/search_regexp.go index 8e6ce52b..3ce61d82 100644 --- a/search/searchers/search_regexp.go +++ b/search/searchers/search_regexp.go @@ -106,13 +106,13 @@ func (s *RegexpSearcher) SetQueryNorm(qnorm float64) { s.searcher.SetQueryNorm(qnorm) } -func (s *RegexpSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - return s.searcher.Next(preAllocated) +func (s *RegexpSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + return s.searcher.Next(ctx) } -func (s *RegexpSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - return s.searcher.Advance(ID, preAllocated) +func (s *RegexpSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + return s.searcher.Advance(ctx, ID) } func (s *RegexpSearcher) Close() error { @@ -122,3 +122,7 @@ func (s *RegexpSearcher) Close() error { func (s *RegexpSearcher) Min() int { return 0 } + +func (s *RegexpSearcher) DocumentMatchPoolSize() int { + return s.searcher.DocumentMatchPoolSize() +} diff --git a/search/searchers/search_regexp_test.go b/search/searchers/search_regexp_test.go index 489297e2..f2859cb9 100644 --- a/search/searchers/search_regexp_test.go +++ b/search/searchers/search_regexp_test.go @@ -86,7 +86,10 @@ func TestRegexpSearch(t *testing.T) { } }() - next, err := test.searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize()), + } + next, err := test.searcher.Next(ctx) i := 0 for err == nil && next != nil { if i < len(test.results) { @@ -98,7 +101,8 @@ func TestRegexpSearch(t *testing.T) { t.Logf("scoring explanation: %s", next.Expl) } } - next, err = test.searcher.Next(nil) + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) i++ } if err != nil { diff --git a/search/searchers/search_term.go b/search/searchers/search_term.go index ac9855a7..3879723c 100644 --- a/search/searchers/search_term.go +++ b/search/searchers/search_term.go @@ -53,7 +53,7 @@ func (s *TermSearcher) SetQueryNorm(qnorm float64) { s.scorer.SetQueryNorm(qnorm) } -func (s *TermSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *TermSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { termMatch, err := s.reader.Next(s.tfd.Reset()) if err != nil { return nil, err @@ -64,13 +64,13 @@ func (s *TermSearcher) Next(preAllocated *search.DocumentMatch) (*search.Documen } // score match - docMatch := s.scorer.Score(termMatch, preAllocated) + docMatch := s.scorer.Score(ctx, termMatch) // return doc match return docMatch, nil } -func (s *TermSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { +func (s *TermSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { termMatch, err := s.reader.Advance(ID, s.tfd.Reset()) if err != nil { return nil, err @@ -81,7 +81,7 @@ func (s *TermSearcher) Advance(ID index.IndexInternalID, preAllocated *search.Do } // score match - docMatch := s.scorer.Score(termMatch, preAllocated) + docMatch := s.scorer.Score(ctx, termMatch) // return doc match return docMatch, nil @@ -94,3 +94,7 @@ func (s *TermSearcher) Close() error { func (s *TermSearcher) Min() int { return 0 } + +func (s *TermSearcher) DocumentMatchPoolSize() int { + return 1 +} diff --git a/search/searchers/search_term_prefix.go b/search/searchers/search_term_prefix.go index 80f22e91..35caf717 100644 --- a/search/searchers/search_term_prefix.go +++ b/search/searchers/search_term_prefix.go @@ -70,13 +70,13 @@ func (s *TermPrefixSearcher) SetQueryNorm(qnorm float64) { s.searcher.SetQueryNorm(qnorm) } -func (s *TermPrefixSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - return s.searcher.Next(preAllocated) +func (s *TermPrefixSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + return s.searcher.Next(ctx) } -func (s *TermPrefixSearcher) Advance(ID index.IndexInternalID, preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) { - return s.searcher.Advance(ID, preAllocated) +func (s *TermPrefixSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + return s.searcher.Advance(ctx, ID) } func (s *TermPrefixSearcher) Close() error { @@ -86,3 +86,7 @@ func (s *TermPrefixSearcher) Close() error { func (s *TermPrefixSearcher) Min() int { return 0 } + +func (s *TermPrefixSearcher) DocumentMatchPoolSize() int { + return s.searcher.DocumentMatchPoolSize() +} diff --git a/search/searchers/search_term_test.go b/search/searchers/search_term_test.go index e60d9184..c94803e1 100644 --- a/search/searchers/search_term_test.go +++ b/search/searchers/search_term_test.go @@ -17,6 +17,7 @@ import ( "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index/store/gtreap" "github.com/blevesearch/bleve/index/upside_down" + "github.com/blevesearch/bleve/search" ) func TestTermSearcher(t *testing.T) { @@ -163,14 +164,18 @@ func TestTermSearcher(t *testing.T) { t.Errorf("expected count of 9, got %d", searcher.Count()) } - docMatch, err := searcher.Next(nil) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(1), + } + docMatch, err := searcher.Next(ctx) if err != nil { t.Errorf("expected result, got %v", err) } if !docMatch.IndexInternalID.Equals(index.IndexInternalID("a")) { t.Errorf("expected result ID to be 'a', got '%s", docMatch.IndexInternalID) } - docMatch, err = searcher.Advance(index.IndexInternalID("c"), nil) + ctx.DocumentMatchPool.Put(docMatch) + docMatch, err = searcher.Advance(ctx, index.IndexInternalID("c")) if err != nil { t.Errorf("expected result, got %v", err) } @@ -179,7 +184,8 @@ func TestTermSearcher(t *testing.T) { } // try advancing past end - docMatch, err = searcher.Advance(index.IndexInternalID("z"), nil) + ctx.DocumentMatchPool.Put(docMatch) + docMatch, err = searcher.Advance(ctx, index.IndexInternalID("z")) if err != nil { t.Fatal(err) } @@ -188,7 +194,8 @@ func TestTermSearcher(t *testing.T) { } // try pushing next past end - docMatch, err = searcher.Next(nil) + ctx.DocumentMatchPool.Put(docMatch) + docMatch, err = searcher.Next(ctx) if err != nil { t.Fatal(err) } From 9333bac2c83ec24e09d2d3653acd5d221353c5e4 Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Tue, 9 Aug 2016 11:35:12 -0400 Subject: [PATCH 13/13] added test case for DocumentMatchPool --- search/pool_test.go | 65 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 search/pool_test.go diff --git a/search/pool_test.go b/search/pool_test.go new file mode 100644 index 00000000..875f607c --- /dev/null +++ b/search/pool_test.go @@ -0,0 +1,65 @@ +// Copyright (c) 2013 Couchbase, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file +// except in compliance with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed under the +// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +// either express or implied. See the License for the specific language governing permissions +// and limitations under the License. + +package search + +import "testing" + +func TestDocumentMatchPool(t *testing.T) { + + tooManyCalled := false + + // create a pool + dmp := NewDocumentMatchPool(10) + dmp.TooSmall = func(inner *DocumentMatchPool) *DocumentMatch { + tooManyCalled = true + return &DocumentMatch{} + } + + // get 10 instances without returning + returned := make(DocumentMatchCollection, 10) + + for i := 0; i < 10; i++ { + returned[i] = dmp.Get() + if tooManyCalled { + t.Fatal("too many function called before expected") + } + } + + // get one more and see if too many function is called + extra := dmp.Get() + if !tooManyCalled { + t.Fatal("expected too many function to be called, but wasnt") + } + + // return the first 10 + for i := 0; i < 10; i++ { + dmp.Put(returned[i]) + } + + // check len and cap + if len(dmp.avail) != 10 { + t.Fatalf("expected 10 available, got %d", len(dmp.avail)) + } + if cap(dmp.avail) != 10 { + t.Fatalf("expected avail cap still 10, got %d", cap(dmp.avail)) + } + + // return the extra + dmp.Put(extra) + + // check len and cap grown to 11 + if len(dmp.avail) != 11 { + t.Fatalf("expected 11 available, got %d", len(dmp.avail)) + } + // cap grows, but not by 1 (append behavior) + if cap(dmp.avail) <= 10 { + t.Fatalf("expected avail cap mpore than 10, got %d", cap(dmp.avail)) + } +}