scorch includeFreq/Norm/Locs params for postingsList.Iterator API
This commit adds boolean flag params to the scorch PostingsList.Iterator() method, so that the caller can specify whether freq/norm/locs information is needed or not. Future changes can leverage these params for optimizations.
This commit is contained in:
parent
fc7584f5a0
commit
192621f402
|
@ -84,7 +84,7 @@ func (e *EmptyDictionaryIterator) Next() (*index.DictEntry, error) {
|
||||||
|
|
||||||
type EmptyPostingsList struct{}
|
type EmptyPostingsList struct{}
|
||||||
|
|
||||||
func (e *EmptyPostingsList) Iterator() PostingsIterator {
|
func (e *EmptyPostingsList) Iterator(includeFreq, includeNorm, includeLocations bool) PostingsIterator {
|
||||||
return &EmptyPostingsIterator{}
|
return &EmptyPostingsIterator{}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -78,7 +78,7 @@ func (p *PostingsList) Count() uint64 {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Iterator returns an iterator for this postings list
|
// Iterator returns an iterator for this postings list
|
||||||
func (p *PostingsList) Iterator() segment.PostingsIterator {
|
func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocations bool) segment.PostingsIterator {
|
||||||
return p.InitIterator(nil)
|
return p.InitIterator(nil)
|
||||||
}
|
}
|
||||||
func (p *PostingsList) InitIterator(prealloc *PostingsIterator) *PostingsIterator {
|
func (p *PostingsList) InitIterator(prealloc *PostingsIterator) *PostingsIterator {
|
||||||
|
|
|
@ -48,7 +48,7 @@ func TestEmpty(t *testing.T) {
|
||||||
t.Fatal("got nil postings list, expected non-nil")
|
t.Fatal("got nil postings list, expected non-nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
postingsItr := postingsList.Iterator()
|
postingsItr := postingsList.Iterator(true, true, true)
|
||||||
if postingsItr == nil {
|
if postingsItr == nil {
|
||||||
t.Fatal("got nil iterator, expected non-nil")
|
t.Fatal("got nil iterator, expected non-nil")
|
||||||
}
|
}
|
||||||
|
@ -211,7 +211,7 @@ func TestSingle(t *testing.T) {
|
||||||
t.Fatal("got nil postings list, expected non-nil")
|
t.Fatal("got nil postings list, expected non-nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
postingsItr := postingsList.Iterator()
|
postingsItr := postingsList.Iterator(true, true, true)
|
||||||
if postingsItr == nil {
|
if postingsItr == nil {
|
||||||
t.Fatal("got nil iterator, expected non-nil")
|
t.Fatal("got nil iterator, expected non-nil")
|
||||||
}
|
}
|
||||||
|
@ -257,7 +257,7 @@ func TestSingle(t *testing.T) {
|
||||||
t.Fatal("got nil postings list, expected non-nil")
|
t.Fatal("got nil postings list, expected non-nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
postingsItr = postingsList.Iterator()
|
postingsItr = postingsList.Iterator(true, true, true)
|
||||||
if postingsItr == nil {
|
if postingsItr == nil {
|
||||||
t.Fatal("got nil iterator, expected non-nil")
|
t.Fatal("got nil iterator, expected non-nil")
|
||||||
}
|
}
|
||||||
|
@ -325,7 +325,7 @@ func TestSingle(t *testing.T) {
|
||||||
t.Fatal("got nil postings list, expected non-nil")
|
t.Fatal("got nil postings list, expected non-nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
postingsItr = postingsList.Iterator()
|
postingsItr = postingsList.Iterator(true, true, true)
|
||||||
if postingsItr == nil {
|
if postingsItr == nil {
|
||||||
t.Fatal("got nil iterator, expected non-nil")
|
t.Fatal("got nil iterator, expected non-nil")
|
||||||
}
|
}
|
||||||
|
@ -394,7 +394,7 @@ func TestSingle(t *testing.T) {
|
||||||
t.Fatal("got nil postings list, expected non-nil")
|
t.Fatal("got nil postings list, expected non-nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
postingsItr = postingsList.Iterator()
|
postingsItr = postingsList.Iterator(true, true, true)
|
||||||
if postingsItr == nil {
|
if postingsItr == nil {
|
||||||
t.Fatal("got nil iterator, expected non-nil")
|
t.Fatal("got nil iterator, expected non-nil")
|
||||||
}
|
}
|
||||||
|
@ -638,7 +638,7 @@ func TestMultiple(t *testing.T) {
|
||||||
t.Fatal("got nil postings list, expected non-nil")
|
t.Fatal("got nil postings list, expected non-nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
postingsItr := postingsList.Iterator()
|
postingsItr := postingsList.Iterator(true, true, true)
|
||||||
if postingsItr == nil {
|
if postingsItr == nil {
|
||||||
t.Fatal("got nil iterator, expected non-nil")
|
t.Fatal("got nil iterator, expected non-nil")
|
||||||
}
|
}
|
||||||
|
@ -677,7 +677,7 @@ func TestMultiple(t *testing.T) {
|
||||||
t.Errorf("expected count from postings list to be 1, got %d", postingsListExcludingCount)
|
t.Errorf("expected count from postings list to be 1, got %d", postingsListExcludingCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
postingsItrExcluding := postingsListExcluding.Iterator()
|
postingsItrExcluding := postingsListExcluding.Iterator(true, true, true)
|
||||||
if postingsItr == nil {
|
if postingsItr == nil {
|
||||||
t.Fatal("got nil iterator, expected non-nil")
|
t.Fatal("got nil iterator, expected non-nil")
|
||||||
}
|
}
|
||||||
|
|
|
@ -55,7 +55,7 @@ type DictionaryIterator interface {
|
||||||
}
|
}
|
||||||
|
|
||||||
type PostingsList interface {
|
type PostingsList interface {
|
||||||
Iterator() PostingsIterator
|
Iterator(includeFreq, includeNorm, includeLocations bool) PostingsIterator
|
||||||
|
|
||||||
Size() int
|
Size() int
|
||||||
|
|
||||||
|
|
|
@ -303,7 +303,7 @@ func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap,
|
||||||
return nil, 0, err2
|
return nil, 0, err2
|
||||||
}
|
}
|
||||||
|
|
||||||
postItr = postings.iterator(postItr)
|
postItr = postings.iterator(true, true, true, postItr)
|
||||||
|
|
||||||
if fieldsSame {
|
if fieldsSame {
|
||||||
// can optimize by copying freq/norm/loc bytes directly
|
// can optimize by copying freq/norm/loc bytes directly
|
||||||
|
|
|
@ -332,8 +332,8 @@ func compareSegments(a, b *Segment) string {
|
||||||
fieldName, next.Term, aplist.Count(), bplist.Count()))
|
fieldName, next.Term, aplist.Count(), bplist.Count()))
|
||||||
}
|
}
|
||||||
|
|
||||||
apitr := aplist.Iterator()
|
apitr := aplist.Iterator(true, true, true)
|
||||||
bpitr := bplist.Iterator()
|
bpitr := bplist.Iterator(true, true, true)
|
||||||
if (apitr != nil) != (bpitr != nil) {
|
if (apitr != nil) != (bpitr != nil) {
|
||||||
rv = append(rv, fmt.Sprintf("field %s, term: %s, postingsList.Iterator() results different: %v %v",
|
rv = append(rv, fmt.Sprintf("field %s, term: %s, postingsList.Iterator() results different: %v %v",
|
||||||
fieldName, next.Term, apitr, bpitr))
|
fieldName, next.Term, apitr, bpitr))
|
||||||
|
|
|
@ -131,11 +131,12 @@ func (p *PostingsList) OrInto(receiver *roaring.Bitmap) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Iterator returns an iterator for this postings list
|
// Iterator returns an iterator for this postings list
|
||||||
func (p *PostingsList) Iterator() segment.PostingsIterator {
|
func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocations bool) segment.PostingsIterator {
|
||||||
return p.iterator(nil)
|
return p.iterator(includeFreq, includeNorm, includeLocations, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *PostingsList) iterator(rv *PostingsIterator) *PostingsIterator {
|
func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocations bool,
|
||||||
|
rv *PostingsIterator) *PostingsIterator {
|
||||||
if rv == nil {
|
if rv == nil {
|
||||||
rv = &PostingsIterator{}
|
rv = &PostingsIterator{}
|
||||||
} else {
|
} else {
|
||||||
|
@ -495,10 +496,10 @@ func (i *PostingsIterator) Next() (segment.Posting, error) {
|
||||||
if cap(i.nextLocs) >= int(rv.freq) {
|
if cap(i.nextLocs) >= int(rv.freq) {
|
||||||
i.nextLocs = i.nextLocs[0:rv.freq]
|
i.nextLocs = i.nextLocs[0:rv.freq]
|
||||||
} else {
|
} else {
|
||||||
i.nextLocs = make([]Location, rv.freq, rv.freq * 2)
|
i.nextLocs = make([]Location, rv.freq, rv.freq*2)
|
||||||
}
|
}
|
||||||
if cap(i.nextSegmentLocs) < int(rv.freq) {
|
if cap(i.nextSegmentLocs) < int(rv.freq) {
|
||||||
i.nextSegmentLocs = make([]segment.Location, rv.freq, rv.freq * 2)
|
i.nextSegmentLocs = make([]segment.Location, rv.freq, rv.freq*2)
|
||||||
}
|
}
|
||||||
rv.locs = i.nextSegmentLocs[0:rv.freq]
|
rv.locs = i.nextSegmentLocs[0:rv.freq]
|
||||||
for j := 0; j < int(rv.freq); j++ {
|
for j := 0; j < int(rv.freq); j++ {
|
||||||
|
|
|
@ -84,7 +84,7 @@ func TestOpen(t *testing.T) {
|
||||||
t.Fatal("got nil postings list, expected non-nil")
|
t.Fatal("got nil postings list, expected non-nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
postingsItr := postingsList.Iterator()
|
postingsItr := postingsList.Iterator(true, true, true)
|
||||||
if postingsItr == nil {
|
if postingsItr == nil {
|
||||||
t.Fatal("got nil iterator, expected non-nil")
|
t.Fatal("got nil iterator, expected non-nil")
|
||||||
}
|
}
|
||||||
|
@ -130,7 +130,7 @@ func TestOpen(t *testing.T) {
|
||||||
t.Fatal("got nil postings list, expected non-nil")
|
t.Fatal("got nil postings list, expected non-nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
postingsItr = postingsList.Iterator()
|
postingsItr = postingsList.Iterator(true, true, true)
|
||||||
if postingsItr == nil {
|
if postingsItr == nil {
|
||||||
t.Fatal("got nil iterator, expected non-nil")
|
t.Fatal("got nil iterator, expected non-nil")
|
||||||
}
|
}
|
||||||
|
@ -198,7 +198,7 @@ func TestOpen(t *testing.T) {
|
||||||
t.Fatal("got nil postings list, expected non-nil")
|
t.Fatal("got nil postings list, expected non-nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
postingsItr = postingsList.Iterator()
|
postingsItr = postingsList.Iterator(true, true, true)
|
||||||
if postingsItr == nil {
|
if postingsItr == nil {
|
||||||
t.Fatal("got nil iterator, expected non-nil")
|
t.Fatal("got nil iterator, expected non-nil")
|
||||||
}
|
}
|
||||||
|
@ -267,7 +267,7 @@ func TestOpen(t *testing.T) {
|
||||||
t.Fatal("got nil postings list, expected non-nil")
|
t.Fatal("got nil postings list, expected non-nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
postingsItr = postingsList.Iterator()
|
postingsItr = postingsList.Iterator(true, true, true)
|
||||||
if postingsItr == nil {
|
if postingsItr == nil {
|
||||||
t.Fatal("got nil iterator, expected non-nil")
|
t.Fatal("got nil iterator, expected non-nil")
|
||||||
}
|
}
|
||||||
|
@ -366,7 +366,7 @@ func TestOpenMulti(t *testing.T) {
|
||||||
t.Fatal("got nil postings list, expected non-nil")
|
t.Fatal("got nil postings list, expected non-nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
postingsItr := postingsList.Iterator()
|
postingsItr := postingsList.Iterator(true, true, true)
|
||||||
if postingsItr == nil {
|
if postingsItr == nil {
|
||||||
t.Fatal("got nil iterator, expected non-nil")
|
t.Fatal("got nil iterator, expected non-nil")
|
||||||
}
|
}
|
||||||
|
@ -405,7 +405,7 @@ func TestOpenMulti(t *testing.T) {
|
||||||
t.Errorf("expected count from postings list to be 1, got %d", postingsListExcludingCount)
|
t.Errorf("expected count from postings list to be 1, got %d", postingsListExcludingCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
postingsItrExcluding := postingsListExcluding.Iterator()
|
postingsItrExcluding := postingsListExcluding.Iterator(true, true, true)
|
||||||
if postingsItr == nil {
|
if postingsItr == nil {
|
||||||
t.Fatal("got nil iterator, expected non-nil")
|
t.Fatal("got nil iterator, expected non-nil")
|
||||||
}
|
}
|
||||||
|
@ -466,7 +466,7 @@ func TestOpenMultiWithTwoChunks(t *testing.T) {
|
||||||
t.Fatal("got nil postings list, expected non-nil")
|
t.Fatal("got nil postings list, expected non-nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
postingsItr := postingsList.Iterator()
|
postingsItr := postingsList.Iterator(true, true, true)
|
||||||
if postingsItr == nil {
|
if postingsItr == nil {
|
||||||
t.Fatal("got nil iterator, expected non-nil")
|
t.Fatal("got nil iterator, expected non-nil")
|
||||||
}
|
}
|
||||||
|
@ -500,7 +500,7 @@ func TestOpenMultiWithTwoChunks(t *testing.T) {
|
||||||
t.Fatal("got nil postings list, expected non-nil")
|
t.Fatal("got nil postings list, expected non-nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
postingsItrExcluding := postingsListExcluding.Iterator()
|
postingsItrExcluding := postingsListExcluding.Iterator(true, true, true)
|
||||||
if postingsItr == nil {
|
if postingsItr == nil {
|
||||||
t.Fatal("got nil iterator, expected non-nil")
|
t.Fatal("got nil iterator, expected non-nil")
|
||||||
}
|
}
|
||||||
|
|
|
@ -394,7 +394,7 @@ func (i *IndexSnapshot) TermFieldReader(term []byte, field string, includeFreq,
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
rv.postings[i] = pl
|
rv.postings[i] = pl
|
||||||
rv.iterators[i] = pl.Iterator()
|
rv.iterators[i] = pl.Iterator(includeFreq, includeNorm, includeTermVectors)
|
||||||
}
|
}
|
||||||
atomic.AddUint64(&i.parent.stats.TotTermSearchersStarted, uint64(1))
|
atomic.AddUint64(&i.parent.stats.TotTermSearchersStarted, uint64(1))
|
||||||
return rv, nil
|
return rv, nil
|
||||||
|
|
|
@ -165,7 +165,7 @@ func (cfd *cachedFieldDocs) prepareFields(field string, ss *SegmentSnapshot) {
|
||||||
}
|
}
|
||||||
|
|
||||||
cfd.size += uint64(size.SizeOfUint64) /* map key */
|
cfd.size += uint64(size.SizeOfUint64) /* map key */
|
||||||
postingsItr := postings.Iterator()
|
postingsItr := postings.Iterator(false, false, false)
|
||||||
nextPosting, err2 := postingsItr.Next()
|
nextPosting, err2 := postingsItr.Next()
|
||||||
for err2 == nil && nextPosting != nil {
|
for err2 == nil && nextPosting != nil {
|
||||||
docNum := nextPosting.Number()
|
docNum := nextPosting.Number()
|
||||||
|
|
Loading…
Reference in New Issue