gibheer
/
bleve
0
0
Fork 0
Code

scorch includeFreq/Norm/Locs params for postingsList.Iterator API 

This commit adds boolean flag params to the scorch
PostingsList.Iterator() method, so that the caller can specify whether
freq/norm/locs information is needed or not.

Future changes can leverage these params for optimizations.
This commit is contained in:
Steve Yen 2018-03-24 10:03:57 -07:00
parent fc7584f5a0
commit 192621f402
10 changed files with 29 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
index/scorch/segment/empty.go
View File

@ -84,7 +84,7 @@ func (e *EmptyDictionaryIterator) Next() (*index.DictEntry, error) {
type EmptyPostingsList struct{}
func (e *EmptyPostingsList) Iterator() PostingsIterator {
func (e *EmptyPostingsList) Iterator(includeFreq, includeNorm, includeLocations bool) PostingsIterator {
return &EmptyPostingsIterator{}
}

2
index/scorch/segment/mem/posting.go
View File

@ -78,7 +78,7 @@ func (p *PostingsList) Count() uint64 {
}
// Iterator returns an iterator for this postings list
func (p *PostingsList) Iterator() segment.PostingsIterator {
func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocations bool) segment.PostingsIterator {
return p.InitIterator(nil)
}
func (p *PostingsList) InitIterator(prealloc *PostingsIterator) *PostingsIterator {

14
index/scorch/segment/mem/segment_test.go
View File

@ -48,7 +48,7 @@ func TestEmpty(t *testing.T) {
t.Fatal("got nil postings list, expected non-nil")
}
postingsItr := postingsList.Iterator()
postingsItr := postingsList.Iterator(true, true, true)
if postingsItr == nil {
t.Fatal("got nil iterator, expected non-nil")
}
@ -211,7 +211,7 @@ func TestSingle(t *testing.T) {
t.Fatal("got nil postings list, expected non-nil")
}
postingsItr := postingsList.Iterator()
postingsItr := postingsList.Iterator(true, true, true)
if postingsItr == nil {
t.Fatal("got nil iterator, expected non-nil")
}
@ -257,7 +257,7 @@ func TestSingle(t *testing.T) {
t.Fatal("got nil postings list, expected non-nil")
}
postingsItr = postingsList.Iterator()
postingsItr = postingsList.Iterator(true, true, true)
if postingsItr == nil {
t.Fatal("got nil iterator, expected non-nil")
}
@ -325,7 +325,7 @@ func TestSingle(t *testing.T) {
t.Fatal("got nil postings list, expected non-nil")
}
postingsItr = postingsList.Iterator()
postingsItr = postingsList.Iterator(true, true, true)
if postingsItr == nil {
t.Fatal("got nil iterator, expected non-nil")
}
@ -394,7 +394,7 @@ func TestSingle(t *testing.T) {
t.Fatal("got nil postings list, expected non-nil")
}
postingsItr = postingsList.Iterator()
postingsItr = postingsList.Iterator(true, true, true)
if postingsItr == nil {
t.Fatal("got nil iterator, expected non-nil")
}
@ -638,7 +638,7 @@ func TestMultiple(t *testing.T) {
t.Fatal("got nil postings list, expected non-nil")
}
postingsItr := postingsList.Iterator()
postingsItr := postingsList.Iterator(true, true, true)
if postingsItr == nil {
t.Fatal("got nil iterator, expected non-nil")
}
@ -677,7 +677,7 @@ func TestMultiple(t *testing.T) {
t.Errorf("expected count from postings list to be 1, got %d", postingsListExcludingCount)
}
postingsItrExcluding := postingsListExcluding.Iterator()
postingsItrExcluding := postingsListExcluding.Iterator(true, true, true)
if postingsItr == nil {
t.Fatal("got nil iterator, expected non-nil")
}

2
index/scorch/segment/segment.go
View File

@ -55,7 +55,7 @@ type DictionaryIterator interface {
}
type PostingsList interface {
Iterator() PostingsIterator
Iterator(includeFreq, includeNorm, includeLocations bool) PostingsIterator
Size() int

2
index/scorch/segment/zap/merge.go
View File

@ -303,7 +303,7 @@ func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap,
return nil, 0, err2
}
postItr = postings.iterator(postItr)
postItr = postings.iterator(true, true, true, postItr)
if fieldsSame {
// can optimize by copying freq/norm/loc bytes directly

4
index/scorch/segment/zap/merge_test.go
View File

@ -332,8 +332,8 @@ func compareSegments(a, b *Segment) string {
fieldName, next.Term, aplist.Count(), bplist.Count()))
}
apitr := aplist.Iterator()
bpitr := bplist.Iterator()
apitr := aplist.Iterator(true, true, true)
bpitr := bplist.Iterator(true, true, true)
if (apitr != nil) != (bpitr != nil) {
rv = append(rv, fmt.Sprintf("field %s, term: %s, postingsList.Iterator() results different: %v %v",
fieldName, next.Term, apitr, bpitr))

11
index/scorch/segment/zap/posting.go
View File

@ -131,11 +131,12 @@ func (p *PostingsList) OrInto(receiver *roaring.Bitmap) {
}
// Iterator returns an iterator for this postings list
func (p *PostingsList) Iterator() segment.PostingsIterator {
return p.iterator(nil)
func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocations bool) segment.PostingsIterator {
return p.iterator(includeFreq, includeNorm, includeLocations, nil)
}
func (p *PostingsList) iterator(rv *PostingsIterator) *PostingsIterator {
func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocations bool,
rv *PostingsIterator) *PostingsIterator {
if rv == nil {
rv = &PostingsIterator{}
} else {
@ -495,10 +496,10 @@ func (i *PostingsIterator) Next() (segment.Posting, error) {
if cap(i.nextLocs) >= int(rv.freq) {
i.nextLocs = i.nextLocs[0:rv.freq]
} else {
i.nextLocs = make([]Location, rv.freq, rv.freq * 2)
i.nextLocs = make([]Location, rv.freq, rv.freq*2)
}
if cap(i.nextSegmentLocs) < int(rv.freq) {
i.nextSegmentLocs = make([]segment.Location, rv.freq, rv.freq * 2)
i.nextSegmentLocs = make([]segment.Location, rv.freq, rv.freq*2)
}
rv.locs = i.nextSegmentLocs[0:rv.freq]
for j := 0; j < int(rv.freq); j++ {

16
index/scorch/segment/zap/segment_test.go
View File

@ -84,7 +84,7 @@ func TestOpen(t *testing.T) {
t.Fatal("got nil postings list, expected non-nil")
}
postingsItr := postingsList.Iterator()
postingsItr := postingsList.Iterator(true, true, true)
if postingsItr == nil {
t.Fatal("got nil iterator, expected non-nil")
}
@ -130,7 +130,7 @@ func TestOpen(t *testing.T) {
t.Fatal("got nil postings list, expected non-nil")
}
postingsItr = postingsList.Iterator()
postingsItr = postingsList.Iterator(true, true, true)
if postingsItr == nil {
t.Fatal("got nil iterator, expected non-nil")
}
@ -198,7 +198,7 @@ func TestOpen(t *testing.T) {
t.Fatal("got nil postings list, expected non-nil")
}
postingsItr = postingsList.Iterator()
postingsItr = postingsList.Iterator(true, true, true)
if postingsItr == nil {
t.Fatal("got nil iterator, expected non-nil")
}
@ -267,7 +267,7 @@ func TestOpen(t *testing.T) {
t.Fatal("got nil postings list, expected non-nil")
}
postingsItr = postingsList.Iterator()
postingsItr = postingsList.Iterator(true, true, true)
if postingsItr == nil {
t.Fatal("got nil iterator, expected non-nil")
}
@ -366,7 +366,7 @@ func TestOpenMulti(t *testing.T) {
t.Fatal("got nil postings list, expected non-nil")
}
postingsItr := postingsList.Iterator()
postingsItr := postingsList.Iterator(true, true, true)
if postingsItr == nil {
t.Fatal("got nil iterator, expected non-nil")
}
@ -405,7 +405,7 @@ func TestOpenMulti(t *testing.T) {
t.Errorf("expected count from postings list to be 1, got %d", postingsListExcludingCount)
}
postingsItrExcluding := postingsListExcluding.Iterator()
postingsItrExcluding := postingsListExcluding.Iterator(true, true, true)
if postingsItr == nil {
t.Fatal("got nil iterator, expected non-nil")
}
@ -466,7 +466,7 @@ func TestOpenMultiWithTwoChunks(t *testing.T) {
t.Fatal("got nil postings list, expected non-nil")
}
postingsItr := postingsList.Iterator()
postingsItr := postingsList.Iterator(true, true, true)
if postingsItr == nil {
t.Fatal("got nil iterator, expected non-nil")
}
@ -500,7 +500,7 @@ func TestOpenMultiWithTwoChunks(t *testing.T) {
t.Fatal("got nil postings list, expected non-nil")
}
postingsItrExcluding := postingsListExcluding.Iterator()
postingsItrExcluding := postingsListExcluding.Iterator(true, true, true)
if postingsItr == nil {
t.Fatal("got nil iterator, expected non-nil")
}

2
index/scorch/snapshot_index.go
View File

@ -394,7 +394,7 @@ func (i *IndexSnapshot) TermFieldReader(term []byte, field string, includeFreq,
return nil, err
}
rv.postings[i] = pl
rv.iterators[i] = pl.Iterator()
rv.iterators[i] = pl.Iterator(includeFreq, includeNorm, includeTermVectors)
}
atomic.AddUint64(&i.parent.stats.TotTermSearchersStarted, uint64(1))
return rv, nil

2
index/scorch/snapshot_segment.go
View File

@ -165,7 +165,7 @@ func (cfd *cachedFieldDocs) prepareFields(field string, ss *SegmentSnapshot) {
}
cfd.size += uint64(size.SizeOfUint64) /* map key */
postingsItr := postings.Iterator()
postingsItr := postings.Iterator(false, false, false)
nextPosting, err2 := postingsItr.Next()
for err2 == nil && nextPosting != nil {
docNum := nextPosting.Number()