gibheer
/
bleve
0
0
Fork 0
Code

scorch zap optimize via postingsList reuse 

pprof graphs were showing many postingsList allocations during
merging, so this change optimizes by reusing postingList memory in the
merging loops.
This commit is contained in:
Steve Yen 2018-02-07 14:29:51 -08:00
parent 03c8b2b7ec
commit 8c2520d55c
4 changed files with 17 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
index/scorch/segment/zap/dict.go
View File

@ -34,15 +34,18 @@ type Dictionary struct {
// PostingsList returns the postings list for the specified term // PostingsList returns the postings list for the specified term
func (d *Dictionary) PostingsList(term string, except *roaring.Bitmap) (segment.PostingsList, error) { func (d *Dictionary) PostingsList(term string, except *roaring.Bitmap) (segment.PostingsList, error) {
return d.postingsList([]byte(term), except) return d.postingsList([]byte(term), except, nil)
} }
func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap) (*PostingsList, error) { func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) {
rv := &PostingsList{ if rv == nil {
sb: d.sb, rv = &PostingsList{}
term: term, } else {
except: except, *rv = PostingsList{} // clear the struct
} }
rv.sb = d.sb
rv.term = term
rv.except = except
if d.fst != nil { if d.fst != nil {
postingsOffset, exists, err := d.fst.Get(term) postingsOffset, exists, err := d.fst.Get(term)

5
index/scorch/segment/zap/merge.go
View File

@ -155,6 +155,8 @@ func persistMergedRest(segments []*Segment, drops []*roaring.Bitmap,
var bufMaxVarintLen64 []byte = make([]byte, binary.MaxVarintLen64) var bufMaxVarintLen64 []byte = make([]byte, binary.MaxVarintLen64)
var bufLoc []uint64 var bufLoc []uint64
var postings *PostingsList
rv := make([]uint64, len(fieldsInv)) rv := make([]uint64, len(fieldsInv))
fieldDvLocs := make([]uint64, len(fieldsInv)) fieldDvLocs := make([]uint64, len(fieldsInv))
fieldDvLocsOffset := uint64(fieldNotUninverted) fieldDvLocsOffset := uint64(fieldNotUninverted)
@ -231,7 +233,8 @@ func persistMergedRest(segments []*Segment, drops []*roaring.Bitmap,
if dict == nil { if dict == nil {
continue continue
} }
postings, err2 := dict.postingsList(term, drops[dictI]) var err2 error
postings, err2 = dict.postingsList(term, drops[dictI], postings)
if err2 != nil { if err2 != nil {
return nil, 0, err2 return nil, 0, err2
} }

4
index/scorch/segment/zap/merge_test.go
View File

@ -310,8 +310,8 @@ func compareSegments(a, b *Segment) string {
continue continue
} }
aplist, aerr := adict.(*Dictionary).postingsList([]byte(next.Term), nil) aplist, aerr := adict.(*Dictionary).postingsList([]byte(next.Term), nil, nil)
bplist, berr := bdict.(*Dictionary).postingsList([]byte(next.Term), nil) bplist, berr := bdict.(*Dictionary).postingsList([]byte(next.Term), nil, nil)
if aerr != berr { if aerr != berr {
rv = append(rv, fmt.Sprintf("field %s, term: %s, postingsList() errors different: %v %v", rv = append(rv, fmt.Sprintf("field %s, term: %s, postingsList() errors different: %v %v",
fieldName, next.Term, aerr, berr)) fieldName, next.Term, aerr, berr))

3
index/scorch/segment/zap/segment.go
View File

@ -343,8 +343,9 @@ func (s *SegmentBase) DocNumbers(ids []string) (*roaring.Bitmap, error) {
return nil, err return nil, err
} }
var postings *PostingsList
for _, id := range ids { for _, id := range ids {
postings, err := idDict.postingsList([]byte(id), nil) postings, err = idDict.postingsList([]byte(id), nil, postings)
if err != nil { if err != nil {
return nil, err return nil, err
} }