0
0
Fork 0

scorch removed worker goroutines from TermFieldReader()

On a couple of micro benchmarks on a dev macbook using bleve-query on
an index of 50K wikipedia docs, scorch is now in more the same
neighborhood of upsidedown/moss...

high-freq term search "text:date"...
   400 qps - upsidedown/moss
   360 qps - scorch before
   404 qps - scorch after

zero-freq term search "text:mschoch"...
  100K qps - upsidedown/moss
   55K qps - scorch before
   99K qps - scorch after

Of note, the scorch index had ~150 *.zap files in it, which likely
made made the worker goroutine overhead more costly than for a case
with few segments, where goroutine and channel related work appeared
relatively prominently in the pprof SVG's.
This commit is contained in:
Steve Yen 2017-12-15 11:11:18 -08:00
parent 562b473e36
commit f05794c6aa
1 changed files with 10 additions and 31 deletions

View File

@ -341,27 +341,6 @@ func (i *IndexSnapshot) InternalID(id string) (rv index.IndexInternalID, err err
func (i *IndexSnapshot) TermFieldReader(term []byte, field string, includeFreq,
includeNorm, includeTermVectors bool) (index.TermFieldReader, error) {
results := make(chan *asynchSegmentResult)
for index, segment := range i.segment {
go func(index int, segment *SegmentSnapshot) {
dict, err := segment.Dictionary(field)
if err != nil {
results <- &asynchSegmentResult{err: err}
} else {
pl, err := dict.PostingsList(string(term), nil)
if err != nil {
results <- &asynchSegmentResult{err: err}
} else {
results <- &asynchSegmentResult{
index: index,
postings: pl,
}
}
}
}(index, segment)
}
var err error
rv := &IndexSnapshotTermFieldReader{
term: term,
snapshot: i,
@ -371,17 +350,17 @@ func (i *IndexSnapshot) TermFieldReader(term []byte, field string, includeFreq,
includeNorm: includeNorm,
includeTermVectors: includeTermVectors,
}
for count := 0; count < len(i.segment); count++ {
asr := <-results
if asr.err != nil && err == nil {
err = asr.err
} else {
rv.postings[asr.index] = asr.postings
rv.iterators[asr.index] = asr.postings.Iterator()
for i, segment := range i.segment {
dict, err := segment.Dictionary(field)
if err != nil {
return nil, err
}
}
if err != nil {
return nil, err
pl, err := dict.PostingsList(string(term), nil)
if err != nil {
return nil, err
}
rv.postings[i] = pl
rv.iterators[i] = pl.Iterator()
}
return rv, nil
}