scorch removed worker goroutines from TermFieldReader()
On a couple of micro benchmarks on a dev macbook using bleve-query on an index of 50K wikipedia docs, scorch is now in more the same neighborhood of upsidedown/moss... high-freq term search "text:date"... 400 qps - upsidedown/moss 360 qps - scorch before 404 qps - scorch after zero-freq term search "text:mschoch"... 100K qps - upsidedown/moss 55K qps - scorch before 99K qps - scorch after Of note, the scorch index had ~150 *.zap files in it, which likely made made the worker goroutine overhead more costly than for a case with few segments, where goroutine and channel related work appeared relatively prominently in the pprof SVG's.
This commit is contained in:
parent
562b473e36
commit
f05794c6aa
|
@ -341,27 +341,6 @@ func (i *IndexSnapshot) InternalID(id string) (rv index.IndexInternalID, err err
|
|||
func (i *IndexSnapshot) TermFieldReader(term []byte, field string, includeFreq,
|
||||
includeNorm, includeTermVectors bool) (index.TermFieldReader, error) {
|
||||
|
||||
results := make(chan *asynchSegmentResult)
|
||||
for index, segment := range i.segment {
|
||||
go func(index int, segment *SegmentSnapshot) {
|
||||
dict, err := segment.Dictionary(field)
|
||||
if err != nil {
|
||||
results <- &asynchSegmentResult{err: err}
|
||||
} else {
|
||||
pl, err := dict.PostingsList(string(term), nil)
|
||||
if err != nil {
|
||||
results <- &asynchSegmentResult{err: err}
|
||||
} else {
|
||||
results <- &asynchSegmentResult{
|
||||
index: index,
|
||||
postings: pl,
|
||||
}
|
||||
}
|
||||
}
|
||||
}(index, segment)
|
||||
}
|
||||
|
||||
var err error
|
||||
rv := &IndexSnapshotTermFieldReader{
|
||||
term: term,
|
||||
snapshot: i,
|
||||
|
@ -371,17 +350,17 @@ func (i *IndexSnapshot) TermFieldReader(term []byte, field string, includeFreq,
|
|||
includeNorm: includeNorm,
|
||||
includeTermVectors: includeTermVectors,
|
||||
}
|
||||
for count := 0; count < len(i.segment); count++ {
|
||||
asr := <-results
|
||||
if asr.err != nil && err == nil {
|
||||
err = asr.err
|
||||
} else {
|
||||
rv.postings[asr.index] = asr.postings
|
||||
rv.iterators[asr.index] = asr.postings.Iterator()
|
||||
for i, segment := range i.segment {
|
||||
dict, err := segment.Dictionary(field)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
pl, err := dict.PostingsList(string(term), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rv.postings[i] = pl
|
||||
rv.iterators[i] = pl.Iterator()
|
||||
}
|
||||
return rv, nil
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue