scorch zap postingsIter skips freq/norm/locs parsing if allowed
In this optimization, the zap PostingsIterator skips the parsing of freq/norm/locs chunks based on the includeFreq|Norm|Locs flags. In bleve-query microbenchmark on dev macbookpro, with 50K en-wiki docs, on a medium frequency term search that does not ask for term vectors, throughput was ~750 q/sec before the change and went to ~1400 q/sec after the change.
This commit is contained in:
Steve Yen
parent
192621f402
commit
1cab701f85
|
|
@ -131,11 +131,11 @@ func (p *PostingsList) OrInto(receiver *roaring.Bitmap) {
|
|||
}
|
||||
|
||||
// Iterator returns an iterator for this postings list
|
||||
func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocations bool) segment.PostingsIterator {
|
||||
return p.iterator(includeFreq, includeNorm, includeLocations, nil)
|
||||
func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocs bool) segment.PostingsIterator {
|
||||
return p.iterator(includeFreq, includeNorm, includeLocs, nil)
|
||||
}
|
||||
|
||||
func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocations bool,
|
||||
func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocs bool,
|
||||
rv *PostingsIterator) *PostingsIterator {
|
||||
if rv == nil {
|
||||
rv = &PostingsIterator{}
|
||||
|
|
@ -195,38 +195,45 @@ func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocations bool,
|
|||
return rv
|
||||
}
|
||||
|
||||
// prepare the freq chunk details
|
||||
var n uint64
|
||||
var read int
|
||||
var numFreqChunks uint64
|
||||
numFreqChunks, read = binary.Uvarint(p.sb.mem[p.freqOffset+n : p.freqOffset+n+binary.MaxVarintLen64])
|
||||
n += uint64(read)
|
||||
if cap(rv.freqChunkOffsets) >= int(numFreqChunks) {
|
||||
rv.freqChunkOffsets = rv.freqChunkOffsets[:int(numFreqChunks)]
|
||||
} else {
|
||||
rv.freqChunkOffsets = make([]uint64, int(numFreqChunks))
|
||||
}
|
||||
for i := 0; i < int(numFreqChunks); i++ {
|
||||
rv.freqChunkOffsets[i], read = binary.Uvarint(p.sb.mem[p.freqOffset+n : p.freqOffset+n+binary.MaxVarintLen64])
|
||||
|
||||
// prepare the freq chunk details
|
||||
rv.includeFreqNorm = includeFreq || includeNorm
|
||||
if rv.includeFreqNorm {
|
||||
var numFreqChunks uint64
|
||||
numFreqChunks, read = binary.Uvarint(p.sb.mem[p.freqOffset+n : p.freqOffset+n+binary.MaxVarintLen64])
|
||||
n += uint64(read)
|
||||
if cap(rv.freqChunkOffsets) >= int(numFreqChunks) {
|
||||
rv.freqChunkOffsets = rv.freqChunkOffsets[:int(numFreqChunks)]
|
||||
} else {
|
||||
rv.freqChunkOffsets = make([]uint64, int(numFreqChunks))
|
||||
}
|
||||
for i := 0; i < int(numFreqChunks); i++ {
|
||||
rv.freqChunkOffsets[i], read = binary.Uvarint(p.sb.mem[p.freqOffset+n : p.freqOffset+n+binary.MaxVarintLen64])
|
||||
n += uint64(read)
|
||||
}
|
||||
rv.freqChunkStart = p.freqOffset + n
|
||||
}
|
||||
rv.freqChunkStart = p.freqOffset + n
|
||||
|
||||
// prepare the loc chunk details
|
||||
n = 0
|
||||
var numLocChunks uint64
|
||||
numLocChunks, read = binary.Uvarint(p.sb.mem[p.locOffset+n : p.locOffset+n+binary.MaxVarintLen64])
|
||||
n += uint64(read)
|
||||
if cap(rv.locChunkOffsets) >= int(numLocChunks) {
|
||||
rv.locChunkOffsets = rv.locChunkOffsets[:int(numLocChunks)]
|
||||
} else {
|
||||
rv.locChunkOffsets = make([]uint64, int(numLocChunks))
|
||||
}
|
||||
for i := 0; i < int(numLocChunks); i++ {
|
||||
rv.locChunkOffsets[i], read = binary.Uvarint(p.sb.mem[p.locOffset+n : p.locOffset+n+binary.MaxVarintLen64])
|
||||
rv.includeLocs = includeLocs
|
||||
if rv.includeLocs {
|
||||
n = 0
|
||||
var numLocChunks uint64
|
||||
numLocChunks, read = binary.Uvarint(p.sb.mem[p.locOffset+n : p.locOffset+n+binary.MaxVarintLen64])
|
||||
n += uint64(read)
|
||||
if cap(rv.locChunkOffsets) >= int(numLocChunks) {
|
||||
rv.locChunkOffsets = rv.locChunkOffsets[:int(numLocChunks)]
|
||||
} else {
|
||||
rv.locChunkOffsets = make([]uint64, int(numLocChunks))
|
||||
}
|
||||
for i := 0; i < int(numLocChunks); i++ {
|
||||
rv.locChunkOffsets[i], read = binary.Uvarint(p.sb.mem[p.locOffset+n : p.locOffset+n+binary.MaxVarintLen64])
|
||||
n += uint64(read)
|
||||
}
|
||||
rv.locChunkStart = p.locOffset + n
|
||||
}
|
||||
rv.locChunkStart = p.locOffset + n
|
||||
|
||||
rv.all = p.postings.Iterator()
|
||||
if p.except != nil {
|
||||
|
|
@ -329,6 +336,9 @@ type PostingsIterator struct {
|
|||
normBits1Hit uint64
|
||||
|
||||
buf []byte
|
||||
|
||||
includeFreqNorm bool
|
||||
includeLocs bool
|
||||
}
|
||||
|
||||
func (i *PostingsIterator) Size() int {
|
||||
|
|
@ -347,32 +357,42 @@ func (i *PostingsIterator) Size() int {
|
|||
}
|
||||
|
||||
func (i *PostingsIterator) loadChunk(chunk int) error {
|
||||
if chunk >= len(i.freqChunkOffsets) || chunk >= len(i.locChunkOffsets) {
|
||||
return fmt.Errorf("tried to load chunk that doesn't exist %d/(%d %d)", chunk, len(i.freqChunkOffsets), len(i.locChunkOffsets))
|
||||
if i.includeFreqNorm {
|
||||
if chunk >= len(i.freqChunkOffsets) {
|
||||
return fmt.Errorf("tried to load freq chunk that doesn't exist %d/(%d)",
|
||||
chunk, len(i.freqChunkOffsets))
|
||||
}
|
||||
|
||||
end, start := i.freqChunkStart, i.freqChunkStart
|
||||
s, e := readChunkBoundary(chunk, i.freqChunkOffsets)
|
||||
start += s
|
||||
end += e
|
||||
i.currChunkFreqNorm = i.postings.sb.mem[start:end]
|
||||
if i.freqNormReader == nil {
|
||||
i.freqNormReader = bytes.NewReader(i.currChunkFreqNorm)
|
||||
i.freqNormDecoder = govarint.NewU64Base128Decoder(i.freqNormReader)
|
||||
} else {
|
||||
i.freqNormReader.Reset(i.currChunkFreqNorm)
|
||||
}
|
||||
}
|
||||
|
||||
end, start := i.freqChunkStart, i.freqChunkStart
|
||||
s, e := readChunkBoundary(chunk, i.freqChunkOffsets)
|
||||
start += s
|
||||
end += e
|
||||
i.currChunkFreqNorm = i.postings.sb.mem[start:end]
|
||||
if i.freqNormReader == nil {
|
||||
i.freqNormReader = bytes.NewReader(i.currChunkFreqNorm)
|
||||
i.freqNormDecoder = govarint.NewU64Base128Decoder(i.freqNormReader)
|
||||
} else {
|
||||
i.freqNormReader.Reset(i.currChunkFreqNorm)
|
||||
}
|
||||
if i.includeLocs {
|
||||
if chunk >= len(i.locChunkOffsets) {
|
||||
return fmt.Errorf("tried to load loc chunk that doesn't exist %d/(%d)",
|
||||
chunk, len(i.locChunkOffsets))
|
||||
}
|
||||
|
||||
end, start = i.locChunkStart, i.locChunkStart
|
||||
s, e = readChunkBoundary(chunk, i.locChunkOffsets)
|
||||
start += s
|
||||
end += e
|
||||
i.currChunkLoc = i.postings.sb.mem[start:end]
|
||||
if i.locReader == nil {
|
||||
i.locReader = bytes.NewReader(i.currChunkLoc)
|
||||
i.locDecoder = govarint.NewU64Base128Decoder(i.locReader)
|
||||
} else {
|
||||
i.locReader.Reset(i.currChunkLoc)
|
||||
end, start := i.locChunkStart, i.locChunkStart
|
||||
s, e := readChunkBoundary(chunk, i.locChunkOffsets)
|
||||
start += s
|
||||
end += e
|
||||
i.currChunkLoc = i.postings.sb.mem[start:end]
|
||||
if i.locReader == nil {
|
||||
i.locReader = bytes.NewReader(i.currChunkLoc)
|
||||
i.locDecoder = govarint.NewU64Base128Decoder(i.locReader)
|
||||
} else {
|
||||
i.locReader.Reset(i.currChunkLoc)
|
||||
}
|
||||
}
|
||||
|
||||
i.currChunk = uint32(chunk)
|
||||
|
|
@ -481,6 +501,10 @@ func (i *PostingsIterator) Next() (segment.Posting, error) {
|
|||
rv := &i.next
|
||||
rv.docNum = docNum
|
||||
|
||||
if !i.includeFreqNorm {
|
||||
return rv, nil
|
||||
}
|
||||
|
||||
var normBits uint64
|
||||
var hasLocs bool
|
||||
|
||||
|
|
@ -491,7 +515,7 @@ func (i *PostingsIterator) Next() (segment.Posting, error) {
|
|||
|
||||
rv.norm = math.Float32frombits(uint32(normBits))
|
||||
|
||||
if hasLocs {
|
||||
if i.includeLocs && hasLocs {
|
||||
// read off 'freq' locations, into reused slices
|
||||
if cap(i.nextLocs) >= int(rv.freq) {
|
||||
i.nextLocs = i.nextLocs[0:rv.freq]
|
||||
|
|
@ -591,7 +615,7 @@ func (i *PostingsIterator) nextDocNum() (uint64, bool, error) {
|
|||
// if they don't match, move 'all' forwards until they do
|
||||
for allN != n {
|
||||
// in the same chunk, so move the freq/norm/loc decoders forward
|
||||
if allNChunk == nChunk {
|
||||
if i.includeFreqNorm && allNChunk == nChunk {
|
||||
if i.currChunk != nChunk || i.currChunkFreqNorm == nil {
|
||||
err := i.loadChunk(int(nChunk))
|
||||
if err != nil {
|
||||
|
|
@ -605,7 +629,7 @@ func (i *PostingsIterator) nextDocNum() (uint64, bool, error) {
|
|||
return 0, false, err
|
||||
}
|
||||
|
||||
if hasLocs {
|
||||
if i.includeLocs && hasLocs {
|
||||
for j := 0; j < int(freq); j++ {
|
||||
err := i.readLocation(nil)
|
||||
if err != nil {
|
||||
|
|
@ -619,7 +643,7 @@ func (i *PostingsIterator) nextDocNum() (uint64, bool, error) {
|
|||
allNChunk = allN / i.postings.sb.chunkFactor
|
||||
}
|
||||
|
||||
if i.currChunk != nChunk || i.currChunkFreqNorm == nil {
|
||||
if i.includeFreqNorm && (i.currChunk != nChunk || i.currChunkFreqNorm == nil) {
|
||||
err := i.loadChunk(int(nChunk))
|
||||
if err != nil {
|
||||
return 0, false, fmt.Errorf("error loading chunk: %v", err)
|
||||
|
|
|
|||
Loading…
Reference in New Issue