scorch zap postingsIter skips freq/norm/locs parsing if allowed
In this optimization, the zap PostingsIterator skips the parsing of freq/norm/locs chunks based on the includeFreq|Norm|Locs flags. In bleve-query microbenchmark on dev macbookpro, with 50K en-wiki docs, on a medium frequency term search that does not ask for term vectors, throughput was ~750 q/sec before the change and went to ~1400 q/sec after the change.
This commit is contained in:
parent
192621f402
commit
1cab701f85
|
@ -131,11 +131,11 @@ func (p *PostingsList) OrInto(receiver *roaring.Bitmap) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Iterator returns an iterator for this postings list
|
// Iterator returns an iterator for this postings list
|
||||||
func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocations bool) segment.PostingsIterator {
|
func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocs bool) segment.PostingsIterator {
|
||||||
return p.iterator(includeFreq, includeNorm, includeLocations, nil)
|
return p.iterator(includeFreq, includeNorm, includeLocs, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocations bool,
|
func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocs bool,
|
||||||
rv *PostingsIterator) *PostingsIterator {
|
rv *PostingsIterator) *PostingsIterator {
|
||||||
if rv == nil {
|
if rv == nil {
|
||||||
rv = &PostingsIterator{}
|
rv = &PostingsIterator{}
|
||||||
|
@ -195,38 +195,45 @@ func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocations bool,
|
||||||
return rv
|
return rv
|
||||||
}
|
}
|
||||||
|
|
||||||
// prepare the freq chunk details
|
|
||||||
var n uint64
|
var n uint64
|
||||||
var read int
|
var read int
|
||||||
var numFreqChunks uint64
|
|
||||||
numFreqChunks, read = binary.Uvarint(p.sb.mem[p.freqOffset+n : p.freqOffset+n+binary.MaxVarintLen64])
|
// prepare the freq chunk details
|
||||||
n += uint64(read)
|
rv.includeFreqNorm = includeFreq || includeNorm
|
||||||
if cap(rv.freqChunkOffsets) >= int(numFreqChunks) {
|
if rv.includeFreqNorm {
|
||||||
rv.freqChunkOffsets = rv.freqChunkOffsets[:int(numFreqChunks)]
|
var numFreqChunks uint64
|
||||||
} else {
|
numFreqChunks, read = binary.Uvarint(p.sb.mem[p.freqOffset+n : p.freqOffset+n+binary.MaxVarintLen64])
|
||||||
rv.freqChunkOffsets = make([]uint64, int(numFreqChunks))
|
|
||||||
}
|
|
||||||
for i := 0; i < int(numFreqChunks); i++ {
|
|
||||||
rv.freqChunkOffsets[i], read = binary.Uvarint(p.sb.mem[p.freqOffset+n : p.freqOffset+n+binary.MaxVarintLen64])
|
|
||||||
n += uint64(read)
|
n += uint64(read)
|
||||||
|
if cap(rv.freqChunkOffsets) >= int(numFreqChunks) {
|
||||||
|
rv.freqChunkOffsets = rv.freqChunkOffsets[:int(numFreqChunks)]
|
||||||
|
} else {
|
||||||
|
rv.freqChunkOffsets = make([]uint64, int(numFreqChunks))
|
||||||
|
}
|
||||||
|
for i := 0; i < int(numFreqChunks); i++ {
|
||||||
|
rv.freqChunkOffsets[i], read = binary.Uvarint(p.sb.mem[p.freqOffset+n : p.freqOffset+n+binary.MaxVarintLen64])
|
||||||
|
n += uint64(read)
|
||||||
|
}
|
||||||
|
rv.freqChunkStart = p.freqOffset + n
|
||||||
}
|
}
|
||||||
rv.freqChunkStart = p.freqOffset + n
|
|
||||||
|
|
||||||
// prepare the loc chunk details
|
// prepare the loc chunk details
|
||||||
n = 0
|
rv.includeLocs = includeLocs
|
||||||
var numLocChunks uint64
|
if rv.includeLocs {
|
||||||
numLocChunks, read = binary.Uvarint(p.sb.mem[p.locOffset+n : p.locOffset+n+binary.MaxVarintLen64])
|
n = 0
|
||||||
n += uint64(read)
|
var numLocChunks uint64
|
||||||
if cap(rv.locChunkOffsets) >= int(numLocChunks) {
|
numLocChunks, read = binary.Uvarint(p.sb.mem[p.locOffset+n : p.locOffset+n+binary.MaxVarintLen64])
|
||||||
rv.locChunkOffsets = rv.locChunkOffsets[:int(numLocChunks)]
|
|
||||||
} else {
|
|
||||||
rv.locChunkOffsets = make([]uint64, int(numLocChunks))
|
|
||||||
}
|
|
||||||
for i := 0; i < int(numLocChunks); i++ {
|
|
||||||
rv.locChunkOffsets[i], read = binary.Uvarint(p.sb.mem[p.locOffset+n : p.locOffset+n+binary.MaxVarintLen64])
|
|
||||||
n += uint64(read)
|
n += uint64(read)
|
||||||
|
if cap(rv.locChunkOffsets) >= int(numLocChunks) {
|
||||||
|
rv.locChunkOffsets = rv.locChunkOffsets[:int(numLocChunks)]
|
||||||
|
} else {
|
||||||
|
rv.locChunkOffsets = make([]uint64, int(numLocChunks))
|
||||||
|
}
|
||||||
|
for i := 0; i < int(numLocChunks); i++ {
|
||||||
|
rv.locChunkOffsets[i], read = binary.Uvarint(p.sb.mem[p.locOffset+n : p.locOffset+n+binary.MaxVarintLen64])
|
||||||
|
n += uint64(read)
|
||||||
|
}
|
||||||
|
rv.locChunkStart = p.locOffset + n
|
||||||
}
|
}
|
||||||
rv.locChunkStart = p.locOffset + n
|
|
||||||
|
|
||||||
rv.all = p.postings.Iterator()
|
rv.all = p.postings.Iterator()
|
||||||
if p.except != nil {
|
if p.except != nil {
|
||||||
|
@ -329,6 +336,9 @@ type PostingsIterator struct {
|
||||||
normBits1Hit uint64
|
normBits1Hit uint64
|
||||||
|
|
||||||
buf []byte
|
buf []byte
|
||||||
|
|
||||||
|
includeFreqNorm bool
|
||||||
|
includeLocs bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (i *PostingsIterator) Size() int {
|
func (i *PostingsIterator) Size() int {
|
||||||
|
@ -347,32 +357,42 @@ func (i *PostingsIterator) Size() int {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (i *PostingsIterator) loadChunk(chunk int) error {
|
func (i *PostingsIterator) loadChunk(chunk int) error {
|
||||||
if chunk >= len(i.freqChunkOffsets) || chunk >= len(i.locChunkOffsets) {
|
if i.includeFreqNorm {
|
||||||
return fmt.Errorf("tried to load chunk that doesn't exist %d/(%d %d)", chunk, len(i.freqChunkOffsets), len(i.locChunkOffsets))
|
if chunk >= len(i.freqChunkOffsets) {
|
||||||
|
return fmt.Errorf("tried to load freq chunk that doesn't exist %d/(%d)",
|
||||||
|
chunk, len(i.freqChunkOffsets))
|
||||||
|
}
|
||||||
|
|
||||||
|
end, start := i.freqChunkStart, i.freqChunkStart
|
||||||
|
s, e := readChunkBoundary(chunk, i.freqChunkOffsets)
|
||||||
|
start += s
|
||||||
|
end += e
|
||||||
|
i.currChunkFreqNorm = i.postings.sb.mem[start:end]
|
||||||
|
if i.freqNormReader == nil {
|
||||||
|
i.freqNormReader = bytes.NewReader(i.currChunkFreqNorm)
|
||||||
|
i.freqNormDecoder = govarint.NewU64Base128Decoder(i.freqNormReader)
|
||||||
|
} else {
|
||||||
|
i.freqNormReader.Reset(i.currChunkFreqNorm)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
end, start := i.freqChunkStart, i.freqChunkStart
|
if i.includeLocs {
|
||||||
s, e := readChunkBoundary(chunk, i.freqChunkOffsets)
|
if chunk >= len(i.locChunkOffsets) {
|
||||||
start += s
|
return fmt.Errorf("tried to load loc chunk that doesn't exist %d/(%d)",
|
||||||
end += e
|
chunk, len(i.locChunkOffsets))
|
||||||
i.currChunkFreqNorm = i.postings.sb.mem[start:end]
|
}
|
||||||
if i.freqNormReader == nil {
|
|
||||||
i.freqNormReader = bytes.NewReader(i.currChunkFreqNorm)
|
|
||||||
i.freqNormDecoder = govarint.NewU64Base128Decoder(i.freqNormReader)
|
|
||||||
} else {
|
|
||||||
i.freqNormReader.Reset(i.currChunkFreqNorm)
|
|
||||||
}
|
|
||||||
|
|
||||||
end, start = i.locChunkStart, i.locChunkStart
|
end, start := i.locChunkStart, i.locChunkStart
|
||||||
s, e = readChunkBoundary(chunk, i.locChunkOffsets)
|
s, e := readChunkBoundary(chunk, i.locChunkOffsets)
|
||||||
start += s
|
start += s
|
||||||
end += e
|
end += e
|
||||||
i.currChunkLoc = i.postings.sb.mem[start:end]
|
i.currChunkLoc = i.postings.sb.mem[start:end]
|
||||||
if i.locReader == nil {
|
if i.locReader == nil {
|
||||||
i.locReader = bytes.NewReader(i.currChunkLoc)
|
i.locReader = bytes.NewReader(i.currChunkLoc)
|
||||||
i.locDecoder = govarint.NewU64Base128Decoder(i.locReader)
|
i.locDecoder = govarint.NewU64Base128Decoder(i.locReader)
|
||||||
} else {
|
} else {
|
||||||
i.locReader.Reset(i.currChunkLoc)
|
i.locReader.Reset(i.currChunkLoc)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
i.currChunk = uint32(chunk)
|
i.currChunk = uint32(chunk)
|
||||||
|
@ -481,6 +501,10 @@ func (i *PostingsIterator) Next() (segment.Posting, error) {
|
||||||
rv := &i.next
|
rv := &i.next
|
||||||
rv.docNum = docNum
|
rv.docNum = docNum
|
||||||
|
|
||||||
|
if !i.includeFreqNorm {
|
||||||
|
return rv, nil
|
||||||
|
}
|
||||||
|
|
||||||
var normBits uint64
|
var normBits uint64
|
||||||
var hasLocs bool
|
var hasLocs bool
|
||||||
|
|
||||||
|
@ -491,7 +515,7 @@ func (i *PostingsIterator) Next() (segment.Posting, error) {
|
||||||
|
|
||||||
rv.norm = math.Float32frombits(uint32(normBits))
|
rv.norm = math.Float32frombits(uint32(normBits))
|
||||||
|
|
||||||
if hasLocs {
|
if i.includeLocs && hasLocs {
|
||||||
// read off 'freq' locations, into reused slices
|
// read off 'freq' locations, into reused slices
|
||||||
if cap(i.nextLocs) >= int(rv.freq) {
|
if cap(i.nextLocs) >= int(rv.freq) {
|
||||||
i.nextLocs = i.nextLocs[0:rv.freq]
|
i.nextLocs = i.nextLocs[0:rv.freq]
|
||||||
|
@ -591,7 +615,7 @@ func (i *PostingsIterator) nextDocNum() (uint64, bool, error) {
|
||||||
// if they don't match, move 'all' forwards until they do
|
// if they don't match, move 'all' forwards until they do
|
||||||
for allN != n {
|
for allN != n {
|
||||||
// in the same chunk, so move the freq/norm/loc decoders forward
|
// in the same chunk, so move the freq/norm/loc decoders forward
|
||||||
if allNChunk == nChunk {
|
if i.includeFreqNorm && allNChunk == nChunk {
|
||||||
if i.currChunk != nChunk || i.currChunkFreqNorm == nil {
|
if i.currChunk != nChunk || i.currChunkFreqNorm == nil {
|
||||||
err := i.loadChunk(int(nChunk))
|
err := i.loadChunk(int(nChunk))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -605,7 +629,7 @@ func (i *PostingsIterator) nextDocNum() (uint64, bool, error) {
|
||||||
return 0, false, err
|
return 0, false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if hasLocs {
|
if i.includeLocs && hasLocs {
|
||||||
for j := 0; j < int(freq); j++ {
|
for j := 0; j < int(freq); j++ {
|
||||||
err := i.readLocation(nil)
|
err := i.readLocation(nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -619,7 +643,7 @@ func (i *PostingsIterator) nextDocNum() (uint64, bool, error) {
|
||||||
allNChunk = allN / i.postings.sb.chunkFactor
|
allNChunk = allN / i.postings.sb.chunkFactor
|
||||||
}
|
}
|
||||||
|
|
||||||
if i.currChunk != nChunk || i.currChunkFreqNorm == nil {
|
if i.includeFreqNorm && (i.currChunk != nChunk || i.currChunkFreqNorm == nil) {
|
||||||
err := i.loadChunk(int(nChunk))
|
err := i.loadChunk(int(nChunk))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, false, fmt.Errorf("error loading chunk: %v", err)
|
return 0, false, fmt.Errorf("error loading chunk: %v", err)
|
||||||
|
|
Loading…
Reference in New Issue