scorch zap fix postingsIter.nextBytes() when 1-bit encoded
The previous commit's optimization that replaced the locsBitmap was incorrectly handling the case when there was a 1-bit encoding optimization in the postingsIterator.nextBytes() method, incorrectly generating the freq-norm bytes. Also as part of this change, more unused locsBitmap's were removed.
This commit is contained in:
parent
7a19e6fd7e
commit
ba644f3893
|
@ -22,7 +22,7 @@ import (
|
||||||
"github.com/Smerity/govarint"
|
"github.com/Smerity/govarint"
|
||||||
)
|
)
|
||||||
|
|
||||||
const version uint32 = 7
|
const version uint32 = 8
|
||||||
|
|
||||||
const fieldNotUninverted = math.MaxUint64
|
const fieldNotUninverted = math.MaxUint64
|
||||||
|
|
||||||
|
|
|
@ -188,7 +188,6 @@ func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap,
|
||||||
}
|
}
|
||||||
|
|
||||||
newRoaring := roaring.NewBitmap()
|
newRoaring := roaring.NewBitmap()
|
||||||
newRoaringLocs := roaring.NewBitmap()
|
|
||||||
|
|
||||||
// for each field
|
// for each field
|
||||||
for fieldID, fieldName := range fieldsInv {
|
for fieldID, fieldName := range fieldsInv {
|
||||||
|
@ -234,7 +233,6 @@ func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap,
|
||||||
var prevTerm []byte
|
var prevTerm []byte
|
||||||
|
|
||||||
newRoaring.Clear()
|
newRoaring.Clear()
|
||||||
newRoaringLocs.Clear()
|
|
||||||
|
|
||||||
var lastDocNum, lastFreq, lastNorm uint64
|
var lastDocNum, lastFreq, lastNorm uint64
|
||||||
|
|
||||||
|
@ -273,7 +271,6 @@ func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap,
|
||||||
}
|
}
|
||||||
|
|
||||||
newRoaring.Clear()
|
newRoaring.Clear()
|
||||||
newRoaringLocs.Clear()
|
|
||||||
|
|
||||||
tfEncoder.Reset()
|
tfEncoder.Reset()
|
||||||
locEncoder.Reset()
|
locEncoder.Reset()
|
||||||
|
@ -311,11 +308,11 @@ func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap,
|
||||||
if fieldsSame {
|
if fieldsSame {
|
||||||
// can optimize by copying freq/norm/loc bytes directly
|
// can optimize by copying freq/norm/loc bytes directly
|
||||||
lastDocNum, lastFreq, lastNorm, err = mergeTermFreqNormLocsByCopying(
|
lastDocNum, lastFreq, lastNorm, err = mergeTermFreqNormLocsByCopying(
|
||||||
term, postItr, newDocNums[itrI], newRoaring, newRoaringLocs,
|
term, postItr, newDocNums[itrI], newRoaring,
|
||||||
tfEncoder, locEncoder, docTermMap)
|
tfEncoder, locEncoder, docTermMap)
|
||||||
} else {
|
} else {
|
||||||
lastDocNum, lastFreq, lastNorm, bufLoc, err = mergeTermFreqNormLocs(
|
lastDocNum, lastFreq, lastNorm, bufLoc, err = mergeTermFreqNormLocs(
|
||||||
fieldsMap, term, postItr, newDocNums[itrI], newRoaring, newRoaringLocs,
|
fieldsMap, term, postItr, newDocNums[itrI], newRoaring,
|
||||||
tfEncoder, locEncoder, docTermMap, bufLoc)
|
tfEncoder, locEncoder, docTermMap, bufLoc)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -406,7 +403,7 @@ func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap,
|
||||||
}
|
}
|
||||||
|
|
||||||
func mergeTermFreqNormLocs(fieldsMap map[string]uint16, term []byte, postItr *PostingsIterator,
|
func mergeTermFreqNormLocs(fieldsMap map[string]uint16, term []byte, postItr *PostingsIterator,
|
||||||
newDocNums []uint64, newRoaring *roaring.Bitmap, newRoaringLocs *roaring.Bitmap,
|
newDocNums []uint64, newRoaring *roaring.Bitmap,
|
||||||
tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder, docTermMap [][]byte,
|
tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder, docTermMap [][]byte,
|
||||||
bufLoc []uint64) (
|
bufLoc []uint64) (
|
||||||
lastDocNum uint64, lastFreq uint64, lastNorm uint64, bufLocOut []uint64, err error) {
|
lastDocNum uint64, lastFreq uint64, lastNorm uint64, bufLocOut []uint64, err error) {
|
||||||
|
@ -431,8 +428,6 @@ func mergeTermFreqNormLocs(fieldsMap map[string]uint16, term []byte, postItr *Po
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(locs) > 0 {
|
if len(locs) > 0 {
|
||||||
newRoaringLocs.Add(uint32(hitNewDocNum))
|
|
||||||
|
|
||||||
for _, loc := range locs {
|
for _, loc := range locs {
|
||||||
if cap(bufLoc) < 5+len(loc.ArrayPositions()) {
|
if cap(bufLoc) < 5+len(loc.ArrayPositions()) {
|
||||||
bufLoc = make([]uint64, 0, 5+len(loc.ArrayPositions()))
|
bufLoc = make([]uint64, 0, 5+len(loc.ArrayPositions()))
|
||||||
|
@ -465,7 +460,7 @@ func mergeTermFreqNormLocs(fieldsMap map[string]uint16, term []byte, postItr *Po
|
||||||
}
|
}
|
||||||
|
|
||||||
func mergeTermFreqNormLocsByCopying(term []byte, postItr *PostingsIterator,
|
func mergeTermFreqNormLocsByCopying(term []byte, postItr *PostingsIterator,
|
||||||
newDocNums []uint64, newRoaring *roaring.Bitmap, newRoaringLocs *roaring.Bitmap,
|
newDocNums []uint64, newRoaring *roaring.Bitmap,
|
||||||
tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder, docTermMap [][]byte) (
|
tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder, docTermMap [][]byte) (
|
||||||
lastDocNum uint64, lastFreq uint64, lastNorm uint64, err error) {
|
lastDocNum uint64, lastFreq uint64, lastNorm uint64, err error) {
|
||||||
nextDocNum, nextFreq, nextNorm, nextFreqNormBytes, nextLocBytes, err :=
|
nextDocNum, nextFreq, nextNorm, nextFreqNormBytes, nextLocBytes, err :=
|
||||||
|
@ -483,7 +478,6 @@ func mergeTermFreqNormLocsByCopying(term []byte, postItr *PostingsIterator,
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(nextLocBytes) > 0 {
|
if len(nextLocBytes) > 0 {
|
||||||
newRoaringLocs.Add(uint32(hitNewDocNum))
|
|
||||||
err = locEncoder.AddBytes(hitNewDocNum, nextLocBytes)
|
err = locEncoder.AddBytes(hitNewDocNum, nextLocBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, 0, err
|
return 0, 0, 0, err
|
||||||
|
|
|
@ -525,7 +525,7 @@ func (i *PostingsIterator) nextBytes() (
|
||||||
i.buf = make([]byte, binary.MaxVarintLen64*2)
|
i.buf = make([]byte, binary.MaxVarintLen64*2)
|
||||||
}
|
}
|
||||||
n := binary.PutUvarint(i.buf, freqHasLocs1Hit)
|
n := binary.PutUvarint(i.buf, freqHasLocs1Hit)
|
||||||
n += binary.PutUvarint(i.buf, i.normBits1Hit)
|
n += binary.PutUvarint(i.buf[n:], i.normBits1Hit)
|
||||||
return docNum, uint64(1), i.normBits1Hit, i.buf[:n], nil, nil
|
return docNum, uint64(1), i.normBits1Hit, i.buf[:n], nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue