From d1e2b55c72f543300a1848f6c68fa4096846ed86 Mon Sep 17 00:00:00 2001 From: Steve Yen Date: Wed, 21 Mar 2018 17:47:56 -0700 Subject: [PATCH 1/2] scorch zap postingsItr.nextDocNum() maintains allNChunk correctly When PostingsIterator.nextDocNum() moves the 'all' roaring bitmap iterator forwards, it was incorrectly not keeping the allNChunk value aligned. --- index/scorch/segment/zap/posting.go | 1 + 1 file changed, 1 insertion(+) diff --git a/index/scorch/segment/zap/posting.go b/index/scorch/segment/zap/posting.go index 081ec5f6..4b4b4933 100644 --- a/index/scorch/segment/zap/posting.go +++ b/index/scorch/segment/zap/posting.go @@ -620,6 +620,7 @@ func (i *PostingsIterator) nextDocNum() (uint64, bool, error) { } allN = i.all.Next() + allNChunk = allN / i.postings.sb.chunkFactor } if i.currChunk != nChunk || i.currChunkFreqNorm == nil { From b506fae4f73dcb73674b6d2f8f88df8f5726a78e Mon Sep 17 00:00:00 2001 From: Steve Yen Date: Wed, 21 Mar 2018 17:54:22 -0700 Subject: [PATCH 2/2] scorch zap postingsItr remove unused offset/locoffset fields --- index/scorch/segment/zap/posting.go | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/index/scorch/segment/zap/posting.go b/index/scorch/segment/zap/posting.go index 4b4b4933..f5ccad1a 100644 --- a/index/scorch/segment/zap/posting.go +++ b/index/scorch/segment/zap/posting.go @@ -316,11 +316,9 @@ func (rv *PostingsList) init1Hit(fstVal uint64) error { // PostingsIterator provides a way to iterate through the postings list type PostingsIterator struct { - postings *PostingsList - all roaring.IntIterable - offset int - locoffset int - actual roaring.IntIterable + postings *PostingsList + all roaring.IntIterable + actual roaring.IntIterable currChunk uint32 currChunkFreqNorm []byte @@ -584,16 +582,12 @@ func (i *PostingsIterator) nextDocNum() (uint64, bool, error) { nChunk := n / i.postings.sb.chunkFactor allNChunk := allN / i.postings.sb.chunkFactor - // n is the next actual hit (excluding some postings) - // allN is the next hit in the full postings - // if they don't match, adjust offsets to factor in item we're skipping over - // incr the all iterator, and check again + // n is the next actual hit (excluding some postings), and + // allN is the next hit in the full postings, and + // if they don't match, move 'all' forwards until they do for allN != n { - // in different chunks, reset offsets - if allNChunk != nChunk { - i.locoffset = 0 - i.offset = 0 - } else { + // in the same chunk, so move the freq/norm/loc decoders forward + if allNChunk == nChunk { if i.currChunk != nChunk || i.currChunkFreqNorm == nil { err := i.loadChunk(int(nChunk)) if err != nil { @@ -614,9 +608,6 @@ func (i *PostingsIterator) nextDocNum() (uint64, bool, error) { } } } - - // in same chunk, need to account for offsets - i.offset++ } allN = i.all.Next()