gibheer
/
bleve
0
0
Fork 0
Code

additional refactoring of build/merge

This commit is contained in:
Marty Schoch 2017-12-13 15:22:13 -05:00
parent 50441e5065
commit 6e2207c445
5 changed files with 60 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
index/scorch/segment/zap/build.go
View File

@ -182,19 +182,12 @@ func persistStored(memSegment *mem.Segment, w *CountHashWriter) (uint64, error)
// record where we're about to start writing
docNumOffsets[docNum] = uint64(w.Count())
buf := make([]byte, binary.MaxVarintLen64)
// write out the meta length
n := binary.PutUvarint(buf, uint64(len(metaBytes)))
_, err := w.Write(buf[:n])
if err != nil {
return 0, err
}
// write out the compressed data length
n = binary.PutUvarint(buf, uint64(len(compressed)))
_, err = w.Write(buf[:n])
// write out the meta len and compressed data len
_, err := writeUvarints(w, uint64(len(metaBytes)), uint64(len(compressed)))
if err != nil {
return 0, err
}
// now write the meta
_, err = w.Write(metaBytes)
if err != nil {
@ -444,25 +437,14 @@ func persistPostingsLists(memSegment *mem.Segment, w *CountHashWriter,
for postingID := range memSegment.Postings {
// record where we start this posting list
rv = append(rv, uint64(w.Count()))
// write out the start of the term info
buf := make([]byte, binary.MaxVarintLen64)
n := binary.PutUvarint(buf, freqOffsets[postingID])
_, err = w.Write(buf[:n])
if err != nil {
return nil, err
}
// write out the start of the loc info
n = binary.PutUvarint(buf, locOffsets[postingID])
_, err = w.Write(buf[:n])
if err != nil {
return nil, err
}
// write out the start of the loc posting list
n = binary.PutUvarint(buf, postingsListLocs[postingID])
_, err = w.Write(buf[:n])
// write out the term info, loc info, and loc posting list offset
_, err = writeUvarints(w, freqOffsets[postingID],
locOffsets[postingID], postingsListLocs[postingID])
if err != nil {
return nil, err
}
// write out the length and bitmap
_, err = writeRoaringWithLen(memSegment.Postings[postingID], w)
if err != nil {

13
index/scorch/segment/zap/merge.go
View File

@ -399,16 +399,9 @@ func mergeStoredAndRemap(segments []*Segment, drops []*roaring.Bitmap,
// record where we're about to start writing
docNumOffsets[newDocNum] = uint64(w.Count())
buf := make([]byte, binary.MaxVarintLen64)
// write out the meta length
n := binary.PutUvarint(buf, uint64(len(metaBytes)))
_, err = w.Write(buf[:n])
if err != nil {
return 0, nil, err
}
// write out the compressed data length
n = binary.PutUvarint(buf, uint64(len(compressed)))
_, err = w.Write(buf[:n])
// write out the meta len and compressed data len
_, err = writeUvarints(w,
uint64(len(metaBytes)), uint64(len(compressed)))
if err != nil {
return 0, nil, err
}

31
index/scorch/segment/zap/read.go Normal file
View File

@ -0,0 +1,31 @@
// Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package zap
import "encoding/binary"
func (s *Segment) getStoredMetaAndCompressed(docNum uint64) ([]byte, []byte) {
docStoredStartAddr := s.storedIndexOffset + (8 * docNum)
docStoredStart := binary.BigEndian.Uint64(s.mm[docStoredStartAddr : docStoredStartAddr+8])
var n uint64
metaLen, read := binary.Uvarint(s.mm[docStoredStart : docStoredStart+binary.MaxVarintLen64])
n += uint64(read)
var dataLen uint64
dataLen, read = binary.Uvarint(s.mm[docStoredStart+n : docStoredStart+n+binary.MaxVarintLen64])
n += uint64(read)
meta := s.mm[docStoredStart+n : docStoredStart+n+metaLen]
data := s.mm[docStoredStart+n+metaLen : docStoredStart+n+metaLen+dataLen]
return meta, data
}

13
index/scorch/segment/zap/segment.go
View File

@ -177,17 +177,8 @@ func (s *Segment) dictionary(field string) (*Dictionary, error) {
func (s *Segment) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error {
// first make sure this is a valid number in this segment
if num < s.numDocs {
docStoredStartAddr := s.storedIndexOffset + (8 * num)
docStoredStart := binary.BigEndian.Uint64(s.mm[docStoredStartAddr : docStoredStartAddr+8])
var n uint64
metaLen, read := binary.Uvarint(s.mm[docStoredStart : docStoredStart+binary.MaxVarintLen64])
n += uint64(read)
var dataLen uint64
dataLen, read = binary.Uvarint(s.mm[docStoredStart+n : docStoredStart+n+binary.MaxVarintLen64])
n += uint64(read)
meta := s.mm[docStoredStart+n : docStoredStart+n+metaLen]
data := s.mm[docStoredStart+n+metaLen : docStoredStart+n+metaLen+dataLen]
uncompressed, err := snappy.Decode(nil, data)
meta, compressed := s.getStoredMetaAndCompressed(num)
uncompressed, err := snappy.Decode(nil, compressed)
if err != nil {
panic(err)
}

27
index/scorch/segment/zap/write.go
View File

@ -60,17 +60,8 @@ func persistFields(fieldsInv []string, w *CountHashWriter, dictLocs []uint64) (u
// record start of this field
fieldStarts = append(fieldStarts, uint64(w.Count()))
buf := make([]byte, binary.MaxVarintLen64)
// write out dict location for this field
n := binary.PutUvarint(buf, dictLocs[fieldID])
_, err := w.Write(buf[:n])
if err != nil {
return 0, err
}
// write out the length of the field name
n = binary.PutUvarint(buf, uint64(len(fieldName)))
_, err = w.Write(buf[:n])
// write out the dict location and field name length
_, err := writeUvarints(w, dictLocs[fieldID], uint64(len(fieldName)))
if err != nil {
return 0, err
}
@ -132,3 +123,17 @@ func persistFooter(numDocs, storedIndexOffset, fieldIndexOffset uint64,
}
return nil
}
func writeUvarints(w io.Writer, vals ...uint64) (tw int, err error) {
buf := make([]byte, binary.MaxVarintLen64)
for _, val := range vals {
n := binary.PutUvarint(buf, val)
var nw int
nw, err = w.Write(buf[:n])
tw += nw
if err != nil {
return tw, err
}
}
return tw, err
}