0
0

scorch zap provide full buffer capacity to snappy Encode/Decode()

The snappy Encode/Decode() API's accept an optional destination buffer
param where their encoded/decoded output results will be placed, but
they only check that the buffer has enough len() rather than enough
capacity before deciding to allocate a new buffer.
This commit is contained in:
Steve Yen 2018-03-17 10:40:34 -07:00
parent 84424edcad
commit 6540b197d4
4 changed files with 10 additions and 8 deletions

View File

@ -42,6 +42,8 @@ type chunkedContentCoder struct {
chunkBuf bytes.Buffer chunkBuf bytes.Buffer
chunkMeta []MetaData chunkMeta []MetaData
compressed []byte // temp buf for snappy compression
} }
// MetaData represents the data information inside a // MetaData represents the data information inside a
@ -105,10 +107,10 @@ func (c *chunkedContentCoder) flushContents() error {
metaData := c.chunkMetaBuf.Bytes() metaData := c.chunkMetaBuf.Bytes()
c.final = append(c.final, c.chunkMetaBuf.Bytes()...) c.final = append(c.final, c.chunkMetaBuf.Bytes()...)
// write the compressed data to the final data // write the compressed data to the final data
compressedData := snappy.Encode(nil, c.chunkBuf.Bytes()) c.compressed = snappy.Encode(c.compressed[:cap(c.compressed)], c.chunkBuf.Bytes())
c.final = append(c.final, compressedData...) c.final = append(c.final, c.compressed...)
c.chunkLens[c.currChunk] = uint64(len(compressedData) + len(metaData)) c.chunkLens[c.currChunk] = uint64(len(c.compressed) + len(metaData))
return nil return nil
} }

View File

@ -42,6 +42,7 @@ type docValueIterator struct {
dvDataLoc uint64 dvDataLoc uint64
curChunkHeader []MetaData curChunkHeader []MetaData
curChunkData []byte // compressed data cache curChunkData []byte // compressed data cache
uncompressed []byte // temp buf for snappy decompression
} }
func (di *docValueIterator) size() int { func (di *docValueIterator) size() int {
@ -135,10 +136,11 @@ func (di *docValueIterator) visitDocValues(docNum uint64,
return nil return nil
} }
// uncompress the already loaded data // uncompress the already loaded data
uncompressed, err := snappy.Decode(nil, di.curChunkData) uncompressed, err := snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData)
if err != nil { if err != nil {
return err return err
} }
di.uncompressed = uncompressed
// pick the terms for the given docNum // pick the terms for the given docNum
uncompressed = uncompressed[start:end] uncompressed = uncompressed[start:end]

View File

@ -604,7 +604,6 @@ func mergeStoredAndRemap(segments []*SegmentBase, drops []*roaring.Bitmap,
curr = 0 curr = 0
metaBuf.Reset() metaBuf.Reset()
data = data[:0] data = data[:0]
compressed = compressed[:0]
// collect all the data // collect all the data
for i := 0; i < len(fieldsInv); i++ { for i := 0; i < len(fieldsInv); i++ {
@ -641,7 +640,7 @@ func mergeStoredAndRemap(segments []*SegmentBase, drops []*roaring.Bitmap,
metaEncoder.Close() metaEncoder.Close()
metaBytes := metaBuf.Bytes() metaBytes := metaBuf.Bytes()
compressed = snappy.Encode(compressed, data) compressed = snappy.Encode(compressed[:cap(compressed)], data)
// record where we're about to start writing // record where we're about to start writing
docNumOffsets[newDocNum] = uint64(w.Count()) docNumOffsets[newDocNum] = uint64(w.Count())

View File

@ -517,7 +517,6 @@ func (s *interim) writeStoredFields() (
s.metaBuf.Reset() s.metaBuf.Reset()
data = data[:0] data = data[:0]
compressed = compressed[:0]
for fieldID := range s.FieldsInv { for fieldID := range s.FieldsInv {
isf, exists := docStoredFields[uint16(fieldID)] isf, exists := docStoredFields[uint16(fieldID)]
@ -534,7 +533,7 @@ func (s *interim) writeStoredFields() (
metaEncoder.Close() metaEncoder.Close()
metaBytes := s.metaBuf.Bytes() metaBytes := s.metaBuf.Bytes()
compressed = snappy.Encode(compressed, data) compressed = snappy.Encode(compressed[:cap(compressed)], data)
docStoredOffsets[docNum] = uint64(s.w.Count()) docStoredOffsets[docNum] = uint64(s.w.Count())