scorch zap sync.Pool for reusable VisitDocument() data structures
As part of this, snappy.Decode() is also provided a reused buffer for decompression.
This commit is contained in:
parent
33b1f065dc
commit
84424edcad
|
@ -273,19 +273,39 @@ func (sb *SegmentBase) dictionary(field string) (rv *Dictionary, err error) {
|
||||||
return rv, nil
|
return rv, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// visitDocumentCtx holds data structures that are reusable across
|
||||||
|
// multiple VisitDocument() calls to avoid memory allocations
|
||||||
|
type visitDocumentCtx struct {
|
||||||
|
buf []byte
|
||||||
|
reader bytes.Reader
|
||||||
|
decoder *govarint.Base128Decoder
|
||||||
|
arrayPos []uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
var visitDocumentCtxPool = sync.Pool{
|
||||||
|
New: func() interface{} {
|
||||||
|
reuse := &visitDocumentCtx{}
|
||||||
|
reuse.decoder = govarint.NewU64Base128Decoder(&reuse.reader)
|
||||||
|
return reuse
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
// VisitDocument invokes the DocFieldValueVistor for each stored field
|
// VisitDocument invokes the DocFieldValueVistor for each stored field
|
||||||
// for the specified doc number
|
// for the specified doc number
|
||||||
func (s *SegmentBase) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error {
|
func (s *SegmentBase) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error {
|
||||||
// first make sure this is a valid number in this segment
|
// first make sure this is a valid number in this segment
|
||||||
if num < s.numDocs {
|
if num < s.numDocs {
|
||||||
|
vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx)
|
||||||
|
|
||||||
meta, compressed := s.getDocStoredMetaAndCompressed(num)
|
meta, compressed := s.getDocStoredMetaAndCompressed(num)
|
||||||
uncompressed, err := snappy.Decode(nil, compressed)
|
uncompressed, err := snappy.Decode(vdc.buf[:cap(vdc.buf)], compressed)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// now decode meta and process
|
// now decode meta and process
|
||||||
reader := bytes.NewReader(meta)
|
vdc.reader.Reset(meta)
|
||||||
decoder := govarint.NewU64Base128Decoder(reader)
|
decoder := vdc.decoder
|
||||||
|
|
||||||
keepGoing := true
|
keepGoing := true
|
||||||
for keepGoing {
|
for keepGoing {
|
||||||
|
@ -314,7 +334,10 @@ func (s *SegmentBase) VisitDocument(num uint64, visitor segment.DocumentFieldVal
|
||||||
}
|
}
|
||||||
var arrayPos []uint64
|
var arrayPos []uint64
|
||||||
if numap > 0 {
|
if numap > 0 {
|
||||||
arrayPos = make([]uint64, numap)
|
if cap(vdc.arrayPos) < int(numap) {
|
||||||
|
vdc.arrayPos = make([]uint64, numap)
|
||||||
|
}
|
||||||
|
arrayPos = vdc.arrayPos[:numap]
|
||||||
for i := 0; i < int(numap); i++ {
|
for i := 0; i < int(numap); i++ {
|
||||||
ap, err := decoder.GetU64()
|
ap, err := decoder.GetU64()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -327,6 +350,9 @@ func (s *SegmentBase) VisitDocument(num uint64, visitor segment.DocumentFieldVal
|
||||||
value := uncompressed[offset : offset+l]
|
value := uncompressed[offset : offset+l]
|
||||||
keepGoing = visitor(s.fieldsInv[field], byte(typ), value, arrayPos)
|
keepGoing = visitor(s.fieldsInv[field], byte(typ), value, arrayPos)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
vdc.buf = uncompressed
|
||||||
|
visitDocumentCtxPool.Put(vdc)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue