gibheer/bleve
0
0
Fork 0
Code

scorch optimize zap.Merge() to reuse some buffers

Browse Source
This commit is contained in:
Steve Yen 2017-12-20 14:55:31 -08:00
parent ea4eb7301b
commit c155255506
1 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
index/scorch/segment/zap/merge.go
View File

@ -41,7 +41,7 @@ func Merge(segments []*Segment, drops []*roaring.Bitmap, path string,
return nil, err return nil, err
} }
// bufer the output // buffer the output
br := bufio.NewWriter(f) br := bufio.NewWriter(f)
// wrap it for counting (tracking offsets) // wrap it for counting (tracking offsets)
@ -128,6 +128,9 @@ func persistMergedRest(segments []*Segment, drops []*roaring.Bitmap,
newSegDocCount uint64, chunkFactor uint32, newSegDocCount uint64, chunkFactor uint32,
w *CountHashWriter) ([]uint64, error) { w *CountHashWriter) ([]uint64, error) {
var bufMaxVarintLen64 []byte = make([]byte, binary.MaxVarintLen64)
var bufLoc []uint64
rv := make([]uint64, len(fieldsInv)) rv := make([]uint64, len(fieldsInv))
var vellumBuf bytes.Buffer var vellumBuf bytes.Buffer
@ -208,7 +211,10 @@ func persistMergedRest(segments []*Segment, drops []*roaring.Bitmap,
if len(locs) > 0 { if len(locs) > 0 {
newRoaringLocs.Add(uint32(hitNewDocNum)) newRoaringLocs.Add(uint32(hitNewDocNum))
for _, loc := range locs { for _, loc := range locs {
args := make([]uint64, 0, 5+len(loc.ArrayPositions())) if cap(bufLoc) < 5+len(loc.ArrayPositions()) {
bufLoc = make([]uint64, 0, 5+len(loc.ArrayPositions()))
}
args := bufLoc[0:0]
args = append(args, uint64(fieldsMap[loc.Field()])) args = append(args, uint64(fieldsMap[loc.Field()]))
args = append(args, loc.Pos()) args = append(args, loc.Pos())
args = append(args, loc.Start()) args = append(args, loc.Start())
@ -250,7 +256,7 @@ func persistMergedRest(segments []*Segment, drops []*roaring.Bitmap,
} }
postingOffset := uint64(w.Count()) postingOffset := uint64(w.Count())
// write out the start of the term info // write out the start of the term info
buf := make([]byte, binary.MaxVarintLen64) buf := bufMaxVarintLen64
n := binary.PutUvarint(buf, freqOffset) n := binary.PutUvarint(buf, freqOffset)
_, err = w.Write(buf[:n]) _, err = w.Write(buf[:n])
if err != nil { if err != nil {
@ -295,7 +301,7 @@ func persistMergedRest(segments []*Segment, drops []*roaring.Bitmap,
vellumData := vellumBuf.Bytes() vellumData := vellumBuf.Bytes()
// write out the length of the vellum data // write out the length of the vellum data
buf := make([]byte, binary.MaxVarintLen64) buf := bufMaxVarintLen64
// write out the number of chunks // write out the number of chunks
n := binary.PutUvarint(buf, uint64(len(vellumData))) n := binary.PutUvarint(buf, uint64(len(vellumData)))
_, err = w.Write(buf[:n]) _, err = w.Write(buf[:n])