From 5df53c8e1ff6677ba1920f8abb0e14195e5273c3 Mon Sep 17 00:00:00 2001 From: Steve Yen Date: Fri, 16 Mar 2018 11:49:50 -0700 Subject: [PATCH] scorch zap file merger uses 1MB buffered writer pprof of bleve-blast was showing file merging was in syscall/write a lot. The bufio.NewWriter() provides a default buffer size of 4K, which is too small, and using bufio.NewWriterSize(1MB buffer size) leads to syscall/write dropping out of the file merging flame graphs. --- index/scorch/segment/zap/merge.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/index/scorch/segment/zap/merge.go b/index/scorch/segment/zap/merge.go index d0d08c10..f3b6a68e 100644 --- a/index/scorch/segment/zap/merge.go +++ b/index/scorch/segment/zap/merge.go @@ -29,6 +29,8 @@ import ( "github.com/golang/snappy" ) +var DefaultFileMergerBufferSize = 1024 * 1024 + const docDropped = math.MaxUint64 // sentinel docNum to represent a deleted doc // Merge takes a slice of zap segments and bit masks describing which @@ -55,7 +57,7 @@ func Merge(segments []*Segment, drops []*roaring.Bitmap, path string, } // buffer the output - br := bufio.NewWriter(f) + br := bufio.NewWriterSize(f, DefaultFileMergerBufferSize) // wrap it for counting (tracking offsets) cr := NewCountHashWriter(br)