scorch stats MaxBatchIntroTime bug fix and more timing stats
Added timing stats for in-mem zap merging and file-based zap merging.
This commit is contained in:
parent
c5ab1f61d7
commit
d44c5ad568
|
@ -179,9 +179,19 @@ func (s *Scorch) planMergeAtSnapshot(ourSnapshot *IndexSnapshot,
|
||||||
filename := zapFileName(newSegmentID)
|
filename := zapFileName(newSegmentID)
|
||||||
s.markIneligibleForRemoval(filename)
|
s.markIneligibleForRemoval(filename)
|
||||||
path := s.path + string(os.PathSeparator) + filename
|
path := s.path + string(os.PathSeparator) + filename
|
||||||
|
|
||||||
|
fileMergeZapStartTime := time.Now()
|
||||||
|
|
||||||
atomic.AddUint64(&s.stats.TotFileMergeZapBeg, 1)
|
atomic.AddUint64(&s.stats.TotFileMergeZapBeg, 1)
|
||||||
newDocNums, err := zap.Merge(segmentsToMerge, docsToDrop, path, 1024)
|
newDocNums, err := zap.Merge(segmentsToMerge, docsToDrop, path, 1024)
|
||||||
atomic.AddUint64(&s.stats.TotFileMergeZapEnd, 1)
|
atomic.AddUint64(&s.stats.TotFileMergeZapEnd, 1)
|
||||||
|
|
||||||
|
fileMergeZapTime := uint64(time.Since(fileMergeZapStartTime))
|
||||||
|
atomic.AddUint64(&s.stats.TotFileMergeZapTime, fileMergeZapTime)
|
||||||
|
if atomic.LoadUint64(&s.stats.MaxFileMergeZapTime) < fileMergeZapTime {
|
||||||
|
atomic.StoreUint64(&s.stats.MaxFileMergeZapTime, fileMergeZapTime)
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.unmarkIneligibleForRemoval(filename)
|
s.unmarkIneligibleForRemoval(filename)
|
||||||
atomic.AddUint64(&s.stats.TotFileMergePlanTasksErr, 1)
|
atomic.AddUint64(&s.stats.TotFileMergePlanTasksErr, 1)
|
||||||
|
@ -258,11 +268,20 @@ func (s *Scorch) mergeSegmentBases(snapshot *IndexSnapshot,
|
||||||
|
|
||||||
cr := zap.NewCountHashWriter(&br)
|
cr := zap.NewCountHashWriter(&br)
|
||||||
|
|
||||||
|
memMergeZapStartTime := time.Now()
|
||||||
|
|
||||||
atomic.AddUint64(&s.stats.TotMemMergeZapBeg, 1)
|
atomic.AddUint64(&s.stats.TotMemMergeZapBeg, 1)
|
||||||
newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset,
|
newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset,
|
||||||
docValueOffset, dictLocs, fieldsInv, fieldsMap, err :=
|
docValueOffset, dictLocs, fieldsInv, fieldsMap, err :=
|
||||||
zap.MergeToWriter(sbs, sbsDrops, chunkFactor, cr)
|
zap.MergeToWriter(sbs, sbsDrops, chunkFactor, cr)
|
||||||
atomic.AddUint64(&s.stats.TotMemMergeZapEnd, 1)
|
atomic.AddUint64(&s.stats.TotMemMergeZapEnd, 1)
|
||||||
|
|
||||||
|
memMergeZapTime := uint64(time.Since(memMergeZapStartTime))
|
||||||
|
atomic.AddUint64(&s.stats.TotMemMergeZapTime, memMergeZapTime)
|
||||||
|
if atomic.LoadUint64(&s.stats.MaxMemMergeZapTime) < memMergeZapTime {
|
||||||
|
atomic.StoreUint64(&s.stats.MaxMemMergeZapTime, memMergeZapTime)
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
atomic.AddUint64(&s.stats.TotMemMergeErr, 1)
|
atomic.AddUint64(&s.stats.TotMemMergeErr, 1)
|
||||||
return 0, nil, 0, err
|
return 0, nil, 0, err
|
||||||
|
|
|
@ -365,7 +365,7 @@ func (s *Scorch) prepareSegment(newSegment segment.Segment, ids []string,
|
||||||
introTime := uint64(time.Since(introStartTime))
|
introTime := uint64(time.Since(introStartTime))
|
||||||
atomic.AddUint64(&s.stats.TotBatchIntroTime, introTime)
|
atomic.AddUint64(&s.stats.TotBatchIntroTime, introTime)
|
||||||
if atomic.LoadUint64(&s.stats.MaxBatchIntroTime) < introTime {
|
if atomic.LoadUint64(&s.stats.MaxBatchIntroTime) < introTime {
|
||||||
atomic.AddUint64(&s.stats.MaxBatchIntroTime, introTime)
|
atomic.StoreUint64(&s.stats.MaxBatchIntroTime, introTime)
|
||||||
}
|
}
|
||||||
|
|
||||||
return err
|
return err
|
||||||
|
|
|
@ -88,8 +88,10 @@ type Stats struct {
|
||||||
TotFileMergeSegmentsEmpty uint64
|
TotFileMergeSegmentsEmpty uint64
|
||||||
TotFileMergeSegments uint64
|
TotFileMergeSegments uint64
|
||||||
|
|
||||||
TotFileMergeZapBeg uint64
|
TotFileMergeZapBeg uint64
|
||||||
TotFileMergeZapEnd uint64
|
TotFileMergeZapEnd uint64
|
||||||
|
TotFileMergeZapTime uint64
|
||||||
|
MaxFileMergeZapTime uint64
|
||||||
|
|
||||||
TotFileMergeIntroductions uint64
|
TotFileMergeIntroductions uint64
|
||||||
TotFileMergeIntroductionsDone uint64
|
TotFileMergeIntroductionsDone uint64
|
||||||
|
@ -99,6 +101,8 @@ type Stats struct {
|
||||||
TotMemMergeDone uint64
|
TotMemMergeDone uint64
|
||||||
TotMemMergeZapBeg uint64
|
TotMemMergeZapBeg uint64
|
||||||
TotMemMergeZapEnd uint64
|
TotMemMergeZapEnd uint64
|
||||||
|
TotMemMergeZapTime uint64
|
||||||
|
MaxMemMergeZapTime uint64
|
||||||
TotMemMergeSegments uint64
|
TotMemMergeSegments uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue