Adding num_items_introduced, num_items_persisted stats

+ Adding new entries to the stats struct of scorch.
+ These stats are atomically incremented upon every segment
  introduction, and upon successful persistence.

index/scorch/introducer.go

@@ -16,6 +16,7 @@ package scorch
 
 import (
 	"fmt"
+	"sync/atomic"
 
 	"github.com/RoaringBitmap/roaring"
 	"github.com/blevesearch/bleve/index/scorch/segment"
@@ -142,12 +143,17 @@ func (s *Scorch) introduceSegment(next *segmentIntroduction) error {
 	}
 	// append new segment, if any, to end of the new index snapshot
 	if next.data != nil {
-		newSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{
+		newSegmentSnapshot := &SegmentSnapshot{
 			id:         next.id,
 			segment:    next.data, // take ownership of next.data's ref-count
 			cachedDocs: &cachedDocs{cache: nil},
-		})
+		}
+		newSnapshot.segment = append(newSnapshot.segment, newSegmentSnapshot)
 		newSnapshot.offsets = append(newSnapshot.offsets, running)
+
+		// increment numItemsIntroduced which tracks the number of items
+		// queued for persistence.
+		atomic.AddUint64(&s.stats.numItemsIntroduced, newSegmentSnapshot.Count())
 	}
 	// copy old values
 	for key, oldVal := range s.root.internal {

index/scorch/persister.go

@@ -24,6 +24,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync/atomic"
 
 	"github.com/RoaringBitmap/roaring"
 	"github.com/blevesearch/bleve/index/scorch/segment"
@@ -75,6 +76,7 @@ OUTER:
 				_ = ourSnapshot.DecRef()
 				continue OUTER
 			}
+
 			lastPersistedEpoch = ourSnapshot.epoch
 			for _, notifyCh := range notifyChs {
 				close(notifyCh)
@@ -243,6 +245,8 @@ func (s *Scorch) persistSnapshot(snapshot *IndexSnapshot) error {
 				cachedDocs: segmentSnapshot.cachedDocs,
 			}
 			newIndexSnapshot.segment[i] = newSegmentSnapshot
+			// update items persisted incase of a new segment snapshot
+			atomic.AddUint64(&s.stats.numItemsPersisted, newSegmentSnapshot.Count())
 		} else {
 			newIndexSnapshot.segment[i] = s.root.segment[i]
 			newIndexSnapshot.segment[i].segment.AddRef()

index/scorch/stats.go

@@ -22,6 +22,8 @@ import (
 // Stats tracks statistics about the index
 type Stats struct {
 	analysisTime, indexTime uint64
+	numItemsIntroduced      uint64
+	numItemsPersisted       uint64
 }
 
 // FIXME wire up these other stats again
@@ -36,6 +38,8 @@ func (s *Stats) statsMap() map[string]interface{} {
 	// m["term_searchers_started"] = atomic.LoadUint64(&i.termSearchersStarted)
 	// m["term_searchers_finished"] = atomic.LoadUint64(&i.termSearchersFinished)
 	// m["num_plain_text_bytes_indexed"] = atomic.LoadUint64(&i.numPlainTextBytesIndexed)
+	m["num_items_introduced"] = atomic.LoadUint64(&s.numItemsIntroduced)
+	m["num_items_persisted"] = atomic.LoadUint64(&s.numItemsPersisted)
 
 	return m
 }