merge deletion and cacheddocs fixes discussed in meeting

index/scorch/introducer.go

@@ -93,9 +93,9 @@ func (s *Scorch) introduceSegment(next *segmentIntroduction) error {
 			}
 		}
 		newSnapshot.segment[i] = &SegmentSnapshot{
-			id:      s.root.segment[i].id,
-			segment: s.root.segment[i].segment,
-			notify:  s.root.segment[i].notify,
+			id:         s.root.segment[i].id,
+			segment:    s.root.segment[i].segment,
+			notify:     s.root.segment[i].notify,
 			cachedDocs: s.root.segment[i].cachedDocs,
 		}
 		s.root.segment[i].segment.AddRef()
@@ -112,8 +112,8 @@ func (s *Scorch) introduceSegment(next *segmentIntroduction) error {
 	}
 	// put new segment at end
 	newSnapshot.segment[len(s.root.segment)] = &SegmentSnapshot{
-		id:      next.id,
-		segment: next.data, // Take ownership of next.data's ref-count.
+		id:         next.id,
+		segment:    next.data, // Take ownership of next.data's ref-count.
 		cachedDocs: &cachedDocs{cache: nil},
 	}
 	newSnapshot.offsets[len(s.root.segment)] = running
@@ -191,10 +191,11 @@ func (s *Scorch) introduceMerge(nextMerge *segmentMerge) {
 		} else {
 			// this segment is staying
 			newSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{
-				id:      s.root.segment[i].id,
-				segment: s.root.segment[i].segment,
-				notify:  s.root.segment[i].notify,
-				deleted: s.root.segment[i].deleted,
+				id:         s.root.segment[i].id,
+				segment:    s.root.segment[i].segment,
+				notify:     s.root.segment[i].notify,
+				deleted:    s.root.segment[i].deleted,
+				cachedDocs: s.root.segment[i].cachedDocs,
 			})
 			s.root.segment[i].segment.AddRef()
 			newSnapshot.offsets = append(newSnapshot.offsets, running)
@@ -204,9 +205,10 @@ func (s *Scorch) introduceMerge(nextMerge *segmentMerge) {
 
 	// put new segment at end
 	newSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{
-		id:      nextMerge.id,
-		segment: nextMerge.new, // Take ownership for nextMerge.new's ref-count.
-		deleted: newSegmentDeleted,
+		id:         nextMerge.id,
+		segment:    nextMerge.new, // Take ownership for nextMerge.new's ref-count.
+		deleted:    newSegmentDeleted,
+		cachedDocs: &cachedDocs{cache: nil},
 	})
 	newSnapshot.offsets = append(newSnapshot.offsets, running)
 

index/scorch/merge.go

@@ -129,7 +129,7 @@ func (s *Scorch) planMergeAtSnapshot(ourSnapshot *IndexSnapshot) error {
 			}
 		}
 
-		filename := fmt.Sprintf("%x.zap", newSegmentID)
+		filename := fmt.Sprintf("%08x.zap", newSegmentID)
 		path := s.path + string(os.PathSeparator) + filename
 		newDocNums, err := zap.Merge(segmentsToMerge, docsToDrop, path, 1024)
 		if err != nil {

index/scorch/persister.go

@@ -171,7 +171,7 @@ func (s *Scorch) persistSnapshot(snapshot *IndexSnapshot) error {
 		switch seg := segmentSnapshot.segment.(type) {
 		case *mem.Segment:
 			// need to persist this to disk
-			filename := fmt.Sprintf("%x.zap", segmentSnapshot.id)
+			filename := fmt.Sprintf("%08x.zap", segmentSnapshot.id)
 			path := s.path + string(os.PathSeparator) + filename
 			err2 := zap.PersistSegment(seg, path, 1024)
 			if err2 != nil {
@@ -234,9 +234,10 @@ func (s *Scorch) persistSnapshot(snapshot *IndexSnapshot) error {
 		// see if this segment has been replaced
 		if replacement, ok := newSegments[segmentSnapshot.id]; ok {
 			newSegmentSnapshot := &SegmentSnapshot{
-				segment: replacement,
-				deleted: segmentSnapshot.deleted,
-				id:      segmentSnapshot.id,
+				segment:    replacement,
+				deleted:    segmentSnapshot.deleted,
+				id:         segmentSnapshot.id,
+				cachedDocs: segmentSnapshot.cachedDocs,
 			}
 			newIndexSnapshot.segment[i] = newSegmentSnapshot
 			// add the old segment snapshots notifications to the list
@@ -382,7 +383,8 @@ func (s *Scorch) loadSegment(segmentBucket *bolt.Bucket) (*SegmentSnapshot, erro
 	}
 
 	rv := &SegmentSnapshot{
-		segment: segment,
+		segment:    segment,
+		cachedDocs: &cachedDocs{cache: nil},
 	}
 	deletedBytes := segmentBucket.Get(boltDeletedKey)
 	if deletedBytes != nil {
@@ -474,7 +476,7 @@ func (s *Scorch) removeOldBoltSnapshots() (numRemoved int, err error) {
 
 // Removes any *.zap files which aren't listed in the rootBolt.
 func (s *Scorch) removeOldZapFiles() error {
-	liveFileNames, err := s.loadZapFileNames()
+	liveFileNames, highestName, err := s.loadZapFileNames()
 	if err != nil {
 		return err
 	}
@@ -487,7 +489,7 @@ func (s *Scorch) removeOldZapFiles() error {
 	for _, finfo := range currFileInfos {
 		fname := finfo.Name()
 		if filepath.Ext(fname) == ".zap" {
-			if _, exists := liveFileNames[fname]; !exists {
+			if _, exists := liveFileNames[fname]; !exists && fname < highestName {
 				err := os.Remove(s.path + string(os.PathSeparator) + fname)
 				if err != nil {
 					log.Printf("got err removing file: %s, err: %v", fname, err)
@@ -500,8 +502,9 @@ func (s *Scorch) removeOldZapFiles() error {
 }
 
 // Returns the *.zap file names that are listed in the rootBolt.
-func (s *Scorch) loadZapFileNames() (map[string]struct{}, error) {
+func (s *Scorch) loadZapFileNames() (map[string]struct{}, string, error) {
 	rv := map[string]struct{}{}
+	var highest string
 	err := s.rootBolt.View(func(tx *bolt.Tx) error {
 		snapshots := tx.Bucket(boltSnapshotsBucket)
 		if snapshots == nil {
@@ -526,11 +529,15 @@ func (s *Scorch) loadZapFileNames() (map[string]struct{}, error) {
 				if pathBytes == nil {
 					continue
 				}
-				rv[string(pathBytes)] = struct{}{}
+				pathString := string(pathBytes)
+				if pathString > highest {
+					highest = pathString
+				}
+				rv[string(pathString)] = struct{}{}
 			}
 		}
 		return nil
 	})
 
-	return rv, err
+	return rv, highest, err
 }

index/scorch/snapshot_index.go

@@ -401,6 +401,7 @@ func docInternalToNumber(in index.IndexInternalID) (uint64, error) {
 
 func (i *IndexSnapshot) DocumentVisitFieldTerms(id index.IndexInternalID,
 	fields []string, visitor index.DocumentFieldTermVisitor) error {
+
 	docNum, err := docInternalToNumber(id)
 	if err != nil {
 		return err
@@ -410,12 +411,7 @@ func (i *IndexSnapshot) DocumentVisitFieldTerms(id index.IndexInternalID,
 		return nil
 	}
 
-	i.m.Lock()
 	ss := i.segment[segmentIndex]
-	if ss.cachedDocs == nil {
-		ss.cachedDocs = &cachedDocs{cache: nil}
-	}
-	i.m.Unlock()
 
 	err = ss.cachedDocs.prepareFields(localDocNum, fields, ss)
 	if err != nil {