improving command-line tool for scorch

cmd/bleve/cmd/scorch/ascii.go

@@ -0,0 +1,59 @@
+//  Copyright (c) 2017 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scorch
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/blevesearch/bleve/index/scorch/mergeplan"
+	"github.com/spf13/cobra"
+)
+
+// asciiCmd represents the snapshots command
+var asciiCmd = &cobra.Command{
+	Use:   "ascii",
+	Short: "ascii prints details an ascii representation of the  snapshots in the index",
+	Long:  `The ascii command prints an ascii representation of the  snapshots in the index.`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+
+		if len(args) < 2 {
+			return fmt.Errorf("snapshot epoch required")
+		} else if len(args) < 3 {
+			snapshotEpoch, err := strconv.ParseUint(args[1], 10, 64)
+			if err != nil {
+				return err
+			}
+			snapshot, err := index.LoadSnapshot(snapshotEpoch)
+			if err != nil {
+				return err
+			}
+			segments := snapshot.Segments()
+			var mergePlanSegments []mergeplan.Segment
+			for _, v := range segments {
+				mergePlanSegments = append(mergePlanSegments, v)
+			}
+
+			str := mergeplan.ToBarChart(args[1], 25, mergePlanSegments, nil)
+			fmt.Printf("%s\n", str)
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(asciiCmd)
+}

cmd/bleve/cmd/scorch/deleted.go

@@ -0,0 +1,55 @@
+//  Copyright (c) 2017 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scorch
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/spf13/cobra"
+)
+
+// deletedCmd represents the deleted command
+var deletedCmd = &cobra.Command{
+	Use:   "deleted",
+	Short: "deleted prints the deleted bitmap for segments in the index snapshot",
+	Long:  `The delete command prints the deleted bitmap for segments in the index snapshot.`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+
+		if len(args) < 2 {
+			return fmt.Errorf("snapshot epoch required")
+		} else if len(args) < 3 {
+			snapshotEpoch, err := strconv.ParseUint(args[1], 10, 64)
+			if err != nil {
+				return err
+			}
+			snapshot, err := index.LoadSnapshot(snapshotEpoch)
+			if err != nil {
+				return err
+			}
+			segments := snapshot.Segments()
+			for i, segmentSnap := range segments {
+				deleted := segmentSnap.Deleted()
+				fmt.Printf("%d %v\n", i, deleted)
+			}
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(deletedCmd)
+}

cmd/bleve/cmd/scorch/internal.go

@@ -0,0 +1,61 @@
+//  Copyright (c) 2017 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scorch
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/spf13/cobra"
+)
+
+var ascii bool
+
+// internalCmd represents the snapshots command
+var internalCmd = &cobra.Command{
+	Use:   "internal",
+	Short: "internal prints the internal k/v pairs in a snapshot",
+	Long:  `The internal command prints the internal k/v pairs in a snapshot.`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+
+		if len(args) < 2 {
+			return fmt.Errorf("snapshot epoch required")
+		} else if len(args) < 3 {
+			snapshotEpoch, err := strconv.ParseUint(args[1], 10, 64)
+			if err != nil {
+				return err
+			}
+			snapshot, err := index.LoadSnapshot(snapshotEpoch)
+			if err != nil {
+				return err
+			}
+			internal := snapshot.Internal()
+			for k, v := range internal {
+				if ascii {
+					fmt.Printf("%s %s\n", k, string(v))
+				} else {
+					fmt.Printf("%x %x\n", k, v)
+				}
+			}
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(internalCmd)
+	internalCmd.Flags().BoolVarP(&ascii, "ascii", "a", false, "print key/value in ascii")
+}

cmd/bleve/cmd/scorch/snapshot.go

@@ -16,11 +16,13 @@ package scorch
 
 import (
 	"fmt"
+	"strconv"
 
+	"github.com/blevesearch/bleve/index/scorch/segment/zap"
 	"github.com/spf13/cobra"
 )
 
-// snapshotsCmd represents the snapshots command
+// snapshotCmd represents the snapshot command
 var snapshotCmd = &cobra.Command{
 	Use:   "snapshot",
 	Short: "info prints details about the snapshots in the index",
@@ -35,6 +37,22 @@ var snapshotCmd = &cobra.Command{
 			for _, snapshotEpoch := range snapshotEpochs {
 				fmt.Printf("%d\n", snapshotEpoch)
 			}
+		} else if len(args) < 3 {
+			snapshotEpoch, err := strconv.ParseUint(args[1], 10, 64)
+			if err != nil {
+				return err
+			}
+			snapshot, err := index.LoadSnapshot(snapshotEpoch)
+			if err != nil {
+				return err
+			}
+			segments := snapshot.Segments()
+			for i, segmentSnap := range segments {
+				segment := segmentSnap.Segment()
+				if segment, ok := segment.(*zap.Segment); ok {
+					fmt.Printf("%d %s\n", i, segment.Path())
+				}
+			}
 		}
 
 		return nil

index/scorch/persister.go

@@ -340,6 +340,28 @@ func (s *Scorch) loadFromBolt() error {
 	})
 }
 
+// LoadSnapshot loads the segment with the specified epoch
+// NOTE: this is currently ONLY intended to be used by the command-line tool
+func (s *Scorch) LoadSnapshot(epoch uint64) (rv *IndexSnapshot, err error) {
+	err = s.rootBolt.View(func(tx *bolt.Tx) error {
+		snapshots := tx.Bucket(boltSnapshotsBucket)
+		if snapshots == nil {
+			return nil
+		}
+		snapshotKey := segment.EncodeUvarintAscending(nil, epoch)
+		snapshot := snapshots.Bucket(snapshotKey)
+		if snapshot == nil {
+			return nil
+		}
+		rv, err = s.loadSnapshot(snapshot)
+		return err
+	})
+	if err != nil {
+		return nil, err
+	}
+	return rv, nil
+}
+
 func (s *Scorch) loadSnapshot(snapshot *bolt.Bucket) (*IndexSnapshot, error) {
 
 	rv := &IndexSnapshot{

index/scorch/snapshot_index.go

@@ -51,6 +51,14 @@ type IndexSnapshot struct {
 	refs int64
 }
 
+func (i *IndexSnapshot) Segments() []*SegmentSnapshot {
+	return i.segment
+}
+
+func (i *IndexSnapshot) Internal() map[string][]byte {
+	return i.internal
+}
+
 func (i *IndexSnapshot) AddRef() {
 	i.m.Lock()
 	i.refs++

index/scorch/snapshot_segment.go

@@ -56,6 +56,14 @@ type SegmentSnapshot struct {
 	cachedDocs *cachedDocs
 }
 
+func (s *SegmentSnapshot) Segment() segment.Segment {
+	return s.segment
+}
+
+func (s *SegmentSnapshot) Deleted() *roaring.Bitmap {
+	return s.deleted
+}
+
 func (s *SegmentSnapshot) Id() uint64 {
 	return s.id
 }