From 57a075afdb5bdd2debe8b228512e50d99e1a352c Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Fri, 5 Jan 2018 11:50:07 -0500 Subject: [PATCH] improving command-line tool for scorch --- cmd/bleve/cmd/scorch/ascii.go | 59 ++++++++++++++++++++++++++++++ cmd/bleve/cmd/scorch/deleted.go | 55 ++++++++++++++++++++++++++++ cmd/bleve/cmd/scorch/internal.go | 61 ++++++++++++++++++++++++++++++++ cmd/bleve/cmd/scorch/snapshot.go | 20 ++++++++++- index/scorch/persister.go | 22 ++++++++++++ index/scorch/snapshot_index.go | 8 +++++ index/scorch/snapshot_segment.go | 8 +++++ 7 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 cmd/bleve/cmd/scorch/ascii.go create mode 100644 cmd/bleve/cmd/scorch/deleted.go create mode 100644 cmd/bleve/cmd/scorch/internal.go diff --git a/cmd/bleve/cmd/scorch/ascii.go b/cmd/bleve/cmd/scorch/ascii.go new file mode 100644 index 00000000..7b36b5b9 --- /dev/null +++ b/cmd/bleve/cmd/scorch/ascii.go @@ -0,0 +1,59 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + "strconv" + + "github.com/blevesearch/bleve/index/scorch/mergeplan" + "github.com/spf13/cobra" +) + +// asciiCmd represents the snapshots command +var asciiCmd = &cobra.Command{ + Use: "ascii", + Short: "ascii prints details an ascii representation of the snapshots in the index", + Long: `The ascii command prints an ascii representation of the snapshots in the index.`, + RunE: func(cmd *cobra.Command, args []string) error { + + if len(args) < 2 { + return fmt.Errorf("snapshot epoch required") + } else if len(args) < 3 { + snapshotEpoch, err := strconv.ParseUint(args[1], 10, 64) + if err != nil { + return err + } + snapshot, err := index.LoadSnapshot(snapshotEpoch) + if err != nil { + return err + } + segments := snapshot.Segments() + var mergePlanSegments []mergeplan.Segment + for _, v := range segments { + mergePlanSegments = append(mergePlanSegments, v) + } + + str := mergeplan.ToBarChart(args[1], 25, mergePlanSegments, nil) + fmt.Printf("%s\n", str) + } + + return nil + }, +} + +func init() { + RootCmd.AddCommand(asciiCmd) +} diff --git a/cmd/bleve/cmd/scorch/deleted.go b/cmd/bleve/cmd/scorch/deleted.go new file mode 100644 index 00000000..cb2a9245 --- /dev/null +++ b/cmd/bleve/cmd/scorch/deleted.go @@ -0,0 +1,55 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + "strconv" + + "github.com/spf13/cobra" +) + +// deletedCmd represents the deleted command +var deletedCmd = &cobra.Command{ + Use: "deleted", + Short: "deleted prints the deleted bitmap for segments in the index snapshot", + Long: `The delete command prints the deleted bitmap for segments in the index snapshot.`, + RunE: func(cmd *cobra.Command, args []string) error { + + if len(args) < 2 { + return fmt.Errorf("snapshot epoch required") + } else if len(args) < 3 { + snapshotEpoch, err := strconv.ParseUint(args[1], 10, 64) + if err != nil { + return err + } + snapshot, err := index.LoadSnapshot(snapshotEpoch) + if err != nil { + return err + } + segments := snapshot.Segments() + for i, segmentSnap := range segments { + deleted := segmentSnap.Deleted() + fmt.Printf("%d %v\n", i, deleted) + } + } + + return nil + }, +} + +func init() { + RootCmd.AddCommand(deletedCmd) +} diff --git a/cmd/bleve/cmd/scorch/internal.go b/cmd/bleve/cmd/scorch/internal.go new file mode 100644 index 00000000..027e9028 --- /dev/null +++ b/cmd/bleve/cmd/scorch/internal.go @@ -0,0 +1,61 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + "strconv" + + "github.com/spf13/cobra" +) + +var ascii bool + +// internalCmd represents the snapshots command +var internalCmd = &cobra.Command{ + Use: "internal", + Short: "internal prints the internal k/v pairs in a snapshot", + Long: `The internal command prints the internal k/v pairs in a snapshot.`, + RunE: func(cmd *cobra.Command, args []string) error { + + if len(args) < 2 { + return fmt.Errorf("snapshot epoch required") + } else if len(args) < 3 { + snapshotEpoch, err := strconv.ParseUint(args[1], 10, 64) + if err != nil { + return err + } + snapshot, err := index.LoadSnapshot(snapshotEpoch) + if err != nil { + return err + } + internal := snapshot.Internal() + for k, v := range internal { + if ascii { + fmt.Printf("%s %s\n", k, string(v)) + } else { + fmt.Printf("%x %x\n", k, v) + } + } + } + + return nil + }, +} + +func init() { + RootCmd.AddCommand(internalCmd) + internalCmd.Flags().BoolVarP(&ascii, "ascii", "a", false, "print key/value in ascii") +} diff --git a/cmd/bleve/cmd/scorch/snapshot.go b/cmd/bleve/cmd/scorch/snapshot.go index 1c51c056..bb035ce5 100644 --- a/cmd/bleve/cmd/scorch/snapshot.go +++ b/cmd/bleve/cmd/scorch/snapshot.go @@ -16,11 +16,13 @@ package scorch import ( "fmt" + "strconv" + "github.com/blevesearch/bleve/index/scorch/segment/zap" "github.com/spf13/cobra" ) -// snapshotsCmd represents the snapshots command +// snapshotCmd represents the snapshot command var snapshotCmd = &cobra.Command{ Use: "snapshot", Short: "info prints details about the snapshots in the index", @@ -35,6 +37,22 @@ var snapshotCmd = &cobra.Command{ for _, snapshotEpoch := range snapshotEpochs { fmt.Printf("%d\n", snapshotEpoch) } + } else if len(args) < 3 { + snapshotEpoch, err := strconv.ParseUint(args[1], 10, 64) + if err != nil { + return err + } + snapshot, err := index.LoadSnapshot(snapshotEpoch) + if err != nil { + return err + } + segments := snapshot.Segments() + for i, segmentSnap := range segments { + segment := segmentSnap.Segment() + if segment, ok := segment.(*zap.Segment); ok { + fmt.Printf("%d %s\n", i, segment.Path()) + } + } } return nil diff --git a/index/scorch/persister.go b/index/scorch/persister.go index e0614151..40108654 100644 --- a/index/scorch/persister.go +++ b/index/scorch/persister.go @@ -340,6 +340,28 @@ func (s *Scorch) loadFromBolt() error { }) } +// LoadSnapshot loads the segment with the specified epoch +// NOTE: this is currently ONLY intended to be used by the command-line tool +func (s *Scorch) LoadSnapshot(epoch uint64) (rv *IndexSnapshot, err error) { + err = s.rootBolt.View(func(tx *bolt.Tx) error { + snapshots := tx.Bucket(boltSnapshotsBucket) + if snapshots == nil { + return nil + } + snapshotKey := segment.EncodeUvarintAscending(nil, epoch) + snapshot := snapshots.Bucket(snapshotKey) + if snapshot == nil { + return nil + } + rv, err = s.loadSnapshot(snapshot) + return err + }) + if err != nil { + return nil, err + } + return rv, nil +} + func (s *Scorch) loadSnapshot(snapshot *bolt.Bucket) (*IndexSnapshot, error) { rv := &IndexSnapshot{ diff --git a/index/scorch/snapshot_index.go b/index/scorch/snapshot_index.go index 074d4c41..a9b105a0 100644 --- a/index/scorch/snapshot_index.go +++ b/index/scorch/snapshot_index.go @@ -51,6 +51,14 @@ type IndexSnapshot struct { refs int64 } +func (i *IndexSnapshot) Segments() []*SegmentSnapshot { + return i.segment +} + +func (i *IndexSnapshot) Internal() map[string][]byte { + return i.internal +} + func (i *IndexSnapshot) AddRef() { i.m.Lock() i.refs++ diff --git a/index/scorch/snapshot_segment.go b/index/scorch/snapshot_segment.go index b3b8d828..0c111c50 100644 --- a/index/scorch/snapshot_segment.go +++ b/index/scorch/snapshot_segment.go @@ -56,6 +56,14 @@ type SegmentSnapshot struct { cachedDocs *cachedDocs } +func (s *SegmentSnapshot) Segment() segment.Segment { + return s.segment +} + +func (s *SegmentSnapshot) Deleted() *roaring.Bitmap { + return s.deleted +} + func (s *SegmentSnapshot) Id() uint64 { return s.id }