gibheer
/
bleve
0
0
Fork 0
Code

refactor dump methods 

improved test coverage
This commit is contained in:
Marty Schoch 2014-08-15 13:12:55 -04:00
parent 4d53db9fc8
commit c33f1668f7
11 changed files with 294 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
http/debug.go
View File

@ -44,21 +44,21 @@ func (h *DebugDocumentHandler) ServeHTTP(w http.ResponseWriter, req *http.Reques
// find the docID
docID := mux.Vars(req)["docID"]
rows, err := index.DumpDoc(docID)
if err != nil {
showError(w, req, fmt.Sprintf("error debugging document: %v", err), 500)
return
}
rv := make([]interface{}, 0)
for _, row := range rows {
udcRow, ok := row.(upside_down.UpsideDownCouchRow)
if ok {
rowChan := index.DumpDoc(docID)
for row := range rowChan {
switch row := row.(type) {
case error:
showError(w, req, fmt.Sprintf("error debugging document: %v", row), 500)
return
case upside_down.UpsideDownCouchRow:
tmp := struct {
Key []byte `json:"key"`
Val []byte `json:"val"`
}{
Key: udcRow.Key(),
Val: udcRow.Value(),
Key: row.Key(),
Val: row.Value(),
}
rv = append(rv, tmp)
}

6
index.go
View File

@ -39,9 +39,9 @@ type Index interface {
Fields() ([]string, error)
Dump()
DumpDoc(id string) ([]interface{}, error)
DumpFields()
DumpAll() chan interface{}
DumpDoc(id string) chan interface{}
DumpFields() chan interface{}
Close()
}

6
index/index.go
View File

@ -36,9 +36,9 @@ type Index interface {
GetInternal(key []byte) ([]byte, error)
DeleteInternal(key []byte) error
Dump()
DumpDoc(id string) ([]interface{}, error)
DumpFields()
DumpAll() chan interface{}
DumpDoc(id string) chan interface{}
DumpFields() chan interface{}
}
type FieldTerms map[string][]string

123
index/upside_down/dump.go Normal file
View File

@ -0,0 +1,123 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package upside_down
import (
"bytes"
"sort"
)
// the functions in this file are only intended to be used by
// the bleve_dump utility and the debug http handlers
// if your application relies on the, you're doing something wrong
// they may change or be removed at any time
func (udc *UpsideDownCouch) dumpPrefix(rv chan interface{}, prefix []byte) {
start := prefix
if start == nil {
start = []byte{0}
}
it := udc.store.Iterator(start)
defer it.Close()
key, val, valid := it.Current()
for valid {
if prefix != nil && !bytes.HasPrefix(key, prefix) {
break
}
row, err := ParseFromKeyValue(key, val)
if err != nil {
rv <- err
return
}
rv <- row
it.Next()
key, val, valid = it.Current()
}
}
func (udc *UpsideDownCouch) DumpAll() chan interface{} {
rv := make(chan interface{})
go func() {
defer close(rv)
udc.dumpPrefix(rv, nil)
}()
return rv
}
func (udc *UpsideDownCouch) DumpFields() chan interface{} {
rv := make(chan interface{})
go func() {
defer close(rv)
udc.dumpPrefix(rv, []byte{'f'})
}()
return rv
}
type keyset [][]byte
func (k keyset) Len() int { return len(k) }
func (k keyset) Swap(i, j int) { k[i], k[j] = k[j], k[i] }
func (k keyset) Less(i, j int) bool { return bytes.Compare(k[i], k[j]) < 0 }
// DumpDoc returns all rows in the index related to this doc id
func (udc *UpsideDownCouch) DumpDoc(id string) chan interface{} {
rv := make(chan interface{})
go func() {
defer close(rv)
back, err := udc.backIndexRowForDoc(id)
if err != nil {
rv <- err
return
}
// no such doc
if back == nil {
return
}
// build sorted list of term keys
keys := make(keyset, 0)
for _, entry := range back.entries {
tfr := NewTermFrequencyRow(entry.term, entry.field, id, 0, 0)
key := tfr.Key()
keys = append(keys, key)
}
sort.Sort(keys)
// first add all the stored rows
storedRowPrefix := NewStoredRow(id, 0, 'x', []byte{}).ScanPrefixForDoc()
udc.dumpPrefix(rv, storedRowPrefix)
// now walk term keys in order and add them as well
if len(keys) > 0 {
it := udc.store.Iterator(keys[0])
defer it.Close()
for _, key := range keys {
it.Seek(key)
rkey, rval, valid := it.Current()
if !valid {
break
}
row, err := ParseFromKeyValue(rkey, rval)
if err != nil {
rv <- err
return
}
rv <- row
}
}
}()
return rv
}

106
index/upside_down/dump_test.go Normal file
View File

@ -0,0 +1,106 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package upside_down
import (
"github.com/couchbaselabs/bleve/index/store/leveldb"
"os"
"testing"
"time"
"github.com/couchbaselabs/bleve/document"
)
func TestDump(t *testing.T) {
defer os.RemoveAll("test")
store, err := leveldb.Open("test", true)
if err != nil {
t.Error(err)
}
idx := NewUpsideDownCouch(store)
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer idx.Close()
var expectedCount uint64 = 0
docCount := idx.DocCount()
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []byte("test"), document.INDEX_FIELD|document.STORE_FIELD))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", 35.99, document.INDEX_FIELD|document.STORE_FIELD))
doc.AddField(document.NewDateTimeFieldWithIndexingOptions("unixEpoch", time.Unix(0, 0), document.INDEX_FIELD|document.STORE_FIELD))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
doc = document.NewDocument("2")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []byte("test2"), document.INDEX_FIELD|document.STORE_FIELD))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", 35.99, document.INDEX_FIELD|document.STORE_FIELD))
doc.AddField(document.NewDateTimeFieldWithIndexingOptions("unixEpoch", time.Unix(0, 0), document.INDEX_FIELD|document.STORE_FIELD))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
fieldsCount := 0
fieldsRows := idx.DumpFields()
for _ = range fieldsRows {
fieldsCount++
}
if fieldsCount != 3 {
t.Errorf("expected 3 fields, got %d", fieldsCount)
}
// 1 text term
// 16 numeric terms
// 16 date terms
// 3 stored fields
expectedDocRowCount := int(1 + (2 * (64 / document.DEFAULT_PRECISION_STEP)) + 3)
docRowCount := 0
docRows := idx.DumpDoc("1")
for _ = range docRows {
docRowCount++
}
if docRowCount != expectedDocRowCount {
t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount)
}
docRowCount = 0
docRows = idx.DumpDoc("2")
for _ = range docRows {
docRowCount++
}
if docRowCount != expectedDocRowCount {
t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount)
}
// 1 version
// fieldsCount field rows
// 2 docs * expectedDocRowCount
// 2 back index rows
// 2 text term row count (2 different text terms)
// 16 numeric term row counts (shared for both docs, same numeric value)
// 16 date term row counts (shared for both docs, same date value)
expectedAllRowCount := int(1 + fieldsCount + (2 * expectedDocRowCount) + 2 + 2 + int((2 * (64 / document.DEFAULT_PRECISION_STEP))))
allRowCount := 0
allRows := idx.DumpAll()
for _ = range allRows {
allRowCount++
}
if allRowCount != expectedAllRowCount {
t.Errorf("expected %d rows for all, got %d", expectedAllRowCount, allRowCount)
}
}

2
index/upside_down/reader.go
View File

@ -155,7 +155,7 @@ func (r *UpsideDownCouchDocIdReader) Advance(docId string) (string, error) {
key, val, valid := r.iterator.Current()
if valid {
bier := NewBackIndexRow(r.end, nil, nil)
if bytes.Compare(key, bier.Key()) < 0 {
if bytes.Compare(key, bier.Key()) > 0 {
// end of the line
return "", nil
}

24
index/upside_down/reader_test.go
View File

@ -194,6 +194,7 @@ func TestIndexDocIdReader(t *testing.T) {
if err != nil {
t.Errorf("Error accessing doc id reader: %v", err)
}
defer reader.Close()
id, err := reader.Next()
count := uint64(0)
@ -204,4 +205,27 @@ func TestIndexDocIdReader(t *testing.T) {
if count != expectedCount {
t.Errorf("expected %d, got %d", expectedCount, count)
}
// try it again, but jump to the second doc this time
reader, err = idx.DocIdReader("", "")
if err != nil {
t.Errorf("Error accessing doc id reader: %v", err)
}
defer reader.Close()
id, err = reader.Advance("2")
if err != nil {
t.Error(err)
}
if id != "2" {
t.Errorf("expected to find id '2', got '%s'", id)
}
id, err = reader.Advance("3")
if err != nil {
t.Error(err)
}
if id != "" {
t.Errorf("expected to find id '', got '%s'", id)
}
}

88
index/upside_down/upside_down.go
View File

@ -12,10 +12,8 @@ import (
"bytes"
"fmt"
"math"
"sort"
"github.com/couchbaselabs/bleve/analysis"
"github.com/couchbaselabs/bleve/document"
"github.com/couchbaselabs/bleve/index"
"github.com/couchbaselabs/bleve/index/store"
@ -715,89 +713,3 @@ func (udc *UpsideDownCouch) DeleteInternal(key []byte) error {
internalRow := NewInternalRow(key, nil)
return udc.store.Delete(internalRow.Key())
}
func (udc *UpsideDownCouch) Dump() {
it := udc.store.Iterator([]byte{0})
defer it.Close()
key, val, valid := it.Current()
for valid {
row, err := ParseFromKeyValue(key, val)
if err != nil {
fmt.Printf("error parsing key/value: %v", err)
return
}
if row != nil {
fmt.Printf("%v\n", row)
fmt.Printf("Key: % -100x\nValue: % -100x\n\n", key, val)
}
it.Next()
key, val, valid = it.Current()
}
}
func (udc *UpsideDownCouch) DumpFields() {
it := udc.store.Iterator([]byte{'f'})
defer it.Close()
key, val, valid := it.Current()
for valid {
if !bytes.HasPrefix(key, []byte{'f'}) {
break
}
row, err := ParseFromKeyValue(key, val)
if err != nil {
fmt.Printf("error parsing key/value: %v", err)
return
}
if row != nil {
fmt.Printf("%v\n", row)
fmt.Printf("Key: % -100x\nValue: % -100x\n\n", key, val)
}
it.Next()
key, val, valid = it.Current()
}
}
type keyset [][]byte
func (k keyset) Len() int { return len(k) }
func (k keyset) Swap(i, j int) { k[i], k[j] = k[j], k[i] }
func (k keyset) Less(i, j int) bool { return bytes.Compare(k[i], k[j]) < 0 }
// DumpDoc returns all rows in the index related to this doc id
func (udc *UpsideDownCouch) DumpDoc(id string) ([]interface{}, error) {
rv := make([]interface{}, 0)
back, err := udc.backIndexRowForDoc(id)
if err != nil {
return nil, err
}
keys := make(keyset, 0)
for _, stored := range back.storedFields {
sr := NewStoredRow(id, stored, 'x', []byte{})
key := sr.Key()
keys = append(keys, key)
}
for _, entry := range back.entries {
tfr := NewTermFrequencyRow(entry.term, entry.field, id, 0, 0)
key := tfr.Key()
keys = append(keys, key)
}
sort.Sort(keys)
for _, key := range keys {
value, err := udc.store.Get(key)
if err != nil {
return nil, err
}
row, err := ParseFromKeyValue(key, value)
if err != nil {
return nil, err
}
rv = append(rv, row)
}
return rv, nil
}

2
index/upside_down/upside_down_test.go
View File

@ -804,7 +804,6 @@ func TestIndexDocumentFieldTerms(t *testing.T) {
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []byte("test"), document.INDEX_FIELD|document.STORE_FIELD|document.INCLUDE_TERM_VECTORS))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []byte("mister"), document.INDEX_FIELD|document.STORE_FIELD|document.INCLUDE_TERM_VECTORS))
doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.INDEX_FIELD|document.INCLUDE_TERM_VECTORS))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
@ -817,7 +816,6 @@ func TestIndexDocumentFieldTerms(t *testing.T) {
expectedFieldTerms := index.FieldTerms{
"name": []string{"test"},
"title": []string{"mister"},
"_all": []string{"test", "mister"},
}
if !reflect.DeepEqual(fieldTerms, expectedFieldTerms) {
t.Errorf("expected field terms: %#v, got: %#v", expectedFieldTerms, fieldTerms)

10
index_impl.go
View File

@ -210,19 +210,19 @@ func (i *indexImpl) Search(req *SearchRequest) (*SearchResult, error) {
}, nil
}
func (i *indexImpl) Dump() {
i.i.Dump()
func (i *indexImpl) DumpAll() chan interface{} {
return i.i.DumpAll()
}
func (i *indexImpl) Fields() ([]string, error) {
return i.i.Fields()
}
func (i *indexImpl) DumpFields() {
i.i.DumpFields()
func (i *indexImpl) DumpFields() chan interface{} {
return i.i.DumpFields()
}
func (i *indexImpl) DumpDoc(id string) ([]interface{}, error) {
func (i *indexImpl) DumpDoc(id string) chan interface{} {
return i.i.DumpDoc(id)
}

22
utils/bleve_dump/main.go
View File

@ -10,14 +10,17 @@ package main
import (
"flag"
"fmt"
"log"
"github.com/couchbaselabs/bleve"
"github.com/couchbaselabs/bleve/index/upside_down"
)
var indexDir = flag.String("indexDir", "index", "index directory")
var fieldsOnly = flag.Bool("fields", false, "fields only")
var docId = flag.String("docId", "", "docId to dump")
func main() {
flag.Parse()
@ -29,9 +32,22 @@ func main() {
}
defer index.Close()
if !*fieldsOnly {
index.Dump()
var dumpChan chan interface{}
if *docId != "" {
dumpChan = index.DumpDoc(*docId)
} else if *fieldsOnly {
dumpChan = index.DumpFields()
} else {
index.DumpFields()
dumpChan = index.DumpAll()
}
for rowOrErr := range dumpChan {
switch rowOrErr := rowOrErr.(type) {
case error:
log.Printf("error dumping: %v", rowOrErr)
case upside_down.UpsideDownCouchRow:
fmt.Printf("%v\n", rowOrErr)
fmt.Printf("Key: % -100x\nValue: % -100x\n\n", rowOrErr.Key(), rowOrErr.Value())
}
}
}