gibheer
/
bleve
0
0
Fork 0
Code

Merge 72d65ab1e1 into 7a98d75fc5

This commit is contained in:
Steve Yen 2018-03-29 03:07:49 +00:00 committed by GitHub
parent 7a98d75fc5 72d65ab1e1
commit 5937c53872
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 64 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
index/scorch/segment/zap/build.go
View File

@ -22,7 +22,7 @@ import (
"github.com/Smerity/govarint"
)
const version uint32 = 8
const version uint32 = 9
const fieldNotUninverted = math.MaxUint64

9
index/scorch/segment/zap/build_test.go
View File

@ -311,7 +311,8 @@ func buildTestAnalysisResultsMultiWithDifferentFields(includeDocA, includeDocB b
doc := &document.Document{
ID: "a",
Fields: []document.Field{
document.NewTextField("_id", []uint64{}, []byte("a")),
document.NewTextFieldCustom("_id", nil, []byte("a"),
document.IndexField|document.StoreField, nil),
document.NewTextField("name", []uint64{}, []byte("ABC")),
document.NewTextField("dept", []uint64{}, []byte("ABC dept")),
document.NewTextField("manages.id", []uint64{}, []byte("XYZ")),
@ -388,7 +389,8 @@ func buildTestAnalysisResultsMultiWithDifferentFields(includeDocA, includeDocB b
doc := &document.Document{
ID: "b",
Fields: []document.Field{
document.NewTextField("_id", []uint64{}, []byte("b")),
document.NewTextFieldCustom("_id", nil, []byte("b"),
document.IndexField|document.StoreField, nil),
document.NewTextField("name", []uint64{}, []byte("XYZ")),
document.NewTextField("dept", []uint64{}, []byte("ABC dept")),
document.NewTextField("reportsTo.id", []uint64{}, []byte("ABC")),
@ -468,7 +470,8 @@ func buildTestSegmentWithDefaultFieldMapping(chunkFactor uint32) (
doc := &document.Document{
ID: "a",
Fields: []document.Field{
document.NewTextField("_id", nil, []byte("a")),
document.NewTextFieldCustom("_id", nil, []byte("a"),
document.IndexField|document.StoreField, nil),
document.NewTextField("name", nil, []byte("wow")),
document.NewTextField("desc", nil, []byte("some thing")),
document.NewTextField("tag", []uint64{0}, []byte("cold")),

26
index/scorch/segment/zap/merge.go
View File

@ -622,12 +622,19 @@ func mergeStoredAndRemap(segments []*SegmentBase, drops []*roaring.Bitmap,
return 0, nil, err
}
// now walk the fields in order
for fieldID := range fieldsInv {
storedFieldValues := vals[int(fieldID)]
// _id field special case optimizes ExternalID() lookups
idFieldVal := vals[uint16(0)][0]
_, err = metaEncoder.PutU64(uint64(len(idFieldVal)))
if err != nil {
return 0, nil, err
}
stf := typs[int(fieldID)]
spf := poss[int(fieldID)]
// now walk the non-"_id" fields in order
for fieldID := 1; fieldID < len(fieldsInv); fieldID++ {
storedFieldValues := vals[fieldID]
stf := typs[fieldID]
spf := poss[fieldID]
var err2 error
curr, data, err2 = persistStoredFieldValues(fieldID,
@ -646,7 +653,9 @@ func mergeStoredAndRemap(segments []*SegmentBase, drops []*roaring.Bitmap,
docNumOffsets[newDocNum] = uint64(w.Count())
// write out the meta len and compressed data len
_, err = writeUvarints(w, uint64(len(metaBytes)), uint64(len(compressed)))
_, err = writeUvarints(w,
uint64(len(metaBytes)),
uint64(len(idFieldVal)+len(compressed)))
if err != nil {
return 0, nil, err
}
@ -655,6 +664,11 @@ func mergeStoredAndRemap(segments []*SegmentBase, drops []*roaring.Bitmap,
if err != nil {
return 0, nil, err
}
// now write the _id field val (counted as part of the 'compressed' data)
_, err = w.Write(idFieldVal)
if err != nil {
return 0, nil, err
}
// now write the compressed data
_, err = w.Write(compressed)
if err != nil {

17
index/scorch/segment/zap/new.go
View File

@ -518,7 +518,15 @@ func (s *interim) writeStoredFields() (
s.metaBuf.Reset()
data = data[:0]
for fieldID := range s.FieldsInv {
// _id field special case optimizes ExternalID() lookups
idFieldVal := docStoredFields[uint16(0)].vals[0]
_, err = metaEncoder.PutU64(uint64(len(idFieldVal)))
if err != nil {
return 0, err
}
// handle non-"_id" fields
for fieldID := 1; fieldID < len(s.FieldsInv); fieldID++ {
isf, exists := docStoredFields[uint16(fieldID)]
if exists {
curr, data, err = persistStoredFieldValues(
@ -539,7 +547,7 @@ func (s *interim) writeStoredFields() (
_, err := writeUvarints(s.w,
uint64(len(metaBytes)),
uint64(len(compressed)))
uint64(len(idFieldVal)+len(compressed)))
if err != nil {
return 0, err
}
@ -549,6 +557,11 @@ func (s *interim) writeStoredFields() (
return 0, err
}
_, err = s.w.Write(idFieldVal)
if err != nil {
return 0, err
}
_, err = s.w.Write(compressed)
if err != nil {
return 0, err

25
index/scorch/segment/zap/segment.go
View File

@ -298,16 +298,31 @@ func (s *SegmentBase) VisitDocument(num uint64, visitor segment.DocumentFieldVal
vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx)
meta, compressed := s.getDocStoredMetaAndCompressed(num)
vdc.reader.Reset(meta)
decoder := vdc.decoder
// handle _id field special case
idFieldValLen, err := decoder.GetU64()
if err != nil {
return err
}
idFieldVal := compressed[:idFieldValLen]
keepGoing := visitor("_id", byte('t'), idFieldVal, nil)
if !keepGoing {
visitDocumentCtxPool.Put(vdc)
return nil
}
// handle non-"_id" fields
compressed = compressed[idFieldValLen:]
uncompressed, err := snappy.Decode(vdc.buf[:cap(vdc.buf)], compressed)
if err != nil {
return err
}
// now decode meta and process
vdc.reader.Reset(meta)
decoder := vdc.decoder
keepGoing := true
for keepGoing {
field, err := decoder.GetU64()
if err == io.EOF {

6
index/scorch/segment/zap/segment_test.go
View File

@ -18,7 +18,6 @@ import (
"math"
"os"
"reflect"
"sort"
"testing"
"github.com/RoaringBitmap/roaring"
@ -552,7 +551,7 @@ func TestSegmentVisitableDocValueFieldsList(t *testing.T) {
}
_ = os.RemoveAll("/tmp/scorch.zap")
testSeg, expectedFields, _ := buildTestSegmentWithDefaultFieldMapping(1)
testSeg, _, _ = buildTestSegmentWithDefaultFieldMapping(1)
err = PersistSegmentBase(testSeg, "/tmp/scorch.zap")
if err != nil {
t.Fatalf("error persisting segment: %v", err)
@ -576,7 +575,7 @@ func TestSegmentVisitableDocValueFieldsList(t *testing.T) {
t.Fatalf("segment VisitableDocValueFields err: %v", err)
}
sort.Strings(expectedFields[1:]) // keep _id as first field
expectedFields := []string{"desc", "name", "tag"}
if !reflect.DeepEqual(fields, expectedFields) {
t.Errorf("expected field terms: %#v, got: %#v", expectedFields, fields)
}
@ -593,7 +592,6 @@ func TestSegmentVisitableDocValueFieldsList(t *testing.T) {
"name": []string{"wow"},
"desc": []string{"some", "thing"},
"tag": []string{"cold"},
"_id": []string{"a"},
}
if !reflect.DeepEqual(fieldTerms, expectedFieldTerms) {
t.Errorf("expected field terms: %#v, got: %#v", expectedFieldTerms, fieldTerms)