281 lines
7.0 KiB
Go
281 lines
7.0 KiB
Go
|
package zap
|
||
|
|
||
|
import (
|
||
|
"os"
|
||
|
"testing"
|
||
|
|
||
|
"github.com/RoaringBitmap/roaring"
|
||
|
"github.com/blevesearch/bleve/analysis"
|
||
|
"github.com/blevesearch/bleve/document"
|
||
|
"github.com/blevesearch/bleve/index"
|
||
|
"github.com/blevesearch/bleve/index/scorch/segment/mem"
|
||
|
)
|
||
|
|
||
|
func TestMerge(t *testing.T) {
|
||
|
_ = os.RemoveAll("/tmp/scorch.zap")
|
||
|
_ = os.RemoveAll("/tmp/scorch2.zap")
|
||
|
_ = os.RemoveAll("/tmp/scorch3.zap")
|
||
|
|
||
|
memSegment := buildMemSegmentMulti()
|
||
|
err := PersistSegment(memSegment, "/tmp/scorch.zap", 1024)
|
||
|
if err != nil {
|
||
|
t.Fatal(err)
|
||
|
}
|
||
|
|
||
|
memSegment2 := buildMemSegmentMulti2()
|
||
|
err = PersistSegment(memSegment2, "/tmp/scorch2.zap", 1024)
|
||
|
if err != nil {
|
||
|
t.Fatal(err)
|
||
|
}
|
||
|
|
||
|
segment, err := Open("/tmp/scorch.zap")
|
||
|
if err != nil {
|
||
|
t.Fatalf("error opening segment: %v", err)
|
||
|
}
|
||
|
defer func() {
|
||
|
cerr := segment.Close()
|
||
|
if cerr != nil {
|
||
|
t.Fatalf("error closing segment: %v", err)
|
||
|
}
|
||
|
}()
|
||
|
|
||
|
segment2, err := Open("/tmp/scorch2.zap")
|
||
|
if err != nil {
|
||
|
t.Fatalf("error opening segment: %v", err)
|
||
|
}
|
||
|
defer func() {
|
||
|
cerr := segment2.Close()
|
||
|
if cerr != nil {
|
||
|
t.Fatalf("error closing segment: %v", err)
|
||
|
}
|
||
|
}()
|
||
|
|
||
|
segsToMerge := make([]*Segment, 2)
|
||
|
segsToMerge[0] = segment.(*Segment)
|
||
|
segsToMerge[1] = segment2.(*Segment)
|
||
|
|
||
|
err = Merge(segsToMerge, []*roaring.Bitmap{nil, nil}, "/tmp/scorch3.zap", 1024)
|
||
|
if err != nil {
|
||
|
t.Fatal(err)
|
||
|
}
|
||
|
}
|
||
|
|
||
|
func TestMergeAndDrop(t *testing.T) {
|
||
|
_ = os.RemoveAll("/tmp/scorch.zap")
|
||
|
_ = os.RemoveAll("/tmp/scorch2.zap")
|
||
|
_ = os.RemoveAll("/tmp/scorch3.zap")
|
||
|
|
||
|
memSegment := buildMemSegmentMulti()
|
||
|
err := PersistSegment(memSegment, "/tmp/scorch.zap", 1024)
|
||
|
if err != nil {
|
||
|
t.Fatal(err)
|
||
|
}
|
||
|
|
||
|
memSegment2 := buildMemSegmentMulti2()
|
||
|
err = PersistSegment(memSegment2, "/tmp/scorch2.zap", 1024)
|
||
|
if err != nil {
|
||
|
t.Fatal(err)
|
||
|
}
|
||
|
|
||
|
segment, err := Open("/tmp/scorch.zap")
|
||
|
if err != nil {
|
||
|
t.Fatalf("error opening segment: %v", err)
|
||
|
}
|
||
|
defer func() {
|
||
|
cerr := segment.Close()
|
||
|
if cerr != nil {
|
||
|
t.Fatalf("error closing segment: %v", err)
|
||
|
}
|
||
|
}()
|
||
|
|
||
|
segment2, err := Open("/tmp/scorch2.zap")
|
||
|
if err != nil {
|
||
|
t.Fatalf("error opening segment: %v", err)
|
||
|
}
|
||
|
defer func() {
|
||
|
cerr := segment2.Close()
|
||
|
if cerr != nil {
|
||
|
t.Fatalf("error closing segment: %v", err)
|
||
|
}
|
||
|
}()
|
||
|
|
||
|
segsToMerge := make([]*Segment, 2)
|
||
|
segsToMerge[0] = segment.(*Segment)
|
||
|
segsToMerge[1] = segment2.(*Segment)
|
||
|
|
||
|
docsToDrop := make([]*roaring.Bitmap, 2)
|
||
|
docsToDrop[0] = roaring.NewBitmap()
|
||
|
docsToDrop[0].AddInt(1)
|
||
|
docsToDrop[1] = roaring.NewBitmap()
|
||
|
docsToDrop[1].AddInt(1)
|
||
|
|
||
|
err = Merge(segsToMerge, docsToDrop, "/tmp/scorch3.zap", 1024)
|
||
|
if err != nil {
|
||
|
t.Fatal(err)
|
||
|
}
|
||
|
}
|
||
|
|
||
|
func buildMemSegmentMulti2() *mem.Segment {
|
||
|
|
||
|
doc := &document.Document{
|
||
|
ID: "c",
|
||
|
Fields: []document.Field{
|
||
|
document.NewTextFieldCustom("_id", nil, []byte("c"), document.IndexField|document.StoreField, nil),
|
||
|
document.NewTextFieldCustom("name", nil, []byte("mat"), document.IndexField|document.StoreField|document.IncludeTermVectors, nil),
|
||
|
document.NewTextFieldCustom("desc", nil, []byte("some thing"), document.IndexField|document.StoreField|document.IncludeTermVectors, nil),
|
||
|
document.NewTextFieldCustom("tag", []uint64{0}, []byte("cold"), document.IndexField|document.StoreField|document.IncludeTermVectors, nil),
|
||
|
document.NewTextFieldCustom("tag", []uint64{1}, []byte("dark"), document.IndexField|document.StoreField|document.IncludeTermVectors, nil),
|
||
|
},
|
||
|
CompositeFields: []*document.CompositeField{
|
||
|
document.NewCompositeField("_all", true, nil, []string{"_id"}),
|
||
|
},
|
||
|
}
|
||
|
|
||
|
doc2 := &document.Document{
|
||
|
ID: "d",
|
||
|
Fields: []document.Field{
|
||
|
document.NewTextFieldCustom("_id", nil, []byte("d"), document.IndexField|document.StoreField, nil),
|
||
|
document.NewTextFieldCustom("name", nil, []byte("joa"), document.IndexField|document.StoreField|document.IncludeTermVectors, nil),
|
||
|
document.NewTextFieldCustom("desc", nil, []byte("some thing"), document.IndexField|document.StoreField|document.IncludeTermVectors, nil),
|
||
|
document.NewTextFieldCustom("tag", []uint64{0}, []byte("cold"), document.IndexField|document.StoreField|document.IncludeTermVectors, nil),
|
||
|
document.NewTextFieldCustom("tag", []uint64{1}, []byte("dark"), document.IndexField|document.StoreField|document.IncludeTermVectors, nil),
|
||
|
},
|
||
|
CompositeFields: []*document.CompositeField{
|
||
|
document.NewCompositeField("_all", true, nil, []string{"_id"}),
|
||
|
},
|
||
|
}
|
||
|
|
||
|
// forge analyzed docs
|
||
|
results := []*index.AnalysisResult{
|
||
|
&index.AnalysisResult{
|
||
|
Document: doc,
|
||
|
Analyzed: []analysis.TokenFrequencies{
|
||
|
analysis.TokenFrequency(analysis.TokenStream{
|
||
|
&analysis.Token{
|
||
|
Start: 0,
|
||
|
End: 1,
|
||
|
Position: 1,
|
||
|
Term: []byte("c"),
|
||
|
},
|
||
|
}, nil, false),
|
||
|
analysis.TokenFrequency(analysis.TokenStream{
|
||
|
&analysis.Token{
|
||
|
Start: 0,
|
||
|
End: 3,
|
||
|
Position: 1,
|
||
|
Term: []byte("mat"),
|
||
|
},
|
||
|
}, nil, true),
|
||
|
analysis.TokenFrequency(analysis.TokenStream{
|
||
|
&analysis.Token{
|
||
|
Start: 0,
|
||
|
End: 4,
|
||
|
Position: 1,
|
||
|
Term: []byte("some"),
|
||
|
},
|
||
|
&analysis.Token{
|
||
|
Start: 5,
|
||
|
End: 10,
|
||
|
Position: 2,
|
||
|
Term: []byte("thing"),
|
||
|
},
|
||
|
}, nil, true),
|
||
|
analysis.TokenFrequency(analysis.TokenStream{
|
||
|
&analysis.Token{
|
||
|
Start: 0,
|
||
|
End: 4,
|
||
|
Position: 1,
|
||
|
Term: []byte("cold"),
|
||
|
},
|
||
|
}, []uint64{0}, true),
|
||
|
analysis.TokenFrequency(analysis.TokenStream{
|
||
|
&analysis.Token{
|
||
|
Start: 0,
|
||
|
End: 4,
|
||
|
Position: 1,
|
||
|
Term: []byte("dark"),
|
||
|
},
|
||
|
}, []uint64{1}, true),
|
||
|
},
|
||
|
Length: []int{
|
||
|
1,
|
||
|
1,
|
||
|
2,
|
||
|
1,
|
||
|
1,
|
||
|
},
|
||
|
},
|
||
|
&index.AnalysisResult{
|
||
|
Document: doc2,
|
||
|
Analyzed: []analysis.TokenFrequencies{
|
||
|
analysis.TokenFrequency(analysis.TokenStream{
|
||
|
&analysis.Token{
|
||
|
Start: 0,
|
||
|
End: 1,
|
||
|
Position: 1,
|
||
|
Term: []byte("d"),
|
||
|
},
|
||
|
}, nil, false),
|
||
|
analysis.TokenFrequency(analysis.TokenStream{
|
||
|
&analysis.Token{
|
||
|
Start: 0,
|
||
|
End: 3,
|
||
|
Position: 1,
|
||
|
Term: []byte("joa"),
|
||
|
},
|
||
|
}, nil, true),
|
||
|
analysis.TokenFrequency(analysis.TokenStream{
|
||
|
&analysis.Token{
|
||
|
Start: 0,
|
||
|
End: 4,
|
||
|
Position: 1,
|
||
|
Term: []byte("some"),
|
||
|
},
|
||
|
&analysis.Token{
|
||
|
Start: 5,
|
||
|
End: 10,
|
||
|
Position: 2,
|
||
|
Term: []byte("thing"),
|
||
|
},
|
||
|
}, nil, true),
|
||
|
analysis.TokenFrequency(analysis.TokenStream{
|
||
|
&analysis.Token{
|
||
|
Start: 0,
|
||
|
End: 4,
|
||
|
Position: 1,
|
||
|
Term: []byte("cold"),
|
||
|
},
|
||
|
}, []uint64{0}, true),
|
||
|
analysis.TokenFrequency(analysis.TokenStream{
|
||
|
&analysis.Token{
|
||
|
Start: 0,
|
||
|
End: 4,
|
||
|
Position: 1,
|
||
|
Term: []byte("dark"),
|
||
|
},
|
||
|
}, []uint64{1}, true),
|
||
|
},
|
||
|
Length: []int{
|
||
|
1,
|
||
|
1,
|
||
|
2,
|
||
|
1,
|
||
|
1,
|
||
|
},
|
||
|
},
|
||
|
}
|
||
|
|
||
|
// fix up composite fields
|
||
|
for _, ar := range results {
|
||
|
for i, f := range ar.Document.Fields {
|
||
|
for _, cf := range ar.Document.CompositeFields {
|
||
|
cf.Compose(f.Name(), ar.Length[i], ar.Analyzed[i])
|
||
|
}
|
||
|
}
|
||
|
}
|
||
|
|
||
|
segment := mem.NewFromAnalyzedDocs(results)
|
||
|
|
||
|
return segment
|
||
|
}
|