gibheer/bleve
0
0
Fork 0
Code

Merge pull request #665 from steveyen/scorch

Browse Source 
scorch mergeplan added TierGrowth option
This commit is contained in:
Steve Yen 2017-12-16 14:44:27 -08:00 committed by GitHub
commit cf1dd4cb00
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 21 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
index/scorch/mergeplan/merge_plan.go
View File

@ -75,6 +75,10 @@ type MergePlanOptions struct {
// planner’s predicted sizes.
MaxSegmentSize int64
// The growth factor for each tier in a staircase of idealized
// segments computed by CalcBudget().
TierGrowth float64
// The number of segments in any resulting MergeTask. e.g.,
// len(result.Tasks[ * ].Segments) == SegmentsPerMergeTask.
SegmentsPerMergeTask int
@ -115,6 +119,7 @@ func (o *MergePlanOptions) RaiseToFloorSegmentSize(s int64) int64 {
var DefaultMergePlanOptions = MergePlanOptions{
MaxSegmentsPerTier: 10,
MaxSegmentSize: 5000000,
TierGrowth: 10.0,
SegmentsPerMergeTask: 10,
FloorSegmentSize: 2000,
ReclaimDeletesWeight: 2.0,
@ -212,7 +217,8 @@ func plan(segmentsIn []Segment, o *MergePlanOptions) (*MergePlan, error) {
}
// Compute the number of segments that would be needed to cover the
// totalSize, by climbing up a logarithmic staircase of segment tiers.
// totalSize, by climbing up a logarithmically growing staircase of
// segment tiers.
func CalcBudget(totalSize int64, firstTierSize int64, o *MergePlanOptions) (
budgetNumSegments int) {
tierSize := firstTierSize
@ -225,9 +231,9 @@ func CalcBudget(totalSize int64, firstTierSize int64, o *MergePlanOptions) (
maxSegmentsPerTier = 1
}
segmentsPerMergeTask := int64(o.SegmentsPerMergeTask)
if segmentsPerMergeTask < 2 {
segmentsPerMergeTask = 2
tierGrowth := o.TierGrowth
if tierGrowth < 1.0 {
tierGrowth = 1.0
}
for totalSize > 0 {
@ -239,7 +245,7 @@ func CalcBudget(totalSize int64, firstTierSize int64, o *MergePlanOptions) (
budgetNumSegments += maxSegmentsPerTier
totalSize -= int64(maxSegmentsPerTier) * tierSize
tierSize *= segmentsPerMergeTask
tierSize = int64(float64(tierSize) * tierGrowth)
}
return budgetNumSegments

11
index/scorch/mergeplan/merge_plan_test.go
View File

@ -98,6 +98,7 @@ func TestSimplePlan(t *testing.T) {
&MergePlanOptions{
MaxSegmentsPerTier: 1,
MaxSegmentSize: 1000,
TierGrowth: 2.0,
SegmentsPerMergeTask: 2,
FloorSegmentSize: 1,
},
@ -156,11 +157,12 @@ func TestCalcBudget(t *testing.T) {
}{
{0, 0, MergePlanOptions{}, 0},
{1, 0, MergePlanOptions{}, 1},
{9, 0, MergePlanOptions{}, 4},
{9, 0, MergePlanOptions{}, 9},
{1, 1,
MergePlanOptions{
MaxSegmentsPerTier: 1,
MaxSegmentSize: 1000,
TierGrowth: 2.0,
SegmentsPerMergeTask: 2,
FloorSegmentSize: 1,
},
@ -170,6 +172,7 @@ func TestCalcBudget(t *testing.T) {
MergePlanOptions{
MaxSegmentsPerTier: 1,
MaxSegmentSize: 1000,
TierGrowth: 2.0,
SegmentsPerMergeTask: 2,
FloorSegmentSize: 1,
},
@ -179,6 +182,7 @@ func TestCalcBudget(t *testing.T) {
MergePlanOptions{
MaxSegmentsPerTier: 2,
MaxSegmentSize: 1000,
TierGrowth: 2.0,
SegmentsPerMergeTask: 2,
FloorSegmentSize: 1,
},
@ -201,6 +205,7 @@ func TestInsert1SameSizedSegmentBetweenMerges(t *testing.T) {
o := &MergePlanOptions{
MaxSegmentSize: 1000,
MaxSegmentsPerTier: 3,
TierGrowth: 3.0,
SegmentsPerMergeTask: 3,
}
@ -226,6 +231,7 @@ func TestInsertManySameSizedSegmentsBetweenMerges(t *testing.T) {
o := &MergePlanOptions{
MaxSegmentSize: 1000,
MaxSegmentsPerTier: 3,
TierGrowth: 3.0,
SegmentsPerMergeTask: 3,
}
@ -253,6 +259,7 @@ func TestInsertManySameSizedSegmentsWithDeletionsBetweenMerges(t *testing.T) {
o := &MergePlanOptions{
MaxSegmentSize: 1000,
MaxSegmentsPerTier: 3,
TierGrowth: 3.0,
SegmentsPerMergeTask: 3,
}
@ -290,6 +297,7 @@ func TestInsertManyDifferentSizedSegmentsBetweenMerges(t *testing.T) {
o := &MergePlanOptions{
MaxSegmentSize: 1000,
MaxSegmentsPerTier: 3,
TierGrowth: 3.0,
SegmentsPerMergeTask: 3,
}
@ -317,6 +325,7 @@ func TestManySameSizedSegmentsWithDeletesBetweenMerges(t *testing.T) {
o := &MergePlanOptions{
MaxSegmentSize: 1000,
MaxSegmentsPerTier: 3,
TierGrowth: 3.0,
SegmentsPerMergeTask: 3,
}