scorch zap refactored out mergeToWriter() func
This is a step towards supporting in-memory zap segment merging.
This commit is contained in:
parent
eb21bf8315
commit
6578655758
|
@ -52,41 +52,15 @@ func Merge(segments []*Segment, drops []*roaring.Bitmap, path string,
|
||||||
// wrap it for counting (tracking offsets)
|
// wrap it for counting (tracking offsets)
|
||||||
cr := NewCountHashWriter(br)
|
cr := NewCountHashWriter(br)
|
||||||
|
|
||||||
fieldsInv := mergeFields(segments)
|
newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, err :=
|
||||||
fieldsMap := mapFields(fieldsInv)
|
mergeToWriter(segments, drops, chunkFactor, cr)
|
||||||
|
|
||||||
var newDocNums [][]uint64
|
|
||||||
var storedIndexOffset uint64
|
|
||||||
fieldDvLocsOffset := uint64(fieldNotUninverted)
|
|
||||||
var dictLocs []uint64
|
|
||||||
|
|
||||||
newSegDocCount := computeNewDocCount(segments, drops)
|
|
||||||
if newSegDocCount > 0 {
|
|
||||||
storedIndexOffset, newDocNums, err = mergeStoredAndRemap(segments, drops,
|
|
||||||
fieldsMap, fieldsInv, newSegDocCount, cr)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
dictLocs, fieldDvLocsOffset, err = persistMergedRest(segments, drops, fieldsInv, fieldsMap,
|
|
||||||
newDocNums, newSegDocCount, chunkFactor, cr)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
dictLocs = make([]uint64, len(fieldsInv))
|
|
||||||
}
|
|
||||||
|
|
||||||
fieldsIndexOffset, err := persistFields(fieldsInv, cr, dictLocs)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cleanup()
|
cleanup()
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = persistFooter(newSegDocCount, storedIndexOffset,
|
err = persistFooter(numDocs, storedIndexOffset, fieldsIndexOffset,
|
||||||
fieldsIndexOffset, fieldDvLocsOffset, chunkFactor, cr.Sum32(), cr)
|
docValueOffset, chunkFactor, cr.Sum32(), cr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cleanup()
|
cleanup()
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -113,6 +87,43 @@ func Merge(segments []*Segment, drops []*roaring.Bitmap, path string,
|
||||||
return newDocNums, nil
|
return newDocNums, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func mergeToWriter(segments []*Segment, drops []*roaring.Bitmap,
|
||||||
|
chunkFactor uint32, cr *CountHashWriter) (
|
||||||
|
newDocNums [][]uint64,
|
||||||
|
numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset uint64,
|
||||||
|
err error) {
|
||||||
|
docValueOffset = uint64(fieldNotUninverted)
|
||||||
|
|
||||||
|
var dictLocs []uint64
|
||||||
|
|
||||||
|
fieldsInv := mergeFields(segments)
|
||||||
|
fieldsMap := mapFields(fieldsInv)
|
||||||
|
|
||||||
|
numDocs = computeNewDocCount(segments, drops)
|
||||||
|
if numDocs > 0 {
|
||||||
|
storedIndexOffset, newDocNums, err = mergeStoredAndRemap(segments, drops,
|
||||||
|
fieldsMap, fieldsInv, numDocs, cr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, 0, 0, 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
dictLocs, docValueOffset, err = persistMergedRest(segments, drops, fieldsInv, fieldsMap,
|
||||||
|
newDocNums, numDocs, chunkFactor, cr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, 0, 0, 0, err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
dictLocs = make([]uint64, len(fieldsInv))
|
||||||
|
}
|
||||||
|
|
||||||
|
fieldsIndexOffset, err = persistFields(fieldsInv, cr, dictLocs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, 0, 0, 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, nil
|
||||||
|
}
|
||||||
|
|
||||||
// mapFields takes the fieldsInv list and builds the map
|
// mapFields takes the fieldsInv list and builds the map
|
||||||
func mapFields(fields []string) map[string]uint16 {
|
func mapFields(fields []string) map[string]uint16 {
|
||||||
rv := make(map[string]uint16, len(fields))
|
rv := make(map[string]uint16, len(fields))
|
||||||
|
|
Loading…
Reference in New Issue