more refacotring of build/merge
This commit is contained in:
parent
1cd3fd7fbe
commit
289dc398bd
@ -572,42 +572,3 @@ func persistDictionary(memSegment *mem.Segment, w *CountHashWriter, postingsLocs
|
|||||||
|
|
||||||
return rv, nil
|
return rv, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// FooterSize is the size of the footer record in bytes
|
|
||||||
// crc + ver + chunk + field offset + stored offset + num docs
|
|
||||||
const FooterSize = 4 + 4 + 4 + 8 + 8 + 8
|
|
||||||
|
|
||||||
func persistFooter(numDocs, storedIndexOffset, fieldIndexOffset uint64,
|
|
||||||
chunkFactor uint32, w *CountHashWriter) error {
|
|
||||||
// write out the number of docs
|
|
||||||
err := binary.Write(w, binary.BigEndian, numDocs)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// write out the stored field index location:
|
|
||||||
err = binary.Write(w, binary.BigEndian, storedIndexOffset)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// write out the field index location
|
|
||||||
err = binary.Write(w, binary.BigEndian, fieldIndexOffset)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// write out 32-bit chunk factor
|
|
||||||
err = binary.Write(w, binary.BigEndian, chunkFactor)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// write out 32-bit version
|
|
||||||
err = binary.Write(w, binary.BigEndian, version)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// write out CRC-32 of everything upto but not including this CRC
|
|
||||||
err = binary.Write(w, binary.BigEndian, w.Sum32())
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
@ -84,6 +84,7 @@ func Merge(segments []*Segment, drops []*roaring.Bitmap, path string,
|
|||||||
return newDocNums, nil
|
return newDocNums, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// mapFields takes the fieldsInv list and builds the map
|
||||||
func mapFields(fields []string) map[string]uint16 {
|
func mapFields(fields []string) map[string]uint16 {
|
||||||
rv := make(map[string]uint16)
|
rv := make(map[string]uint16)
|
||||||
for i, fieldName := range fields {
|
for i, fieldName := range fields {
|
||||||
@ -92,6 +93,8 @@ func mapFields(fields []string) map[string]uint16 {
|
|||||||
return rv
|
return rv
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// computeNewDocCount determines how many documents will be in the newly
|
||||||
|
// merged segment when obsoleted docs are dropped
|
||||||
func computeNewDocCount(segments []*Segment, drops []*roaring.Bitmap) uint64 {
|
func computeNewDocCount(segments []*Segment, drops []*roaring.Bitmap) uint64 {
|
||||||
var newSegDocCount uint64
|
var newSegDocCount uint64
|
||||||
for segI, segment := range segments {
|
for segI, segment := range segments {
|
||||||
|
@ -93,3 +93,42 @@ func persistFields(fieldsInv []string, w *CountHashWriter, dictLocs []uint64) (u
|
|||||||
|
|
||||||
return rv, nil
|
return rv, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FooterSize is the size of the footer record in bytes
|
||||||
|
// crc + ver + chunk + field offset + stored offset + num docs
|
||||||
|
const FooterSize = 4 + 4 + 4 + 8 + 8 + 8
|
||||||
|
|
||||||
|
func persistFooter(numDocs, storedIndexOffset, fieldIndexOffset uint64,
|
||||||
|
chunkFactor uint32, w *CountHashWriter) error {
|
||||||
|
// write out the number of docs
|
||||||
|
err := binary.Write(w, binary.BigEndian, numDocs)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// write out the stored field index location:
|
||||||
|
err = binary.Write(w, binary.BigEndian, storedIndexOffset)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// write out the field index location
|
||||||
|
err = binary.Write(w, binary.BigEndian, fieldIndexOffset)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// write out 32-bit chunk factor
|
||||||
|
err = binary.Write(w, binary.BigEndian, chunkFactor)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// write out 32-bit version
|
||||||
|
err = binary.Write(w, binary.BigEndian, version)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// write out CRC-32 of everything upto but not including this CRC
|
||||||
|
err = binary.Write(w, binary.BigEndian, w.Sum32())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user