From 289dc398bd88ea1bd9c952772e2323b64c0ec2a5 Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Wed, 13 Dec 2017 14:26:11 -0500 Subject: [PATCH] more refacotring of build/merge --- index/scorch/segment/zap/build.go | 39 ------------------------------- index/scorch/segment/zap/merge.go | 3 +++ index/scorch/segment/zap/write.go | 39 +++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 39 deletions(-) diff --git a/index/scorch/segment/zap/build.go b/index/scorch/segment/zap/build.go index 2cad61a5..56061a2e 100644 --- a/index/scorch/segment/zap/build.go +++ b/index/scorch/segment/zap/build.go @@ -572,42 +572,3 @@ func persistDictionary(memSegment *mem.Segment, w *CountHashWriter, postingsLocs return rv, nil } - -// FooterSize is the size of the footer record in bytes -// crc + ver + chunk + field offset + stored offset + num docs -const FooterSize = 4 + 4 + 4 + 8 + 8 + 8 - -func persistFooter(numDocs, storedIndexOffset, fieldIndexOffset uint64, - chunkFactor uint32, w *CountHashWriter) error { - // write out the number of docs - err := binary.Write(w, binary.BigEndian, numDocs) - if err != nil { - return err - } - // write out the stored field index location: - err = binary.Write(w, binary.BigEndian, storedIndexOffset) - if err != nil { - return err - } - // write out the field index location - err = binary.Write(w, binary.BigEndian, fieldIndexOffset) - if err != nil { - return err - } - // write out 32-bit chunk factor - err = binary.Write(w, binary.BigEndian, chunkFactor) - if err != nil { - return err - } - // write out 32-bit version - err = binary.Write(w, binary.BigEndian, version) - if err != nil { - return err - } - // write out CRC-32 of everything upto but not including this CRC - err = binary.Write(w, binary.BigEndian, w.Sum32()) - if err != nil { - return err - } - return nil -} diff --git a/index/scorch/segment/zap/merge.go b/index/scorch/segment/zap/merge.go index 972b1d16..670c8d79 100644 --- a/index/scorch/segment/zap/merge.go +++ b/index/scorch/segment/zap/merge.go @@ -84,6 +84,7 @@ func Merge(segments []*Segment, drops []*roaring.Bitmap, path string, return newDocNums, nil } +// mapFields takes the fieldsInv list and builds the map func mapFields(fields []string) map[string]uint16 { rv := make(map[string]uint16) for i, fieldName := range fields { @@ -92,6 +93,8 @@ func mapFields(fields []string) map[string]uint16 { return rv } +// computeNewDocCount determines how many documents will be in the newly +// merged segment when obsoleted docs are dropped func computeNewDocCount(segments []*Segment, drops []*roaring.Bitmap) uint64 { var newSegDocCount uint64 for segI, segment := range segments { diff --git a/index/scorch/segment/zap/write.go b/index/scorch/segment/zap/write.go index 9772b3a6..bd63e147 100644 --- a/index/scorch/segment/zap/write.go +++ b/index/scorch/segment/zap/write.go @@ -93,3 +93,42 @@ func persistFields(fieldsInv []string, w *CountHashWriter, dictLocs []uint64) (u return rv, nil } + +// FooterSize is the size of the footer record in bytes +// crc + ver + chunk + field offset + stored offset + num docs +const FooterSize = 4 + 4 + 4 + 8 + 8 + 8 + +func persistFooter(numDocs, storedIndexOffset, fieldIndexOffset uint64, + chunkFactor uint32, w *CountHashWriter) error { + // write out the number of docs + err := binary.Write(w, binary.BigEndian, numDocs) + if err != nil { + return err + } + // write out the stored field index location: + err = binary.Write(w, binary.BigEndian, storedIndexOffset) + if err != nil { + return err + } + // write out the field index location + err = binary.Write(w, binary.BigEndian, fieldIndexOffset) + if err != nil { + return err + } + // write out 32-bit chunk factor + err = binary.Write(w, binary.BigEndian, chunkFactor) + if err != nil { + return err + } + // write out 32-bit version + err = binary.Write(w, binary.BigEndian, version) + if err != nil { + return err + } + // write out CRC-32 of everything upto but not including this CRC + err = binary.Write(w, binary.BigEndian, w.Sum32()) + if err != nil { + return err + } + return nil +}