2017-12-28 07:35:33 +01:00
|
|
|
// Copyright (c) 2017 Couchbase, Inc.
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package zap
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
|
|
|
"encoding/binary"
|
|
|
|
"io"
|
2018-03-02 02:12:16 +01:00
|
|
|
"reflect"
|
2017-12-28 07:35:33 +01:00
|
|
|
|
|
|
|
"github.com/golang/snappy"
|
|
|
|
)
|
|
|
|
|
2018-03-02 02:12:16 +01:00
|
|
|
var reflectStaticSizeMetaData int
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
var md MetaData
|
|
|
|
reflectStaticSizeMetaData = int(reflect.TypeOf(md).Size())
|
|
|
|
}
|
|
|
|
|
2017-12-28 07:35:33 +01:00
|
|
|
var termSeparator byte = 0xff
|
|
|
|
var termSeparatorSplitSlice = []byte{termSeparator}
|
|
|
|
|
|
|
|
type chunkedContentCoder struct {
|
|
|
|
final []byte
|
|
|
|
chunkSize uint64
|
|
|
|
currChunk uint64
|
|
|
|
chunkLens []uint64
|
|
|
|
chunkMetaBuf bytes.Buffer
|
|
|
|
chunkBuf bytes.Buffer
|
|
|
|
|
2017-12-29 17:09:29 +01:00
|
|
|
chunkMeta []MetaData
|
2017-12-28 07:35:33 +01:00
|
|
|
}
|
|
|
|
|
2017-12-29 17:09:29 +01:00
|
|
|
// MetaData represents the data information inside a
|
2017-12-28 07:35:33 +01:00
|
|
|
// chunk.
|
2017-12-29 17:09:29 +01:00
|
|
|
type MetaData struct {
|
2018-03-12 11:06:46 +01:00
|
|
|
DocNum uint64 // docNum of the data inside the chunk
|
|
|
|
DocDvOffset uint64 // offset of data inside the chunk for the given docid
|
2017-12-28 07:35:33 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// newChunkedContentCoder returns a new chunk content coder which
|
|
|
|
// packs data into chunks based on the provided chunkSize
|
2017-12-28 08:35:25 +01:00
|
|
|
func newChunkedContentCoder(chunkSize uint64,
|
|
|
|
maxDocNum uint64) *chunkedContentCoder {
|
2017-12-28 07:35:33 +01:00
|
|
|
total := maxDocNum/chunkSize + 1
|
|
|
|
rv := &chunkedContentCoder{
|
|
|
|
chunkSize: chunkSize,
|
|
|
|
chunkLens: make([]uint64, total),
|
2018-02-03 19:51:24 +01:00
|
|
|
chunkMeta: make([]MetaData, 0, total),
|
2017-12-28 07:35:33 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
return rv
|
|
|
|
}
|
|
|
|
|
|
|
|
// Reset lets you reuse this chunked content coder. Buffers are reset
|
|
|
|
// and re used. You cannot change the chunk size.
|
|
|
|
func (c *chunkedContentCoder) Reset() {
|
|
|
|
c.currChunk = 0
|
|
|
|
c.final = c.final[:0]
|
|
|
|
c.chunkBuf.Reset()
|
|
|
|
c.chunkMetaBuf.Reset()
|
|
|
|
for i := range c.chunkLens {
|
|
|
|
c.chunkLens[i] = 0
|
|
|
|
}
|
2018-02-03 19:51:24 +01:00
|
|
|
c.chunkMeta = c.chunkMeta[:0]
|
2017-12-28 07:35:33 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// Close indicates you are done calling Add() this allows
|
|
|
|
// the final chunk to be encoded.
|
2017-12-29 17:09:29 +01:00
|
|
|
func (c *chunkedContentCoder) Close() error {
|
|
|
|
return c.flushContents()
|
2017-12-28 07:35:33 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
func (c *chunkedContentCoder) flushContents() error {
|
|
|
|
// flush the contents, with meta information at first
|
|
|
|
buf := make([]byte, binary.MaxVarintLen64)
|
|
|
|
n := binary.PutUvarint(buf, uint64(len(c.chunkMeta)))
|
|
|
|
_, err := c.chunkMetaBuf.Write(buf[:n])
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// write out the metaData slice
|
|
|
|
for _, meta := range c.chunkMeta {
|
2018-03-12 11:06:46 +01:00
|
|
|
_, err := writeUvarints(&c.chunkMetaBuf, meta.DocNum, meta.DocDvOffset)
|
2017-12-28 07:35:33 +01:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2017-12-29 17:09:29 +01:00
|
|
|
}
|
2017-12-28 07:35:33 +01:00
|
|
|
|
|
|
|
// write the metadata to final data
|
|
|
|
metaData := c.chunkMetaBuf.Bytes()
|
|
|
|
c.final = append(c.final, c.chunkMetaBuf.Bytes()...)
|
|
|
|
// write the compressed data to the final data
|
|
|
|
compressedData := snappy.Encode(nil, c.chunkBuf.Bytes())
|
|
|
|
c.final = append(c.final, compressedData...)
|
2017-12-28 08:35:25 +01:00
|
|
|
|
2017-12-28 07:35:33 +01:00
|
|
|
c.chunkLens[c.currChunk] = uint64(len(compressedData) + len(metaData))
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Add encodes the provided byte slice into the correct chunk for the provided
|
|
|
|
// doc num. You MUST call Add() with increasing docNums.
|
|
|
|
func (c *chunkedContentCoder) Add(docNum uint64, vals []byte) error {
|
|
|
|
chunk := docNum / c.chunkSize
|
|
|
|
if chunk != c.currChunk {
|
|
|
|
// flush out the previous chunk details
|
|
|
|
err := c.flushContents()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
// clearing the chunk specific meta for next chunk
|
|
|
|
c.chunkBuf.Reset()
|
|
|
|
c.chunkMetaBuf.Reset()
|
2018-02-03 19:51:24 +01:00
|
|
|
c.chunkMeta = c.chunkMeta[:0]
|
2017-12-28 07:35:33 +01:00
|
|
|
c.currChunk = chunk
|
|
|
|
}
|
|
|
|
|
2018-03-14 06:12:19 +01:00
|
|
|
// get the starting offset for this doc
|
|
|
|
dvOffset := c.chunkBuf.Len()
|
2017-12-28 07:35:33 +01:00
|
|
|
dvSize, err := c.chunkBuf.Write(vals)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2017-12-29 17:09:29 +01:00
|
|
|
c.chunkMeta = append(c.chunkMeta, MetaData{
|
2018-03-12 11:06:46 +01:00
|
|
|
DocNum: docNum,
|
2018-03-14 06:12:19 +01:00
|
|
|
DocDvOffset: uint64(dvOffset + dvSize),
|
2017-12-28 07:35:33 +01:00
|
|
|
})
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Write commits all the encoded chunked contents to the provided writer.
|
|
|
|
func (c *chunkedContentCoder) Write(w io.Writer) (int, error) {
|
|
|
|
var tw int
|
|
|
|
buf := make([]byte, binary.MaxVarintLen64)
|
|
|
|
// write out the number of chunks
|
|
|
|
n := binary.PutUvarint(buf, uint64(len(c.chunkLens)))
|
|
|
|
nw, err := w.Write(buf[:n])
|
|
|
|
tw += nw
|
|
|
|
if err != nil {
|
|
|
|
return tw, err
|
|
|
|
}
|
2018-03-09 11:31:37 +01:00
|
|
|
|
2018-03-13 07:43:48 +01:00
|
|
|
chunkOffsets := modifyLengthsToEndOffsets(c.chunkLens)
|
|
|
|
// write out the chunk offsets
|
|
|
|
for _, chunkOffset := range chunkOffsets {
|
|
|
|
n := binary.PutUvarint(buf, chunkOffset)
|
2017-12-28 07:35:33 +01:00
|
|
|
nw, err = w.Write(buf[:n])
|
|
|
|
tw += nw
|
|
|
|
if err != nil {
|
|
|
|
return tw, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// write out the data
|
|
|
|
nw, err = w.Write(c.final)
|
|
|
|
tw += nw
|
|
|
|
if err != nil {
|
|
|
|
return tw, err
|
|
|
|
}
|
|
|
|
return tw, nil
|
|
|
|
}
|
2018-03-13 09:36:48 +01:00
|
|
|
|
|
|
|
// ReadDocValueBoundary elicits the start, end offsets from a
|
2018-03-14 06:12:19 +01:00
|
|
|
// metaData header slice
|
2018-03-13 09:36:48 +01:00
|
|
|
func ReadDocValueBoundary(chunk int, metaHeaders []MetaData) (uint64, uint64) {
|
2018-03-14 06:12:19 +01:00
|
|
|
var start uint64
|
2018-03-13 09:36:48 +01:00
|
|
|
if chunk > 0 {
|
|
|
|
start = metaHeaders[chunk-1].DocDvOffset
|
|
|
|
}
|
2018-03-14 06:12:19 +01:00
|
|
|
return start, metaHeaders[chunk].DocDvOffset
|
2018-03-13 09:36:48 +01:00
|
|
|
}
|