upside_down merge dictionary deltas before using batch.Merge()
This change performs more dictionary delta incr/decr math in batchRows() instead of in the KVStore ExecuteBatch() machinery.
This commit is contained in:
parent
94273d5fa9
commit
7ce7d98cba
|
@ -10,6 +10,7 @@
|
||||||
package upside_down
|
package upside_down
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/binary"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
"math"
|
||||||
|
@ -152,6 +153,8 @@ func (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRows []UpsideDow
|
||||||
// buffer to work with
|
// buffer to work with
|
||||||
rowBuf := GetRowBuffer()
|
rowBuf := GetRowBuffer()
|
||||||
|
|
||||||
|
dictionaryDeltas := make(map[string]int64)
|
||||||
|
|
||||||
// add
|
// add
|
||||||
for _, row := range addRows {
|
for _, row := range addRows {
|
||||||
tfr, ok := row.(*TermFrequencyRow)
|
tfr, ok := row.(*TermFrequencyRow)
|
||||||
|
@ -163,7 +166,7 @@ func (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRows []UpsideDow
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
wb.Merge(rowBuf[:dictKeySize], dictionaryTermIncr)
|
dictionaryDeltas[string(rowBuf[:dictKeySize])] += 1
|
||||||
}
|
}
|
||||||
if row.KeySize()+row.ValueSize() > len(rowBuf) {
|
if row.KeySize()+row.ValueSize() > len(rowBuf) {
|
||||||
rowBuf = make([]byte, row.KeySize()+row.ValueSize())
|
rowBuf = make([]byte, row.KeySize()+row.ValueSize())
|
||||||
|
@ -204,7 +207,7 @@ func (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRows []UpsideDow
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
wb.Merge(rowBuf[:dictKeySize], dictionaryTermDecr)
|
dictionaryDeltas[string(rowBuf[:dictKeySize])] -= 1
|
||||||
}
|
}
|
||||||
if row.KeySize()+row.ValueSize() > len(rowBuf) {
|
if row.KeySize()+row.ValueSize() > len(rowBuf) {
|
||||||
rowBuf = make([]byte, row.KeySize()+row.ValueSize())
|
rowBuf = make([]byte, row.KeySize()+row.ValueSize())
|
||||||
|
@ -216,6 +219,15 @@ func (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRows []UpsideDow
|
||||||
wb.Delete(rowBuf[:keySize])
|
wb.Delete(rowBuf[:keySize])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if 8 > len(rowBuf) {
|
||||||
|
rowBuf = make([]byte, 8)
|
||||||
|
}
|
||||||
|
|
||||||
|
for dictRowKey, delta := range dictionaryDeltas {
|
||||||
|
binary.LittleEndian.PutUint64(rowBuf, uint64(delta))
|
||||||
|
wb.Merge([]byte(dictRowKey), rowBuf[0:8])
|
||||||
|
}
|
||||||
|
|
||||||
PutRowBuffer(rowBuf)
|
PutRowBuffer(rowBuf)
|
||||||
|
|
||||||
// write out the batch
|
// write out the batch
|
||||||
|
|
Loading…
Reference in New Issue
Block a user