0
0

use KeyTo/ValueTo in firestorm.batchRows

After this change, with null kvstore micro-benchmark...

  GOMAXPROCS=8 ./bleve-blast -source=../../tmp/enwiki.txt \
    -count=100000 -numAnalyzers=8 -numIndexers=8 \
    -config=../../configs/null-firestorm.json -batch=100

Then TermFreqRow key and value methods dissapear as large boxes from
the cpu profile graphs.
This commit is contained in:
Steve Yen 2015-12-30 11:18:16 -08:00
parent fd287bdfa4
commit ee5ccda112

View File

@ -195,16 +195,46 @@ func (f *Firestorm) batchRows(writer store.KVWriter, rows []index.IndexRow, dele
// prepare batch // prepare batch
wb := writer.NewBatch() wb := writer.NewBatch()
var kbuf []byte
var vbuf []byte
prepareBuf := func(buf []byte, sizeNeeded int) []byte {
if cap(buf) < sizeNeeded {
return make([]byte, sizeNeeded, sizeNeeded+128)
}
return buf[0:sizeNeeded]
}
dictionaryDeltas := make(map[string]int64) dictionaryDeltas := make(map[string]int64)
for _, row := range rows { for _, row := range rows {
tfr, ok := row.(*TermFreqRow) tfr, ok := row.(*TermFreqRow)
if ok { if ok {
if tfr.Field() != 0 { if tfr.Field() != 0 {
drk := tfr.DictionaryRowKey() kbuf = prepareBuf(kbuf, tfr.DictionaryRowKeySize())
dictionaryDeltas[string(drk)] += 1 klen, err := tfr.DictionaryRowKeyTo(kbuf)
if err != nil {
return nil, err
}
dictionaryDeltas[string(kbuf[0:klen])] += 1
} }
kbuf = prepareBuf(kbuf, tfr.KeySize())
klen, err := tfr.KeyTo(kbuf)
if err != nil {
return nil, err
}
vbuf = prepareBuf(vbuf, tfr.ValueSize())
vlen, err := tfr.ValueTo(vbuf)
if err != nil {
return nil, err
}
wb.Set(kbuf[0:klen], vbuf[0:vlen])
} else {
wb.Set(row.Key(), row.Value())
} }
wb.Set(row.Key(), row.Value())
} }
for _, dk := range deleteKeys { for _, dk := range deleteKeys {