0
0

firestorm.Batch() collects [][]IndexRows instead of []IndexRow

Rather than append() all received rows into a flat []IndexRow during
the result gathering loop, this change instead collects the analysis
result rows into a [][]IndexRow, which avoids extra copying.

As part of this, firestorm batchRows() now takes the [][]IndexRow as
its input.
This commit is contained in:
Steve Yen 2016-01-01 22:25:34 -08:00
parent 1c5b84911d
commit fb8c9a7475

View File

@ -167,7 +167,7 @@ func (f *Firestorm) Update(doc *document.Document) (err error) {
}() }()
var dictionaryDeltas map[string]int64 var dictionaryDeltas map[string]int64
dictionaryDeltas, err = f.batchRows(kvwriter, result.Rows, nil) dictionaryDeltas, err = f.batchRows(kvwriter, [][]index.IndexRow{result.Rows}, nil)
if err != nil { if err != nil {
_ = kvwriter.Close() _ = kvwriter.Close()
atomic.AddUint64(&f.stats.errors, 1) atomic.AddUint64(&f.stats.errors, 1)
@ -190,7 +190,7 @@ func (f *Firestorm) Delete(id string) error {
return nil return nil
} }
func (f *Firestorm) batchRows(writer store.KVWriter, rows []index.IndexRow, deleteKeys [][]byte) (map[string]int64, error) { func (f *Firestorm) batchRows(writer store.KVWriter, rowsOfRows [][]index.IndexRow, deleteKeys [][]byte) (map[string]int64, error) {
// prepare batch // prepare batch
wb := writer.NewBatch() wb := writer.NewBatch()
@ -206,6 +206,8 @@ func (f *Firestorm) batchRows(writer store.KVWriter, rows []index.IndexRow, dele
} }
dictionaryDeltas := make(map[string]int64) dictionaryDeltas := make(map[string]int64)
for _, rows := range rowsOfRows {
for _, row := range rows { for _, row := range rows {
tfr, ok := row.(*TermFreqRow) tfr, ok := row.(*TermFreqRow)
if ok { if ok {
@ -234,6 +236,7 @@ func (f *Firestorm) batchRows(writer store.KVWriter, rows []index.IndexRow, dele
wb.Set(kbuf[0:klen], vbuf[0:vlen]) wb.Set(kbuf[0:klen], vbuf[0:vlen])
} }
}
for _, dk := range deleteKeys { for _, dk := range deleteKeys {
wb.Delete(dk) wb.Delete(dk)
@ -291,12 +294,14 @@ func (f *Firestorm) Batch(batch *index.Batch) (err error) {
} }
}() }()
allRows := make([]index.IndexRow, 0, 1000) // extra 1 capacity for internal updates.
collectRows := make([][]index.IndexRow, 0, docsUpdated+1)
// wait for the result // wait for the result
var itemsDeQueued uint64 var itemsDeQueued uint64
for itemsDeQueued < docsUpdated { for itemsDeQueued < docsUpdated {
result := <-resultChan result := <-resultChan
allRows = append(allRows, result.Rows...) collectRows = append(collectRows, result.Rows)
itemsDeQueued++ itemsDeQueued++
} }
close(resultChan) close(resultChan)
@ -309,8 +314,10 @@ func (f *Firestorm) Batch(batch *index.Batch) (err error) {
atomic.AddUint64(&f.stats.analysisTime, uint64(time.Since(analysisStart))) atomic.AddUint64(&f.stats.analysisTime, uint64(time.Since(analysisStart)))
deleteKeys := make([][]byte, 0) var deleteKeys [][]byte
if len(batch.InternalOps) > 0 {
// add the internal ops // add the internal ops
updateInternalRows := make([]index.IndexRow, 0, len(batch.InternalOps))
for internalKey, internalValue := range batch.InternalOps { for internalKey, internalValue := range batch.InternalOps {
if internalValue == nil { if internalValue == nil {
// delete // delete
@ -318,9 +325,11 @@ func (f *Firestorm) Batch(batch *index.Batch) (err error) {
deleteKeys = append(deleteKeys, deleteInternalRow.Key()) deleteKeys = append(deleteKeys, deleteInternalRow.Key())
} else { } else {
updateInternalRow := NewInternalRow([]byte(internalKey), internalValue) updateInternalRow := NewInternalRow([]byte(internalKey), internalValue)
allRows = append(allRows, updateInternalRow) updateInternalRows = append(updateInternalRows, updateInternalRow)
} }
} }
collectRows = append(collectRows, updateInternalRows)
}
inflightItems := make([]*InFlightItem, 0, len(batch.IndexOps)) inflightItems := make([]*InFlightItem, 0, len(batch.IndexOps))
for docID, doc := range batch.IndexOps { for docID, doc := range batch.IndexOps {
@ -342,7 +351,7 @@ func (f *Firestorm) Batch(batch *index.Batch) (err error) {
} }
var dictionaryDeltas map[string]int64 var dictionaryDeltas map[string]int64
dictionaryDeltas, err = f.batchRows(kvwriter, allRows, deleteKeys) dictionaryDeltas, err = f.batchRows(kvwriter, collectRows, deleteKeys)
if err != nil { if err != nil {
_ = kvwriter.Close() _ = kvwriter.Close()
atomic.AddUint64(&f.stats.errors, 1) atomic.AddUint64(&f.stats.errors, 1)