firestorm.Batch() collects [][]IndexRows instead of []IndexRow
Rather than append() all received rows into a flat []IndexRow during the result gathering loop, this change instead collects the analysis result rows into a [][]IndexRow, which avoids extra copying. As part of this, firestorm batchRows() now takes the [][]IndexRow as its input.
This commit is contained in:
parent
1c5b84911d
commit
fb8c9a7475
@ -167,7 +167,7 @@ func (f *Firestorm) Update(doc *document.Document) (err error) {
|
|||||||
}()
|
}()
|
||||||
|
|
||||||
var dictionaryDeltas map[string]int64
|
var dictionaryDeltas map[string]int64
|
||||||
dictionaryDeltas, err = f.batchRows(kvwriter, result.Rows, nil)
|
dictionaryDeltas, err = f.batchRows(kvwriter, [][]index.IndexRow{result.Rows}, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = kvwriter.Close()
|
_ = kvwriter.Close()
|
||||||
atomic.AddUint64(&f.stats.errors, 1)
|
atomic.AddUint64(&f.stats.errors, 1)
|
||||||
@ -190,7 +190,7 @@ func (f *Firestorm) Delete(id string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Firestorm) batchRows(writer store.KVWriter, rows []index.IndexRow, deleteKeys [][]byte) (map[string]int64, error) {
|
func (f *Firestorm) batchRows(writer store.KVWriter, rowsOfRows [][]index.IndexRow, deleteKeys [][]byte) (map[string]int64, error) {
|
||||||
|
|
||||||
// prepare batch
|
// prepare batch
|
||||||
wb := writer.NewBatch()
|
wb := writer.NewBatch()
|
||||||
@ -206,6 +206,8 @@ func (f *Firestorm) batchRows(writer store.KVWriter, rows []index.IndexRow, dele
|
|||||||
}
|
}
|
||||||
|
|
||||||
dictionaryDeltas := make(map[string]int64)
|
dictionaryDeltas := make(map[string]int64)
|
||||||
|
|
||||||
|
for _, rows := range rowsOfRows {
|
||||||
for _, row := range rows {
|
for _, row := range rows {
|
||||||
tfr, ok := row.(*TermFreqRow)
|
tfr, ok := row.(*TermFreqRow)
|
||||||
if ok {
|
if ok {
|
||||||
@ -234,6 +236,7 @@ func (f *Firestorm) batchRows(writer store.KVWriter, rows []index.IndexRow, dele
|
|||||||
|
|
||||||
wb.Set(kbuf[0:klen], vbuf[0:vlen])
|
wb.Set(kbuf[0:klen], vbuf[0:vlen])
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for _, dk := range deleteKeys {
|
for _, dk := range deleteKeys {
|
||||||
wb.Delete(dk)
|
wb.Delete(dk)
|
||||||
@ -291,12 +294,14 @@ func (f *Firestorm) Batch(batch *index.Batch) (err error) {
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
allRows := make([]index.IndexRow, 0, 1000)
|
// extra 1 capacity for internal updates.
|
||||||
|
collectRows := make([][]index.IndexRow, 0, docsUpdated+1)
|
||||||
|
|
||||||
// wait for the result
|
// wait for the result
|
||||||
var itemsDeQueued uint64
|
var itemsDeQueued uint64
|
||||||
for itemsDeQueued < docsUpdated {
|
for itemsDeQueued < docsUpdated {
|
||||||
result := <-resultChan
|
result := <-resultChan
|
||||||
allRows = append(allRows, result.Rows...)
|
collectRows = append(collectRows, result.Rows)
|
||||||
itemsDeQueued++
|
itemsDeQueued++
|
||||||
}
|
}
|
||||||
close(resultChan)
|
close(resultChan)
|
||||||
@ -309,8 +314,10 @@ func (f *Firestorm) Batch(batch *index.Batch) (err error) {
|
|||||||
|
|
||||||
atomic.AddUint64(&f.stats.analysisTime, uint64(time.Since(analysisStart)))
|
atomic.AddUint64(&f.stats.analysisTime, uint64(time.Since(analysisStart)))
|
||||||
|
|
||||||
deleteKeys := make([][]byte, 0)
|
var deleteKeys [][]byte
|
||||||
|
if len(batch.InternalOps) > 0 {
|
||||||
// add the internal ops
|
// add the internal ops
|
||||||
|
updateInternalRows := make([]index.IndexRow, 0, len(batch.InternalOps))
|
||||||
for internalKey, internalValue := range batch.InternalOps {
|
for internalKey, internalValue := range batch.InternalOps {
|
||||||
if internalValue == nil {
|
if internalValue == nil {
|
||||||
// delete
|
// delete
|
||||||
@ -318,9 +325,11 @@ func (f *Firestorm) Batch(batch *index.Batch) (err error) {
|
|||||||
deleteKeys = append(deleteKeys, deleteInternalRow.Key())
|
deleteKeys = append(deleteKeys, deleteInternalRow.Key())
|
||||||
} else {
|
} else {
|
||||||
updateInternalRow := NewInternalRow([]byte(internalKey), internalValue)
|
updateInternalRow := NewInternalRow([]byte(internalKey), internalValue)
|
||||||
allRows = append(allRows, updateInternalRow)
|
updateInternalRows = append(updateInternalRows, updateInternalRow)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
collectRows = append(collectRows, updateInternalRows)
|
||||||
|
}
|
||||||
|
|
||||||
inflightItems := make([]*InFlightItem, 0, len(batch.IndexOps))
|
inflightItems := make([]*InFlightItem, 0, len(batch.IndexOps))
|
||||||
for docID, doc := range batch.IndexOps {
|
for docID, doc := range batch.IndexOps {
|
||||||
@ -342,7 +351,7 @@ func (f *Firestorm) Batch(batch *index.Batch) (err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var dictionaryDeltas map[string]int64
|
var dictionaryDeltas map[string]int64
|
||||||
dictionaryDeltas, err = f.batchRows(kvwriter, allRows, deleteKeys)
|
dictionaryDeltas, err = f.batchRows(kvwriter, collectRows, deleteKeys)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = kvwriter.Close()
|
_ = kvwriter.Close()
|
||||||
atomic.AddUint64(&f.stats.errors, 1)
|
atomic.AddUint64(&f.stats.errors, 1)
|
||||||
|
Loading…
Reference in New Issue
Block a user