gibheer/bleve
0
0
Fork 0
Code

rewrite to keep using same buffer when possible

Browse Source
This commit is contained in:
Marty Schoch 2015-10-13 14:04:56 -07:00
parent 8de860bf12
commit 4c6bc23043
1 changed files with 22 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
index/upside_down/upside_down.go
View File

@ -120,7 +120,7 @@ func GetRowBuffer() []byte {
if rb, ok := rowBufferPool.Get().([]byte); ok {
return rb
} else {
return make([]byte, 4096)
return make([]byte, 4*1024)
}
}
@ -133,92 +133,60 @@ func (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRows []UpsideDow
// prepare batch
wb := writer.NewBatch()
// buffer to work with
rowBuf := GetRowBuffer()
// add
for _, row := range addRows {
keyBuf := GetRowBuffer()
valBuf := GetRowBuffer()
tfr, ok := row.(*TermFrequencyRow)
if ok {
// need to increment term dictinoary counter
if tfr.DictionaryRowKeySize() > len(keyBuf) {
keyBuf = make([]byte, 2*tfr.DictionaryRowKeySize())
}
dictKeySize, err := tfr.DictionaryRowKeyTo(keyBuf)
dictKeySize, err := tfr.DictionaryRowKeyTo(rowBuf)
if err != nil {
return err
}
wb.Merge(keyBuf[:dictKeySize], dictionaryTermIncr)
wb.Merge(rowBuf[:dictKeySize], dictionaryTermIncr)
}
if row.KeySize() > len(keyBuf) {
// grow buffer
keyBuf = make([]byte, 2*row.KeySize())
}
keySize, err := row.KeyTo(keyBuf)
keySize, err := row.KeyTo(rowBuf)
if err != nil {
return err
}
if row.ValueSize() > len(valBuf) {
// grow buffer
valBuf = make([]byte, 2*row.ValueSize())
}
valSize, err := row.ValueTo(valBuf)
wb.Set(keyBuf[:keySize], valBuf[:valSize])
PutRowBuffer(keyBuf)
PutRowBuffer(valBuf)
valSize, err := row.ValueTo(rowBuf[keySize:])
wb.Set(rowBuf[:keySize], rowBuf[keySize:keySize+valSize])
}
// update
for _, row := range updateRows {
keyBuf := GetRowBuffer()
valBuf := GetRowBuffer()
if row.KeySize() > len(keyBuf) {
// grow buffer
keyBuf = make([]byte, 2*row.KeySize())
}
keySize, err := row.KeyTo(keyBuf)
keySize, err := row.KeyTo(rowBuf)
if err != nil {
return err
}
if row.ValueSize() > len(valBuf) {
// grow buffer
valBuf = make([]byte, 2*row.ValueSize())
valSize, err := row.ValueTo(rowBuf[keySize:])
if err != nil {
return err
}
valSize, err := row.ValueTo(valBuf)
wb.Set(keyBuf[:keySize], valBuf[:valSize])
PutRowBuffer(keyBuf)
PutRowBuffer(valBuf)
wb.Set(rowBuf[:keySize], rowBuf[keySize:keySize+valSize])
}
// delete
for _, row := range deleteRows {
keyBuf := GetRowBuffer()
tfr, ok := row.(*TermFrequencyRow)
if ok {
// need to decrement counter
if tfr.DictionaryRowKeySize() > len(keyBuf) {
keyBuf = make([]byte, 2*tfr.DictionaryRowKeySize())
}
dictKeySize, err := tfr.DictionaryRowKeyTo(keyBuf)
dictKeySize, err := tfr.DictionaryRowKeyTo(rowBuf)
if err != nil {
return err
}
wb.Merge(keyBuf[:dictKeySize], dictionaryTermDecr)
wb.Merge(rowBuf[:dictKeySize], dictionaryTermDecr)
}
if row.KeySize() > len(keyBuf) {
// grow buffer
keyBuf = make([]byte, 2*row.KeySize())
}
keySize, err := row.KeyTo(keyBuf)
keySize, err := row.KeyTo(rowBuf)
if err != nil {
return err
}
wb.Delete(keyBuf[:keySize])
PutRowBuffer(keyBuf)
wb.Delete(rowBuf[:keySize])
}
PutRowBuffer(rowBuf)
// write out the batch
return writer.ExecuteBatch(wb)
}
@ -435,13 +403,10 @@ func (udc *UpsideDownCouch) mergeOldAndNew(backIndexRow *BackIndexRow, rows []in
existingStoredKeys[string(key)] = true
}
keyBuf := GetRowBuffer()
for _, row := range rows {
switch row := row.(type) {
case *TermFrequencyRow:
keyBuf := GetRowBuffer()
if row.KeySize() > len(keyBuf) {
keyBuf = make([]byte, 2*row.KeySize())
}
keySize, _ := row.KeyTo(keyBuf)
if _, ok := existingTermKeys[string(keyBuf[:keySize])]; ok {
updateRows = append(updateRows, row)
@ -449,12 +414,7 @@ func (udc *UpsideDownCouch) mergeOldAndNew(backIndexRow *BackIndexRow, rows []in
} else {
addRows = append(addRows, row)
}
PutRowBuffer(keyBuf)
case *StoredRow:
keyBuf := GetRowBuffer()
if row.KeySize() > len(keyBuf) {
keyBuf = make([]byte, 2*row.KeySize())
}
keySize, _ := row.KeyTo(keyBuf)
if _, ok := existingStoredKeys[string(keyBuf[:keySize])]; ok {
updateRows = append(updateRows, row)
@ -465,8 +425,8 @@ func (udc *UpsideDownCouch) mergeOldAndNew(backIndexRow *BackIndexRow, rows []in
default:
updateRows = append(updateRows, row)
}
}
PutRowBuffer(keyBuf)
// any of the existing rows that weren't updated need to be deleted
for existingTermKey := range existingTermKeys {