0
0
bleve/search/collector/list.go

88 lines
2.1 KiB
Go
Raw Normal View History

// Copyright (c) 2014 Couchbase, Inc.
2016-10-02 16:13:14 +02:00
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package collector
import (
"container/list"
"github.com/blevesearch/bleve/search"
)
type collectStoreList struct {
results *list.List
compare collectorCompare
}
func newStoreList(cap int, compare collectorCompare) *collectStoreList {
rv := &collectStoreList{
results: list.New(),
compare: compare,
}
return rv
}
topn collector switch approach based on size+skip we now use the slice store when size+skip <= 10 and use the heap store when size+skip > 10 here are the new perf numbers: go test -run=xxx -bench=. -benchmem BenchmarkTop10of0Scores-4 1000000 1150 ns/op 2304 B/op 15 allocs/op BenchmarkTop10of3Scores-4 1000000 1417 ns/op 2304 B/op 18 allocs/op BenchmarkTop10of10Scores-4 1000000 2133 ns/op 2312 B/op 25 allocs/op BenchmarkTop10of25Scores-4 500000 3410 ns/op 2464 B/op 26 allocs/op BenchmarkTop10of50Scores-4 300000 5174 ns/op 2464 B/op 26 allocs/op BenchmarkTop10of10000Scores-4 5000 342955 ns/op 2488 B/op 26 allocs/op BenchmarkTop100of0Scores-4 300000 4796 ns/op 18320 B/op 15 allocs/op BenchmarkTop100of3Scores-4 300000 5160 ns/op 18352 B/op 19 allocs/op BenchmarkTop100of10Scores-4 200000 6354 ns/op 18408 B/op 26 allocs/op BenchmarkTop100of25Scores-4 200000 10023 ns/op 18568 B/op 41 allocs/op BenchmarkTop100of50Scores-4 100000 16821 ns/op 18832 B/op 66 allocs/op BenchmarkTop100of10000Scores-4 3000 508989 ns/op 19760 B/op 117 allocs/op BenchmarkTop1000of10000Scores-4 1000 1814198 ns/op 184768 B/op 1017 allocs/op BenchmarkTop10000of100000Scores-4 50 26623920 ns/op 1939592 B/op 19024 allocs/op BenchmarkTop10of100000Scores-4 500 3730204 ns/op 2496 B/op 26 allocs/op BenchmarkTop100of100000Scores-4 300 4057127 ns/op 19912 B/op 117 allocs/op BenchmarkTop1000of100000Scores-4 200 6390180 ns/op 186200 B/op 1017 allocs/op BenchmarkTop10000of1000000Scores-4 20 82785756 ns/op 1963897 B/op 19024 allocs/op PASS ok github.com/blevesearch/bleve/search/collector 31.537s Previously with heap: go test -run=xxx -bench=. -benchmem BenchmarkTop10of0Scores-4 1000000 1216 ns/op 2288 B/op 15 allocs/op BenchmarkTop10of3Scores-4 1000000 1593 ns/op 2320 B/op 19 allocs/op BenchmarkTop10of10Scores-4 500000 2734 ns/op 2376 B/op 26 allocs/op BenchmarkTop10of25Scores-4 300000 5077 ns/op 2520 B/op 27 allocs/op BenchmarkTop10of50Scores-4 200000 6875 ns/op 2528 B/op 27 allocs/op BenchmarkTop10of10000Scores-4 3000 351210 ns/op 2552 B/op 27 allocs/op BenchmarkTop100of0Scores-4 300000 4846 ns/op 18304 B/op 15 allocs/op BenchmarkTop100of3Scores-4 300000 5357 ns/op 18336 B/op 19 allocs/op BenchmarkTop100of10Scores-4 200000 6462 ns/op 18392 B/op 26 allocs/op BenchmarkTop100of25Scores-4 200000 10012 ns/op 18552 B/op 41 allocs/op BenchmarkTop100of50Scores-4 100000 17089 ns/op 18816 B/op 66 allocs/op BenchmarkTop100of10000Scores-4 3000 528193 ns/op 19744 B/op 117 allocs/op BenchmarkTop1000of10000Scores-4 1000 1859447 ns/op 184752 B/op 1017 allocs/op BenchmarkTop10000of100000Scores-4 50 28005664 ns/op 1939576 B/op 19024 allocs/op BenchmarkTop10of100000Scores-4 300 4120091 ns/op 2560 B/op 27 allocs/op BenchmarkTop100of100000Scores-4 300 4325227 ns/op 19896 B/op 117 allocs/op BenchmarkTop1000of100000Scores-4 200 6799804 ns/op 186184 B/op 1017 allocs/op BenchmarkTop10000of1000000Scores-4 20 88494230 ns/op 1963881 B/op 19024 allocs/op PASS ok github.com/blevesearch/bleve/search/collector 30.198s Previously with slice: go test -run=xxx -bench=. -benchmem BenchmarkTop10of0Scores-4 1000000 1202 ns/op 2288 B/op 15 allocs/op BenchmarkTop10of3Scores-4 1000000 1453 ns/op 2288 B/op 18 allocs/op BenchmarkTop10of10Scores-4 1000000 2162 ns/op 2296 B/op 25 allocs/op BenchmarkTop10of25Scores-4 500000 3420 ns/op 2448 B/op 26 allocs/op BenchmarkTop10of50Scores-4 300000 5336 ns/op 2448 B/op 26 allocs/op BenchmarkTop10of10000Scores-4 5000 356733 ns/op 2472 B/op 26 allocs/op BenchmarkTop100of0Scores-4 300000 4877 ns/op 18304 B/op 15 allocs/op BenchmarkTop100of3Scores-4 300000 5132 ns/op 18304 B/op 18 allocs/op BenchmarkTop100of10Scores-4 200000 5787 ns/op 18312 B/op 25 allocs/op BenchmarkTop100of25Scores-4 200000 8083 ns/op 18344 B/op 40 allocs/op BenchmarkTop100of50Scores-4 100000 14419 ns/op 18400 B/op 65 allocs/op BenchmarkTop100of10000Scores-4 2000 665401 ns/op 18848 B/op 116 allocs/op BenchmarkTop1000of10000Scores-4 100 15417063 ns/op 176560 B/op 1016 allocs/op BenchmarkTop10000of100000Scores-4 1 1860011022 ns/op 1857960 B/op 19023 allocs/op BenchmarkTop10of100000Scores-4 300 4099276 ns/op 2480 B/op 26 allocs/op BenchmarkTop100of100000Scores-4 300 4533645 ns/op 18984 B/op 116 allocs/op BenchmarkTop1000of100000Scores-4 50 30519235 ns/op 178008 B/op 1016 allocs/op BenchmarkTop10000of1000000Scores-4 1 3483977385 ns/op 1882072 B/op 19023 allocs/op PASS ok github.com/blevesearch/bleve/search/collector 31.666s It appears that this sucessfully gets the best of both, in these particular benchmark sizes.
2017-04-27 14:52:52 +02:00
func (c *collectStoreList) AddNotExceedingSize(doc *search.DocumentMatch,
size int) *search.DocumentMatch {
c.add(doc)
if c.len() > size {
return c.removeLast()
}
return nil
}
func (c *collectStoreList) add(doc *search.DocumentMatch) {
for e := c.results.Front(); e != nil; e = e.Next() {
curr := e.Value.(*search.DocumentMatch)
if c.compare(doc, curr) >= 0 {
c.results.InsertBefore(doc, e)
return
}
}
// if we got to the end, we still have to add it
c.results.PushBack(doc)
}
topn collector switch approach based on size+skip we now use the slice store when size+skip <= 10 and use the heap store when size+skip > 10 here are the new perf numbers: go test -run=xxx -bench=. -benchmem BenchmarkTop10of0Scores-4 1000000 1150 ns/op 2304 B/op 15 allocs/op BenchmarkTop10of3Scores-4 1000000 1417 ns/op 2304 B/op 18 allocs/op BenchmarkTop10of10Scores-4 1000000 2133 ns/op 2312 B/op 25 allocs/op BenchmarkTop10of25Scores-4 500000 3410 ns/op 2464 B/op 26 allocs/op BenchmarkTop10of50Scores-4 300000 5174 ns/op 2464 B/op 26 allocs/op BenchmarkTop10of10000Scores-4 5000 342955 ns/op 2488 B/op 26 allocs/op BenchmarkTop100of0Scores-4 300000 4796 ns/op 18320 B/op 15 allocs/op BenchmarkTop100of3Scores-4 300000 5160 ns/op 18352 B/op 19 allocs/op BenchmarkTop100of10Scores-4 200000 6354 ns/op 18408 B/op 26 allocs/op BenchmarkTop100of25Scores-4 200000 10023 ns/op 18568 B/op 41 allocs/op BenchmarkTop100of50Scores-4 100000 16821 ns/op 18832 B/op 66 allocs/op BenchmarkTop100of10000Scores-4 3000 508989 ns/op 19760 B/op 117 allocs/op BenchmarkTop1000of10000Scores-4 1000 1814198 ns/op 184768 B/op 1017 allocs/op BenchmarkTop10000of100000Scores-4 50 26623920 ns/op 1939592 B/op 19024 allocs/op BenchmarkTop10of100000Scores-4 500 3730204 ns/op 2496 B/op 26 allocs/op BenchmarkTop100of100000Scores-4 300 4057127 ns/op 19912 B/op 117 allocs/op BenchmarkTop1000of100000Scores-4 200 6390180 ns/op 186200 B/op 1017 allocs/op BenchmarkTop10000of1000000Scores-4 20 82785756 ns/op 1963897 B/op 19024 allocs/op PASS ok github.com/blevesearch/bleve/search/collector 31.537s Previously with heap: go test -run=xxx -bench=. -benchmem BenchmarkTop10of0Scores-4 1000000 1216 ns/op 2288 B/op 15 allocs/op BenchmarkTop10of3Scores-4 1000000 1593 ns/op 2320 B/op 19 allocs/op BenchmarkTop10of10Scores-4 500000 2734 ns/op 2376 B/op 26 allocs/op BenchmarkTop10of25Scores-4 300000 5077 ns/op 2520 B/op 27 allocs/op BenchmarkTop10of50Scores-4 200000 6875 ns/op 2528 B/op 27 allocs/op BenchmarkTop10of10000Scores-4 3000 351210 ns/op 2552 B/op 27 allocs/op BenchmarkTop100of0Scores-4 300000 4846 ns/op 18304 B/op 15 allocs/op BenchmarkTop100of3Scores-4 300000 5357 ns/op 18336 B/op 19 allocs/op BenchmarkTop100of10Scores-4 200000 6462 ns/op 18392 B/op 26 allocs/op BenchmarkTop100of25Scores-4 200000 10012 ns/op 18552 B/op 41 allocs/op BenchmarkTop100of50Scores-4 100000 17089 ns/op 18816 B/op 66 allocs/op BenchmarkTop100of10000Scores-4 3000 528193 ns/op 19744 B/op 117 allocs/op BenchmarkTop1000of10000Scores-4 1000 1859447 ns/op 184752 B/op 1017 allocs/op BenchmarkTop10000of100000Scores-4 50 28005664 ns/op 1939576 B/op 19024 allocs/op BenchmarkTop10of100000Scores-4 300 4120091 ns/op 2560 B/op 27 allocs/op BenchmarkTop100of100000Scores-4 300 4325227 ns/op 19896 B/op 117 allocs/op BenchmarkTop1000of100000Scores-4 200 6799804 ns/op 186184 B/op 1017 allocs/op BenchmarkTop10000of1000000Scores-4 20 88494230 ns/op 1963881 B/op 19024 allocs/op PASS ok github.com/blevesearch/bleve/search/collector 30.198s Previously with slice: go test -run=xxx -bench=. -benchmem BenchmarkTop10of0Scores-4 1000000 1202 ns/op 2288 B/op 15 allocs/op BenchmarkTop10of3Scores-4 1000000 1453 ns/op 2288 B/op 18 allocs/op BenchmarkTop10of10Scores-4 1000000 2162 ns/op 2296 B/op 25 allocs/op BenchmarkTop10of25Scores-4 500000 3420 ns/op 2448 B/op 26 allocs/op BenchmarkTop10of50Scores-4 300000 5336 ns/op 2448 B/op 26 allocs/op BenchmarkTop10of10000Scores-4 5000 356733 ns/op 2472 B/op 26 allocs/op BenchmarkTop100of0Scores-4 300000 4877 ns/op 18304 B/op 15 allocs/op BenchmarkTop100of3Scores-4 300000 5132 ns/op 18304 B/op 18 allocs/op BenchmarkTop100of10Scores-4 200000 5787 ns/op 18312 B/op 25 allocs/op BenchmarkTop100of25Scores-4 200000 8083 ns/op 18344 B/op 40 allocs/op BenchmarkTop100of50Scores-4 100000 14419 ns/op 18400 B/op 65 allocs/op BenchmarkTop100of10000Scores-4 2000 665401 ns/op 18848 B/op 116 allocs/op BenchmarkTop1000of10000Scores-4 100 15417063 ns/op 176560 B/op 1016 allocs/op BenchmarkTop10000of100000Scores-4 1 1860011022 ns/op 1857960 B/op 19023 allocs/op BenchmarkTop10of100000Scores-4 300 4099276 ns/op 2480 B/op 26 allocs/op BenchmarkTop100of100000Scores-4 300 4533645 ns/op 18984 B/op 116 allocs/op BenchmarkTop1000of100000Scores-4 50 30519235 ns/op 178008 B/op 1016 allocs/op BenchmarkTop10000of1000000Scores-4 1 3483977385 ns/op 1882072 B/op 19023 allocs/op PASS ok github.com/blevesearch/bleve/search/collector 31.666s It appears that this sucessfully gets the best of both, in these particular benchmark sizes.
2017-04-27 14:52:52 +02:00
func (c *collectStoreList) removeLast() *search.DocumentMatch {
return c.results.Remove(c.results.Front()).(*search.DocumentMatch)
}
func (c *collectStoreList) Final(skip int, fixup collectorFixup) (search.DocumentMatchCollection, error) {
if c.results.Len()-skip > 0 {
rv := make(search.DocumentMatchCollection, c.results.Len()-skip)
i := 0
skipped := 0
for e := c.results.Back(); e != nil; e = e.Prev() {
if skipped < skip {
skipped++
continue
}
rv[i] = e.Value.(*search.DocumentMatch)
err := fixup(rv[i])
if err != nil {
return nil, err
}
i++
}
return rv, nil
}
return search.DocumentMatchCollection{}, nil
}
topn collector switch approach based on size+skip we now use the slice store when size+skip <= 10 and use the heap store when size+skip > 10 here are the new perf numbers: go test -run=xxx -bench=. -benchmem BenchmarkTop10of0Scores-4 1000000 1150 ns/op 2304 B/op 15 allocs/op BenchmarkTop10of3Scores-4 1000000 1417 ns/op 2304 B/op 18 allocs/op BenchmarkTop10of10Scores-4 1000000 2133 ns/op 2312 B/op 25 allocs/op BenchmarkTop10of25Scores-4 500000 3410 ns/op 2464 B/op 26 allocs/op BenchmarkTop10of50Scores-4 300000 5174 ns/op 2464 B/op 26 allocs/op BenchmarkTop10of10000Scores-4 5000 342955 ns/op 2488 B/op 26 allocs/op BenchmarkTop100of0Scores-4 300000 4796 ns/op 18320 B/op 15 allocs/op BenchmarkTop100of3Scores-4 300000 5160 ns/op 18352 B/op 19 allocs/op BenchmarkTop100of10Scores-4 200000 6354 ns/op 18408 B/op 26 allocs/op BenchmarkTop100of25Scores-4 200000 10023 ns/op 18568 B/op 41 allocs/op BenchmarkTop100of50Scores-4 100000 16821 ns/op 18832 B/op 66 allocs/op BenchmarkTop100of10000Scores-4 3000 508989 ns/op 19760 B/op 117 allocs/op BenchmarkTop1000of10000Scores-4 1000 1814198 ns/op 184768 B/op 1017 allocs/op BenchmarkTop10000of100000Scores-4 50 26623920 ns/op 1939592 B/op 19024 allocs/op BenchmarkTop10of100000Scores-4 500 3730204 ns/op 2496 B/op 26 allocs/op BenchmarkTop100of100000Scores-4 300 4057127 ns/op 19912 B/op 117 allocs/op BenchmarkTop1000of100000Scores-4 200 6390180 ns/op 186200 B/op 1017 allocs/op BenchmarkTop10000of1000000Scores-4 20 82785756 ns/op 1963897 B/op 19024 allocs/op PASS ok github.com/blevesearch/bleve/search/collector 31.537s Previously with heap: go test -run=xxx -bench=. -benchmem BenchmarkTop10of0Scores-4 1000000 1216 ns/op 2288 B/op 15 allocs/op BenchmarkTop10of3Scores-4 1000000 1593 ns/op 2320 B/op 19 allocs/op BenchmarkTop10of10Scores-4 500000 2734 ns/op 2376 B/op 26 allocs/op BenchmarkTop10of25Scores-4 300000 5077 ns/op 2520 B/op 27 allocs/op BenchmarkTop10of50Scores-4 200000 6875 ns/op 2528 B/op 27 allocs/op BenchmarkTop10of10000Scores-4 3000 351210 ns/op 2552 B/op 27 allocs/op BenchmarkTop100of0Scores-4 300000 4846 ns/op 18304 B/op 15 allocs/op BenchmarkTop100of3Scores-4 300000 5357 ns/op 18336 B/op 19 allocs/op BenchmarkTop100of10Scores-4 200000 6462 ns/op 18392 B/op 26 allocs/op BenchmarkTop100of25Scores-4 200000 10012 ns/op 18552 B/op 41 allocs/op BenchmarkTop100of50Scores-4 100000 17089 ns/op 18816 B/op 66 allocs/op BenchmarkTop100of10000Scores-4 3000 528193 ns/op 19744 B/op 117 allocs/op BenchmarkTop1000of10000Scores-4 1000 1859447 ns/op 184752 B/op 1017 allocs/op BenchmarkTop10000of100000Scores-4 50 28005664 ns/op 1939576 B/op 19024 allocs/op BenchmarkTop10of100000Scores-4 300 4120091 ns/op 2560 B/op 27 allocs/op BenchmarkTop100of100000Scores-4 300 4325227 ns/op 19896 B/op 117 allocs/op BenchmarkTop1000of100000Scores-4 200 6799804 ns/op 186184 B/op 1017 allocs/op BenchmarkTop10000of1000000Scores-4 20 88494230 ns/op 1963881 B/op 19024 allocs/op PASS ok github.com/blevesearch/bleve/search/collector 30.198s Previously with slice: go test -run=xxx -bench=. -benchmem BenchmarkTop10of0Scores-4 1000000 1202 ns/op 2288 B/op 15 allocs/op BenchmarkTop10of3Scores-4 1000000 1453 ns/op 2288 B/op 18 allocs/op BenchmarkTop10of10Scores-4 1000000 2162 ns/op 2296 B/op 25 allocs/op BenchmarkTop10of25Scores-4 500000 3420 ns/op 2448 B/op 26 allocs/op BenchmarkTop10of50Scores-4 300000 5336 ns/op 2448 B/op 26 allocs/op BenchmarkTop10of10000Scores-4 5000 356733 ns/op 2472 B/op 26 allocs/op BenchmarkTop100of0Scores-4 300000 4877 ns/op 18304 B/op 15 allocs/op BenchmarkTop100of3Scores-4 300000 5132 ns/op 18304 B/op 18 allocs/op BenchmarkTop100of10Scores-4 200000 5787 ns/op 18312 B/op 25 allocs/op BenchmarkTop100of25Scores-4 200000 8083 ns/op 18344 B/op 40 allocs/op BenchmarkTop100of50Scores-4 100000 14419 ns/op 18400 B/op 65 allocs/op BenchmarkTop100of10000Scores-4 2000 665401 ns/op 18848 B/op 116 allocs/op BenchmarkTop1000of10000Scores-4 100 15417063 ns/op 176560 B/op 1016 allocs/op BenchmarkTop10000of100000Scores-4 1 1860011022 ns/op 1857960 B/op 19023 allocs/op BenchmarkTop10of100000Scores-4 300 4099276 ns/op 2480 B/op 26 allocs/op BenchmarkTop100of100000Scores-4 300 4533645 ns/op 18984 B/op 116 allocs/op BenchmarkTop1000of100000Scores-4 50 30519235 ns/op 178008 B/op 1016 allocs/op BenchmarkTop10000of1000000Scores-4 1 3483977385 ns/op 1882072 B/op 19023 allocs/op PASS ok github.com/blevesearch/bleve/search/collector 31.666s It appears that this sucessfully gets the best of both, in these particular benchmark sizes.
2017-04-27 14:52:52 +02:00
func (c *collectStoreList) len() int {
return c.results.Len()
}