2014-04-17 22:55:53 +02:00
|
|
|
// Copyright (c) 2014 Couchbase, Inc.
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
|
|
|
|
// except in compliance with the License. You may obtain a copy of the License at
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
// Unless required by applicable law or agreed to in writing, software distributed under the
|
|
|
|
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
|
|
|
// either express or implied. See the License for the specific language governing permissions
|
|
|
|
// and limitations under the License.
|
2014-09-02 16:54:50 +02:00
|
|
|
|
2014-09-01 17:15:38 +02:00
|
|
|
package collectors
|
2014-04-17 22:55:53 +02:00
|
|
|
|
|
|
|
import (
|
|
|
|
"container/list"
|
|
|
|
"time"
|
2014-09-01 17:15:38 +02:00
|
|
|
|
2016-03-02 22:55:02 +01:00
|
|
|
"golang.org/x/net/context"
|
|
|
|
|
2014-09-01 17:15:38 +02:00
|
|
|
"github.com/blevesearch/bleve/search"
|
2014-04-17 22:55:53 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
type TopScoreCollector struct {
|
2014-08-11 17:03:29 +02:00
|
|
|
k int
|
|
|
|
skip int
|
|
|
|
results *list.List
|
|
|
|
took time.Duration
|
|
|
|
maxScore float64
|
2016-04-29 18:46:47 +02:00
|
|
|
minScore float64
|
2014-08-11 17:03:29 +02:00
|
|
|
total uint64
|
2014-09-01 17:15:38 +02:00
|
|
|
facetsBuilder *search.FacetsBuilder
|
2014-04-17 22:55:53 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
func NewTopScorerCollector(k int) *TopScoreCollector {
|
|
|
|
return &TopScoreCollector{
|
|
|
|
k: k,
|
2014-07-30 18:30:38 +02:00
|
|
|
skip: 0,
|
|
|
|
results: list.New(),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewTopScorerSkipCollector(k, skip int) *TopScoreCollector {
|
|
|
|
return &TopScoreCollector{
|
|
|
|
k: k,
|
|
|
|
skip: skip,
|
2014-04-17 22:55:53 +02:00
|
|
|
results: list.New(),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (tksc *TopScoreCollector) Total() uint64 {
|
|
|
|
return tksc.total
|
|
|
|
}
|
|
|
|
|
|
|
|
func (tksc *TopScoreCollector) MaxScore() float64 {
|
|
|
|
return tksc.maxScore
|
|
|
|
}
|
|
|
|
|
|
|
|
func (tksc *TopScoreCollector) Took() time.Duration {
|
|
|
|
return tksc.took
|
|
|
|
}
|
|
|
|
|
2016-03-02 22:55:02 +01:00
|
|
|
func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Searcher) error {
|
2014-04-17 22:55:53 +02:00
|
|
|
startTime := time.Now()
|
2016-03-02 22:55:02 +01:00
|
|
|
var err error
|
|
|
|
var next *search.DocumentMatch
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return ctx.Err()
|
|
|
|
default:
|
|
|
|
next, err = searcher.Next()
|
|
|
|
}
|
2014-04-17 22:55:53 +02:00
|
|
|
for err == nil && next != nil {
|
2016-03-02 22:55:02 +01:00
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return ctx.Err()
|
|
|
|
default:
|
|
|
|
tksc.collectSingle(next)
|
|
|
|
if tksc.facetsBuilder != nil {
|
|
|
|
err = tksc.facetsBuilder.Update(next)
|
|
|
|
if err != nil {
|
|
|
|
break
|
|
|
|
}
|
2015-03-06 20:46:29 +01:00
|
|
|
}
|
2016-03-02 22:55:02 +01:00
|
|
|
next, err = searcher.Next()
|
2014-08-11 17:03:29 +02:00
|
|
|
}
|
2014-04-17 22:55:53 +02:00
|
|
|
}
|
|
|
|
// compute search duration
|
|
|
|
tksc.took = time.Since(startTime)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2014-09-01 17:15:38 +02:00
|
|
|
func (tksc *TopScoreCollector) collectSingle(dm *search.DocumentMatch) {
|
2014-04-17 22:55:53 +02:00
|
|
|
// increment total hits
|
2014-09-04 00:47:02 +02:00
|
|
|
tksc.total++
|
2014-04-17 22:55:53 +02:00
|
|
|
|
|
|
|
// update max score
|
|
|
|
if dm.Score > tksc.maxScore {
|
|
|
|
tksc.maxScore = dm.Score
|
|
|
|
}
|
|
|
|
|
2016-04-29 18:46:47 +02:00
|
|
|
if dm.Score <= tksc.minScore {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2014-04-17 22:55:53 +02:00
|
|
|
for e := tksc.results.Front(); e != nil; e = e.Next() {
|
2014-09-01 17:15:38 +02:00
|
|
|
curr := e.Value.(*search.DocumentMatch)
|
fix pagination bug introduced by collector optimization
fixes #378
this bug was introduced by:
https://github.com/blevesearch/bleve/commit/f2aba116c49ea51b27bc9afd3bf15305ef04883c
theory of operation for this collector (top N, skip K)
- collect the highest scoring N+K results
- if K > 0, skip K and return the next N
internal details
- the top N+K are kept in a list
- the list is ordered from lowest scoring (first) to highest scoring (last)
- as a hit comes in, we find where this new hit would fit into this list
- if this caused the list to get too big, trim off the head (lowest scoring hit)
theory of the optimization
- we were not tracking the lowest score in the list
- so if the score was lower than the lowest score, we would add/remove it
- by keeping track of the lowest score in the list, we can avoid these ops
problem with the optimization
- the optimization worked by returning early
- by returning early there was a subtle change to documents which had the same score
- the reason is that which docs end up in the top N+K changed by returning early
- why was that? docs are coming in, in order by key ascending
- when finding the correct position to insert a hit into the list, we checked <, not <= the score
- this has the subtle effect that docs with the same score end up in reverse order
for example consider the following in progress list:
doc ids [ c a b ]
scores [ 1 5 9 ]
if we now see doc d with score 5, we get:
doc ids [ c a d b ]
scores [ 1 5 5 9 ]
While that appears in order (a, d) it is actually reverse order, because when we
produce the top N we start at the end.
theory of the fix
- previous pagination depended on later hits with the same score "bumping" earlier
hits with the same score off the bottom of the list
- however, if we change the logic to <= instead of <, now the list in the previous
example would look like:
doc ids [ c d a b ]
scores [ 1 5 5 9 ]
- this small change means that now earlier (lower id) will score higher, and
thus we no longer depend on later hits bumping things down, which means returning
early is a valid thing to do
NOTE: this does depend on the hits coming back in order by ID. this is not
something strictly guaranteed, but it was the same assumption that allowed the
original behavior
This also has the side-effect that 2 hits with the same score come back in
ascending ID order, which is somehow more pleasing to me than reverse order.
2016-06-01 16:43:14 +02:00
|
|
|
if dm.Score <= curr.Score {
|
2014-04-17 22:55:53 +02:00
|
|
|
|
|
|
|
tksc.results.InsertBefore(dm, e)
|
|
|
|
// if we just made the list too long
|
2014-07-30 18:30:38 +02:00
|
|
|
if tksc.results.Len() > (tksc.k + tksc.skip) {
|
2014-04-17 22:55:53 +02:00
|
|
|
// remove the head
|
2016-04-29 18:46:47 +02:00
|
|
|
tksc.minScore = tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch).Score
|
2014-04-17 22:55:53 +02:00
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// if we got to the end, we still have to add it
|
|
|
|
tksc.results.PushBack(dm)
|
2014-07-30 18:30:38 +02:00
|
|
|
if tksc.results.Len() > (tksc.k + tksc.skip) {
|
2014-04-17 22:55:53 +02:00
|
|
|
// remove the head
|
2016-04-29 18:46:47 +02:00
|
|
|
tksc.minScore = tksc.results.Remove(tksc.results.Front()).(*search.DocumentMatch).Score
|
2014-04-17 22:55:53 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-09-01 17:15:38 +02:00
|
|
|
func (tksc *TopScoreCollector) Results() search.DocumentMatchCollection {
|
2014-07-30 18:30:38 +02:00
|
|
|
if tksc.results.Len()-tksc.skip > 0 {
|
2014-09-01 17:15:38 +02:00
|
|
|
rv := make(search.DocumentMatchCollection, tksc.results.Len()-tksc.skip)
|
2014-07-30 18:30:38 +02:00
|
|
|
i := 0
|
|
|
|
skipped := 0
|
|
|
|
for e := tksc.results.Back(); e != nil; e = e.Prev() {
|
|
|
|
if skipped < tksc.skip {
|
|
|
|
skipped++
|
|
|
|
continue
|
|
|
|
}
|
2014-09-01 17:15:38 +02:00
|
|
|
rv[i] = e.Value.(*search.DocumentMatch)
|
2014-07-30 18:30:38 +02:00
|
|
|
i++
|
|
|
|
}
|
|
|
|
return rv
|
2014-04-17 22:55:53 +02:00
|
|
|
}
|
2014-09-01 17:15:38 +02:00
|
|
|
return search.DocumentMatchCollection{}
|
2014-04-17 22:55:53 +02:00
|
|
|
}
|
2014-08-11 17:03:29 +02:00
|
|
|
|
2014-09-01 17:15:38 +02:00
|
|
|
func (tksc *TopScoreCollector) SetFacetsBuilder(facetsBuilder *search.FacetsBuilder) {
|
2014-08-11 17:03:29 +02:00
|
|
|
tksc.facetsBuilder = facetsBuilder
|
|
|
|
}
|
|
|
|
|
2014-09-01 17:15:38 +02:00
|
|
|
func (tksc *TopScoreCollector) FacetResults() search.FacetResults {
|
2014-08-11 17:03:29 +02:00
|
|
|
if tksc.facetsBuilder != nil {
|
|
|
|
return tksc.facetsBuilder.Results()
|
|
|
|
}
|
2014-09-04 00:47:02 +02:00
|
|
|
return search.FacetResults{}
|
2014-08-11 17:03:29 +02:00
|
|
|
}
|