gibheer
/
bleve
0
0
Fork 0
Code

optimize upside_down reader Next() with doc match reuse 

This optimization changes the search.Search.Next() interface API,
adding an optional, pre-allocated *DocumentMatch parameter.

When it's non-nil, the TermSearcher and TermQueryScorer will use that
pre-allocated *DocumentMatch, instead of allocating a brand new
DocumentMatch instance.
This commit is contained in:
Steve Yen 2016-07-20 16:29:20 -07:00
parent 2498ccc913
commit 988ca62182
27 changed files with 80 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
search/collectors/collector_top_score.go
View File

@ -60,12 +60,13 @@ func (tksc *TopScoreCollector) Took() time.Duration {
func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Searcher) error {
startTime := time.Now()
var err error
var pre search.DocumentMatch // Pre-alloc'ed instance.
var next *search.DocumentMatch
select {
case <-ctx.Done():
return ctx.Err()
default:
next, err = searcher.Next()
next, err = searcher.Next(&pre)
}
for err == nil && next != nil {
select {
@ -79,7 +80,7 @@ func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Sear
break
}
}
next, err = searcher.Next()
next, err = searcher.Next(pre.Reset())
}
}
// compute search duration
@ -90,19 +91,22 @@ func (tksc *TopScoreCollector) Collect(ctx context.Context, searcher search.Sear
return nil
}
func (tksc *TopScoreCollector) collectSingle(dm *search.DocumentMatch) {
func (tksc *TopScoreCollector) collectSingle(dmIn *search.DocumentMatch) {
// increment total hits
tksc.total++
// update max score
if dm.Score > tksc.maxScore {
tksc.maxScore = dm.Score
if dmIn.Score > tksc.maxScore {
tksc.maxScore = dmIn.Score
}
if dm.Score <= tksc.minScore {
if dmIn.Score <= tksc.minScore {
return
}
dm := &search.DocumentMatch{}
*dm = *dmIn
for e := tksc.results.Front(); e != nil; e = e.Next() {
curr := e.Value.(*search.DocumentMatch)
if dm.Score <= curr.Score {

2
search/collectors/search_test.go
View File

@ -18,7 +18,7 @@ type stubSearcher struct {
matches search.DocumentMatchCollection
}
func (ss *stubSearcher) Next() (*search.DocumentMatch, error) {
func (ss *stubSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
if ss.index < len(ss.matches) {
rv := ss.matches[ss.index]
ss.index++

12
search/scorers/scorer_term.go
View File

@ -83,7 +83,7 @@ func (s *TermQueryScorer) SetQueryNorm(qnorm float64) {
}
}
func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc) *search.DocumentMatch {
func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc, preAllocated *search.DocumentMatch) *search.DocumentMatch {
var scoreExplanation *search.Explanation
// need to compute score
@ -128,10 +128,12 @@ func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc) *search.DocumentM
}
}
rv := search.DocumentMatch{
ID: termMatch.ID,
Score: score,
rv := preAllocated
if rv == nil {
rv = &search.DocumentMatch{}
}
rv.ID = termMatch.ID
rv.Score = score
if s.explain {
rv.Expl = scoreExplanation
}
@ -172,5 +174,5 @@ func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc) *search.DocumentM
}
return &rv
return rv
}

4
search/scorers/scorer_term_test.go
View File

@ -144,7 +144,7 @@ func TestTermScorer(t *testing.T) {
}
for _, test := range tests {
actual := scorer.Score(test.termMatch)
actual := scorer.Score(test.termMatch, nil)
if !reflect.DeepEqual(actual, test.result) {
t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch)
@ -231,7 +231,7 @@ func TestTermScorerWithQueryNorm(t *testing.T) {
}
for _, test := range tests {
actual := scorer.Score(test.termMatch)
actual := scorer.Score(test.termMatch, nil)
if !reflect.DeepEqual(actual, test.result) {
t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch)

7
search/search.go
View File

@ -85,6 +85,11 @@ func (dm *DocumentMatch) AddFieldValue(name string, value interface{}) {
dm.Fields[name] = valSlice
}
func (dm *DocumentMatch) Reset() *DocumentMatch {
*dm = DocumentMatch{}
return dm
}
type DocumentMatchCollection []*DocumentMatch
func (c DocumentMatchCollection) Len() int { return len(c) }
@ -92,7 +97,7 @@ func (c DocumentMatchCollection) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
func (c DocumentMatchCollection) Less(i, j int) bool { return c[i].Score > c[j].Score }
type Searcher interface {
Next() (*DocumentMatch, error)
Next(preAllocated *DocumentMatch) (*DocumentMatch, error)
Advance(ID string) (*DocumentMatch, error)
Close() error
Weight() float64

14
search/searchers/search_boolean.go
View File

@ -70,21 +70,21 @@ func (s *BooleanSearcher) initSearchers() error {
var err error
// get all searchers pointing at their first match
if s.mustSearcher != nil {
s.currMust, err = s.mustSearcher.Next()
s.currMust, err = s.mustSearcher.Next(nil)
if err != nil {
return err
}
}
if s.shouldSearcher != nil {
s.currShould, err = s.shouldSearcher.Next()
s.currShould, err = s.shouldSearcher.Next(nil)
if err != nil {
return err
}
}
if s.mustNotSearcher != nil {
s.currMustNot, err = s.mustNotSearcher.Next()
s.currMustNot, err = s.mustNotSearcher.Next(nil)
if err != nil {
return err
}
@ -106,12 +106,12 @@ func (s *BooleanSearcher) advanceNextMust() error {
var err error
if s.mustSearcher != nil {
s.currMust, err = s.mustSearcher.Next()
s.currMust, err = s.mustSearcher.Next(nil)
if err != nil {
return err
}
} else if s.mustSearcher == nil {
s.currShould, err = s.shouldSearcher.Next()
s.currShould, err = s.shouldSearcher.Next(nil)
if err != nil {
return err
}
@ -148,7 +148,7 @@ func (s *BooleanSearcher) SetQueryNorm(qnorm float64) {
}
}
func (s *BooleanSearcher) Next() (*search.DocumentMatch, error) {
func (s *BooleanSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
if !s.initialized {
err := s.initSearchers()
@ -292,7 +292,7 @@ func (s *BooleanSearcher) Advance(ID string) (*search.DocumentMatch, error) {
s.currentID = ""
}
return s.Next()
return s.Next(nil)
}
func (s *BooleanSearcher) Count() uint64 {

4
search/searchers/search_boolean_test.go
View File

@ -342,7 +342,7 @@ func TestBooleanSearch(t *testing.T) {
}
}()
next, err := test.searcher.Next()
next, err := test.searcher.Next(nil)
i := 0
for err == nil && next != nil {
if i < len(test.results) {
@ -354,7 +354,7 @@ func TestBooleanSearch(t *testing.T) {
t.Logf("scoring explanation: %s", next.Expl)
}
}
next, err = test.searcher.Next()
next, err = test.searcher.Next(nil)
i++
}
if err != nil {

8
search/searchers/search_conjunction.go
View File

@ -67,7 +67,7 @@ func (s *ConjunctionSearcher) initSearchers() error {
var err error
// get all searchers pointing at their first match
for i, termSearcher := range s.searchers {
s.currs[i], err = termSearcher.Next()
s.currs[i], err = termSearcher.Next(nil)
if err != nil {
return err
}
@ -99,7 +99,7 @@ func (s *ConjunctionSearcher) SetQueryNorm(qnorm float64) {
}
}
func (s *ConjunctionSearcher) Next() (*search.DocumentMatch, error) {
func (s *ConjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
if !s.initialized {
err := s.initSearchers()
if err != nil {
@ -140,7 +140,7 @@ OUTER:
rv = s.scorer.Score(s.currs)
// prepare for next entry
s.currs[0], err = s.searchers[0].Next()
s.currs[0], err = s.searchers[0].Next(nil)
if err != nil {
return nil, err
}
@ -170,7 +170,7 @@ func (s *ConjunctionSearcher) Advance(ID string) (*search.DocumentMatch, error)
}
}
s.currentID = ID
return s.Next()
return s.Next(nil)
}
func (s *ConjunctionSearcher) Count() uint64 {

4
search/searchers/search_conjunction_test.go
View File

@ -187,7 +187,7 @@ func TestConjunctionSearch(t *testing.T) {
}
}()
next, err := test.searcher.Next()
next, err := test.searcher.Next(nil)
i := 0
for err == nil && next != nil {
if i < len(test.results) {
@ -199,7 +199,7 @@ func TestConjunctionSearch(t *testing.T) {
t.Logf("scoring explanation: %s", next.Expl)
}
}
next, err = test.searcher.Next()
next, err = test.searcher.Next(nil)
i++
}
if err != nil {

8
search/searchers/search_disjunction.go
View File

@ -87,7 +87,7 @@ func (s *DisjunctionSearcher) initSearchers() error {
var err error
// get all searchers pointing at their first match
for i, termSearcher := range s.searchers {
s.currs[i], err = termSearcher.Next()
s.currs[i], err = termSearcher.Next(nil)
if err != nil {
return err
}
@ -122,7 +122,7 @@ func (s *DisjunctionSearcher) SetQueryNorm(qnorm float64) {
}
}
func (s *DisjunctionSearcher) Next() (*search.DocumentMatch, error) {
func (s *DisjunctionSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
if !s.initialized {
err := s.initSearchers()
if err != nil {
@ -153,7 +153,7 @@ func (s *DisjunctionSearcher) Next() (*search.DocumentMatch, error) {
for i, curr := range s.currs {
if curr != nil && curr.ID == s.currentID {
searcher := s.searchers[i]
s.currs[i], err = searcher.Next()
s.currs[i], err = searcher.Next(nil)
if err != nil {
return nil, err
}
@ -182,7 +182,7 @@ func (s *DisjunctionSearcher) Advance(ID string) (*search.DocumentMatch, error)
s.currentID = s.nextSmallestID()
return s.Next()
return s.Next(nil)
}
func (s *DisjunctionSearcher) Count() uint64 {

4
search/searchers/search_disjunction_test.go
View File

@ -108,7 +108,7 @@ func TestDisjunctionSearch(t *testing.T) {
}
}()
next, err := test.searcher.Next()
next, err := test.searcher.Next(nil)
i := 0
for err == nil && next != nil {
if i < len(test.results) {
@ -120,7 +120,7 @@ func TestDisjunctionSearch(t *testing.T) {
t.Logf("scoring explanation: %s", next.Expl)
}
}
next, err = test.searcher.Next()
next, err = test.searcher.Next(nil)
i++
}
if err != nil {

4
search/searchers/search_docid.go
View File

@ -77,7 +77,7 @@ func (s *DocIDSearcher) SetQueryNorm(qnorm float64) {
s.scorer.SetQueryNorm(qnorm)
}
func (s *DocIDSearcher) Next() (*search.DocumentMatch, error) {
func (s *DocIDSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
if s.current >= len(s.ids) {
return nil, nil
}
@ -90,7 +90,7 @@ func (s *DocIDSearcher) Next() (*search.DocumentMatch, error) {
func (s *DocIDSearcher) Advance(ID string) (*search.DocumentMatch, error) {
s.current = sort.SearchStrings(s.ids, ID)
return s.Next()
return s.Next(nil)
}
func (s *DocIDSearcher) Close() error {

4
search/searchers/search_docid_test.go
View File

@ -68,7 +68,7 @@ func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) {
// Check the sequence
for i, id := range wanted {
m, err := searcher.Next()
m, err := searcher.Next(nil)
if err != nil {
t.Fatal(err)
}
@ -76,7 +76,7 @@ func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) {
t.Fatalf("expected %v at position %v, got %v", id, i, m.ID)
}
}
m, err := searcher.Next()
m, err := searcher.Next(nil)
if err != nil {
t.Fatal(err)
}

4
search/searchers/search_fuzzy.go
View File

@ -107,8 +107,8 @@ func (s *FuzzySearcher) SetQueryNorm(qnorm float64) {
s.searcher.SetQueryNorm(qnorm)
}
func (s *FuzzySearcher) Next() (*search.DocumentMatch, error) {
return s.searcher.Next()
func (s *FuzzySearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
return s.searcher.Next(preAllocated)
}

4
search/searchers/search_fuzzy_test.go
View File

@ -105,7 +105,7 @@ func TestFuzzySearch(t *testing.T) {
}
}()
next, err := test.searcher.Next()
next, err := test.searcher.Next(nil)
i := 0
for err == nil && next != nil {
if i < len(test.results) {
@ -117,7 +117,7 @@ func TestFuzzySearch(t *testing.T) {
t.Logf("scoring explanation: %s", next.Expl)
}
}
next, err = test.searcher.Next()
next, err = test.searcher.Next(nil)
i++
}
if err != nil {

2
search/searchers/search_match_all.go
View File

@ -46,7 +46,7 @@ func (s *MatchAllSearcher) SetQueryNorm(qnorm float64) {
s.scorer.SetQueryNorm(qnorm)
}
func (s *MatchAllSearcher) Next() (*search.DocumentMatch, error) {
func (s *MatchAllSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
id, err := s.reader.Next()
if err != nil {
return nil, err

4
search/searchers/search_match_all_test.go
View File

@ -109,7 +109,7 @@ func TestMatchAllSearch(t *testing.T) {
}
}()
next, err := test.searcher.Next()
next, err := test.searcher.Next(nil)
i := 0
for err == nil && next != nil {
if i < len(test.results) {
@ -121,7 +121,7 @@ func TestMatchAllSearch(t *testing.T) {
t.Logf("scoring explanation: %s", next.Expl)
}
}
next, err = test.searcher.Next()
next, err = test.searcher.Next(nil)
i++
}
if err != nil {

2
search/searchers/search_match_none.go
View File

@ -36,7 +36,7 @@ func (s *MatchNoneSearcher) SetQueryNorm(qnorm float64) {
}
func (s *MatchNoneSearcher) Next() (*search.DocumentMatch, error) {
func (s *MatchNoneSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
return nil, nil
}

4
search/searchers/search_match_none_test.go
View File

@ -51,7 +51,7 @@ func TestMatchNoneSearch(t *testing.T) {
}
}()
next, err := test.searcher.Next()
next, err := test.searcher.Next(nil)
i := 0
for err == nil && next != nil {
if i < len(test.results) {
@ -63,7 +63,7 @@ func TestMatchNoneSearch(t *testing.T) {
t.Logf("scoring explanation: %s", next.Expl)
}
}
next, err = test.searcher.Next()
next, err = test.searcher.Next(nil)
i++
}
if err != nil {

4
search/searchers/search_numeric_range.go
View File

@ -96,8 +96,8 @@ func (s *NumericRangeSearcher) SetQueryNorm(qnorm float64) {
s.searcher.SetQueryNorm(qnorm)
}
func (s *NumericRangeSearcher) Next() (*search.DocumentMatch, error) {
return s.searcher.Next()
func (s *NumericRangeSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
return s.searcher.Next(preAllocated)
}
func (s *NumericRangeSearcher) Advance(ID string) (*search.DocumentMatch, error) {

8
search/searchers/search_phrase.go
View File

@ -56,7 +56,7 @@ func (s *PhraseSearcher) initSearchers() error {
var err error
// get all searchers pointing at their first match
if s.mustSearcher != nil {
s.currMust, err = s.mustSearcher.Next()
s.currMust, err = s.mustSearcher.Next(nil)
if err != nil {
return err
}
@ -70,7 +70,7 @@ func (s *PhraseSearcher) advanceNextMust() error {
var err error
if s.mustSearcher != nil {
s.currMust, err = s.mustSearcher.Next()
s.currMust, err = s.mustSearcher.Next(nil)
if err != nil {
return err
}
@ -90,7 +90,7 @@ func (s *PhraseSearcher) SetQueryNorm(qnorm float64) {
s.mustSearcher.SetQueryNorm(qnorm)
}
func (s *PhraseSearcher) Next() (*search.DocumentMatch, error) {
func (s *PhraseSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
if !s.initialized {
err := s.initSearchers()
if err != nil {
@ -172,7 +172,7 @@ func (s *PhraseSearcher) Advance(ID string) (*search.DocumentMatch, error) {
if err != nil {
return nil, err
}
return s.Next()
return s.Next(nil)
}
func (s *PhraseSearcher) Count() uint64 {

4
search/searchers/search_phrase_test.go
View File

@ -68,7 +68,7 @@ func TestPhraseSearch(t *testing.T) {
}
}()
next, err := test.searcher.Next()
next, err := test.searcher.Next(nil)
i := 0
for err == nil && next != nil {
if i < len(test.results) {
@ -80,7 +80,7 @@ func TestPhraseSearch(t *testing.T) {
t.Logf("scoring explanation: %s", next.Expl)
}
}
next, err = test.searcher.Next()
next, err = test.searcher.Next(nil)
i++
}
if err != nil {

4
search/searchers/search_regexp.go
View File

@ -106,8 +106,8 @@ func (s *RegexpSearcher) SetQueryNorm(qnorm float64) {
s.searcher.SetQueryNorm(qnorm)
}
func (s *RegexpSearcher) Next() (*search.DocumentMatch, error) {
return s.searcher.Next()
func (s *RegexpSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
return s.searcher.Next(preAllocated)
}

4
search/searchers/search_regexp_test.go
View File

@ -85,7 +85,7 @@ func TestRegexpSearch(t *testing.T) {
}
}()
next, err := test.searcher.Next()
next, err := test.searcher.Next(nil)
i := 0
for err == nil && next != nil {
if i < len(test.results) {
@ -97,7 +97,7 @@ func TestRegexpSearch(t *testing.T) {
t.Logf("scoring explanation: %s", next.Expl)
}
}
next, err = test.searcher.Next()
next, err = test.searcher.Next(nil)
i++
}
if err != nil {

6
search/searchers/search_term.go
View File

@ -52,7 +52,7 @@ func (s *TermSearcher) SetQueryNorm(qnorm float64) {
s.scorer.SetQueryNorm(qnorm)
}
func (s *TermSearcher) Next() (*search.DocumentMatch, error) {
func (s *TermSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
termMatch, err := s.reader.Next()
if err != nil {
return nil, err
@ -63,7 +63,7 @@ func (s *TermSearcher) Next() (*search.DocumentMatch, error) {
}
// score match
docMatch := s.scorer.Score(termMatch)
docMatch := s.scorer.Score(termMatch, preAllocated)
// return doc match
return docMatch, nil
@ -80,7 +80,7 @@ func (s *TermSearcher) Advance(ID string) (*search.DocumentMatch, error) {
}
// score match
docMatch := s.scorer.Score(termMatch)
docMatch := s.scorer.Score(termMatch, nil)
// return doc match
return docMatch, nil

4
search/searchers/search_term_prefix.go
View File

@ -70,8 +70,8 @@ func (s *TermPrefixSearcher) SetQueryNorm(qnorm float64) {
s.searcher.SetQueryNorm(qnorm)
}
func (s *TermPrefixSearcher) Next() (*search.DocumentMatch, error) {
return s.searcher.Next()
func (s *TermPrefixSearcher) Next(preAllocated *search.DocumentMatch) (*search.DocumentMatch, error) {
return s.searcher.Next(preAllocated)
}

4
search/searchers/search_term_test.go
View File

@ -163,7 +163,7 @@ func TestTermSearcher(t *testing.T) {
t.Errorf("expected count of 9, got %d", searcher.Count())
}
docMatch, err := searcher.Next()
docMatch, err := searcher.Next(nil)
if err != nil {
t.Errorf("expected result, got %v", err)
}
@ -188,7 +188,7 @@ func TestTermSearcher(t *testing.T) {
}
// try pushing next past end
docMatch, err = searcher.Next()
docMatch, err = searcher.Next(nil)
if err != nil {
t.Fatal(err)
}