From 6f62489f21c479a514bb91fb1e42fdd4b1922c47 Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Tue, 4 Apr 2017 10:46:57 -0400 Subject: [PATCH] add option for multi term searcher to skip max disjunction check - geo searches now use this option and skip the check - export ComputeGeoTerms for geo debug visualizations --- search/searcher/search_disjunction.go | 23 +++++++++++++---- search/searcher/search_fuzzy.go | 2 +- search/searcher/search_geoboundingbox.go | 10 ++++---- search/searcher/search_geopointdistance.go | 2 +- search/searcher/search_multi_term.go | 29 ++++++++++++++-------- search/searcher/search_numeric_range.go | 3 ++- search/searcher/search_regexp.go | 2 +- search/searcher/search_term_prefix.go | 2 +- search/searcher/search_term_range.go | 2 +- 9 files changed, 48 insertions(+), 27 deletions(-) diff --git a/search/searcher/search_disjunction.go b/search/searcher/search_disjunction.go index 101c98b8..96bd5447 100644 --- a/search/searcher/search_disjunction.go +++ b/search/searcher/search_disjunction.go @@ -50,11 +50,22 @@ func tooManyClauses(count int) bool { } func tooManyClausesErr() error { - return fmt.Errorf("TooManyClauses[maxClauseCount is set to %d]", DisjunctionMaxClauseCount) + return fmt.Errorf("TooManyClauses[maxClauseCount is set to %d]", + DisjunctionMaxClauseCount) } -func NewDisjunctionSearcher(indexReader index.IndexReader, qsearchers []search.Searcher, min float64, options search.SearcherOptions) (*DisjunctionSearcher, error) { - if tooManyClauses(len(qsearchers)) { +func NewDisjunctionSearcher(indexReader index.IndexReader, + qsearchers []search.Searcher, min float64, options search.SearcherOptions) ( + *DisjunctionSearcher, error) { + return newDisjunctionSearcher(indexReader, qsearchers, min, options, + true) +} + +func newDisjunctionSearcher(indexReader index.IndexReader, + qsearchers []search.Searcher, min float64, options search.SearcherOptions, + limit bool) ( + *DisjunctionSearcher, error) { + if limit && tooManyClauses(len(qsearchers)) { return nil, tooManyClausesErr() } // build the downstream searchers @@ -161,7 +172,8 @@ func (s *DisjunctionSearcher) SetQueryNorm(qnorm float64) { } } -func (s *DisjunctionSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { +func (s *DisjunctionSearcher) Next(ctx *search.SearchContext) ( + *search.DocumentMatch, error) { if !s.initialized { err := s.initSearchers(ctx) if err != nil { @@ -199,7 +211,8 @@ func (s *DisjunctionSearcher) Next(ctx *search.SearchContext) (*search.DocumentM return rv, nil } -func (s *DisjunctionSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { +func (s *DisjunctionSearcher) Advance(ctx *search.SearchContext, + ID index.IndexInternalID) (*search.DocumentMatch, error) { if !s.initialized { err := s.initSearchers(ctx) if err != nil { diff --git a/search/searcher/search_fuzzy.go b/search/searcher/search_fuzzy.go index e17a03ec..90abaa0a 100644 --- a/search/searcher/search_fuzzy.go +++ b/search/searcher/search_fuzzy.go @@ -39,7 +39,7 @@ func NewFuzzySearcher(indexReader index.IndexReader, term string, } return NewMultiTermSearcher(indexReader, candidateTerms, field, - boost, options) + boost, options, true) } func findFuzzyCandidateTerms(indexReader index.IndexReader, term string, diff --git a/search/searcher/search_geoboundingbox.go b/search/searcher/search_geoboundingbox.go index cad8a003..f8b1b4cf 100644 --- a/search/searcher/search_geoboundingbox.go +++ b/search/searcher/search_geoboundingbox.go @@ -36,13 +36,13 @@ func NewGeoBoundingBoxSearcher(indexReader index.IndexReader, minLon, minLat, } // do math to produce list of terms needed for this search - onBoundaryTerms, notOnBoundaryTerms := computeRange(0, (geo.GeoBits<<1)-1, + onBoundaryTerms, notOnBoundaryTerms := ComputeGeoRange(0, (geo.GeoBits<<1)-1, minLon, minLat, maxLon, maxLat, checkBoundaries) var onBoundarySearcher search.Searcher if len(onBoundaryTerms) > 0 { rawOnBoundarySearcher, err := NewMultiTermSearcherBytes(indexReader, - onBoundaryTerms, field, boost, options) + onBoundaryTerms, field, boost, options, false) if err != nil { return nil, err } @@ -56,7 +56,7 @@ func NewGeoBoundingBoxSearcher(indexReader index.IndexReader, minLon, minLat, if len(notOnBoundaryTerms) > 0 { var err error notOnBoundarySearcher, err = NewMultiTermSearcherBytes(indexReader, - notOnBoundaryTerms, field, boost, options) + notOnBoundaryTerms, field, boost, options, false) if err != nil { cleanupOpenedSearchers() return nil, err @@ -88,7 +88,7 @@ func NewGeoBoundingBoxSearcher(indexReader index.IndexReader, minLon, minLat, var geoMaxShift = document.GeoPrecisionStep * 4 var geoDetailLevel = ((geo.GeoBits << 1) - geoMaxShift) / 2 -func computeRange(term uint64, shift uint, +func ComputeGeoRange(term uint64, shift uint, sminLon, sminLat, smaxLon, smaxLat float64, checkBoundaries bool) ( onBoundary [][]byte, notOnBoundary [][]byte) { @@ -138,7 +138,7 @@ func relateAndRecurse(start, end uint64, res uint, } else if level < geoDetailLevel && geo.RectIntersects(minLon, minLat, maxLon, maxLat, sminLon, sminLat, smaxLon, smaxLat) { - return computeRange(start, res-1, sminLon, sminLat, smaxLon, smaxLat, + return ComputeGeoRange(start, res-1, sminLon, sminLat, smaxLon, smaxLat, checkBoundaries) } return nil, nil diff --git a/search/searcher/search_geopointdistance.go b/search/searcher/search_geopointdistance.go index 2a4a4da8..c9d347fa 100644 --- a/search/searcher/search_geopointdistance.go +++ b/search/searcher/search_geopointdistance.go @@ -29,7 +29,7 @@ func NewGeoPointDistanceSearcher(indexReader index.IndexReader, centerLon, topLeftLon, topLeftLat, bottomRightLon, bottomRightLat := geo.ComputeBoundingBox(centerLon, centerLat, dist) - // build a searcher for the box + // build a searcher for the box boxSearcher, err := boxSearcher(indexReader, topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, field, boost, options) diff --git a/search/searcher/search_multi_term.go b/search/searcher/search_multi_term.go index afdc2fee..b469bead 100644 --- a/search/searcher/search_multi_term.go +++ b/search/searcher/search_multi_term.go @@ -20,7 +20,7 @@ import ( ) func NewMultiTermSearcher(indexReader index.IndexReader, terms []string, - field string, boost float64, options search.SearcherOptions) ( + field string, boost float64, options search.SearcherOptions, limit bool) ( search.Searcher, error) { qsearchers := make([]search.Searcher, len(terms)) qsearchersClose := func() { @@ -39,17 +39,12 @@ func NewMultiTermSearcher(indexReader index.IndexReader, terms []string, } } // build disjunction searcher of these ranges - searcher, err := NewDisjunctionSearcher(indexReader, qsearchers, 0, options) - if err != nil { - qsearchersClose() - return nil, err - } - - return searcher, nil + return newMultiTermSearcherBytes(indexReader, qsearchers, field, boost, + options, limit) } func NewMultiTermSearcherBytes(indexReader index.IndexReader, terms [][]byte, - field string, boost float64, options search.SearcherOptions) ( + field string, boost float64, options search.SearcherOptions, limit bool) ( search.Searcher, error) { qsearchers := make([]search.Searcher, len(terms)) qsearchersClose := func() { @@ -67,10 +62,22 @@ func NewMultiTermSearcherBytes(indexReader index.IndexReader, terms [][]byte, return nil, err } } + return newMultiTermSearcherBytes(indexReader, qsearchers, field, boost, + options, limit) +} + +func newMultiTermSearcherBytes(indexReader index.IndexReader, + searchers []search.Searcher, field string, boost float64, + options search.SearcherOptions, limit bool) ( + search.Searcher, error) { + // build disjunction searcher of these ranges - searcher, err := NewDisjunctionSearcher(indexReader, qsearchers, 0, options) + searcher, err := newDisjunctionSearcher(indexReader, searchers, 0, options, + limit) if err != nil { - qsearchersClose() + for _, s := range searchers { + _ = s.Close() + } return nil, err } diff --git a/search/searcher/search_numeric_range.go b/search/searcher/search_numeric_range.go index fbfd1776..224cc9e2 100644 --- a/search/searcher/search_numeric_range.go +++ b/search/searcher/search_numeric_range.go @@ -59,7 +59,8 @@ func NewNumericRangeSearcher(indexReader index.IndexReader, return nil, tooManyClausesErr() } - return NewMultiTermSearcherBytes(indexReader, terms, field, boost, options) + return NewMultiTermSearcherBytes(indexReader, terms, field, boost, options, + true) } type termRange struct { diff --git a/search/searcher/search_regexp.go b/search/searcher/search_regexp.go index 3e17195b..b7cf520a 100644 --- a/search/searcher/search_regexp.go +++ b/search/searcher/search_regexp.go @@ -45,7 +45,7 @@ func NewRegexpSearcher(indexReader index.IndexReader, pattern *regexp.Regexp, } return NewMultiTermSearcher(indexReader, candidateTerms, field, boost, - options) + options, true) } func findRegexpCandidateTerms(indexReader index.IndexReader, diff --git a/search/searcher/search_term_prefix.go b/search/searcher/search_term_prefix.go index 379443fc..05d09224 100644 --- a/search/searcher/search_term_prefix.go +++ b/search/searcher/search_term_prefix.go @@ -35,5 +35,5 @@ func NewTermPrefixSearcher(indexReader index.IndexReader, prefix string, tfd, err = fieldDict.Next() } - return NewMultiTermSearcher(indexReader, terms, field, boost, options) + return NewMultiTermSearcher(indexReader, terms, field, boost, options, true) } diff --git a/search/searcher/search_term_range.go b/search/searcher/search_term_range.go index 7b226dfc..24170c71 100644 --- a/search/searcher/search_term_range.go +++ b/search/searcher/search_term_range.go @@ -71,5 +71,5 @@ func NewTermRangeSearcher(indexReader index.IndexReader, terms = terms[:len(terms)-1] } - return NewMultiTermSearcher(indexReader, terms, field, boost, options) + return NewMultiTermSearcher(indexReader, terms, field, boost, options, true) }