0
0
Fork 0

improve query string compatibility

1) disjunction and conjunction queries now support a
"query string mode".  By default they do not operate
in this mode.  When in this mode, any disjunct/conjunct
which evaluates to MatchNone searcher, will be removed
from the disjunction/conjunction.  If the query ends
up with NO conjuncts/disjuncts, it will itself
return the MatchNone seacher.

2) boolean query also supports a query string mode.  when in
this mode, the Must, Should and MustNot searchers are all put
into query string mode.

3) rewriting of negation only queries (like -foo) now take into
account the rewriting rules above, and those are handled first.
this means that we rewrite correctly in case of +stoword -foo

4) the empty query string is now valid, and returns 0 hits.
previously this was considered a validation error.
This commit is contained in:
Marty Schoch 2017-02-23 12:06:47 -05:00
parent dbdb8b984d
commit f391b991c2
5 changed files with 129 additions and 72 deletions

View File

@ -25,10 +25,11 @@ import (
)
type BooleanQuery struct {
Must Query `json:"must,omitempty"`
Should Query `json:"should,omitempty"`
MustNot Query `json:"must_not,omitempty"`
BoostVal *Boost `json:"boost,omitempty"`
Must Query `json:"must,omitempty"`
Should Query `json:"should,omitempty"`
MustNot Query `json:"must_not,omitempty"`
BoostVal *Boost `json:"boost,omitempty"`
queryStringMode bool
}
// NewBooleanQuery creates a compound Query composed
@ -55,6 +56,15 @@ func NewBooleanQuery(must []Query, should []Query, mustNot []Query) *BooleanQuer
return &rv
}
func NewBooleanQueryForQueryString(must []Query, should []Query, mustNot []Query) *BooleanQuery {
rv := NewBooleanQuery(nil, nil, nil)
rv.queryStringMode = true
rv.AddMust(must...)
rv.AddShould(should...)
rv.AddMustNot(mustNot...)
return rv
}
// SetMinShould requires that at least minShould of the
// should Queries must be satisfied.
func (q *BooleanQuery) SetMinShould(minShould float64) {
@ -63,7 +73,9 @@ func (q *BooleanQuery) SetMinShould(minShould float64) {
func (q *BooleanQuery) AddMust(m ...Query) {
if q.Must == nil {
q.Must = NewConjunctionQuery([]Query{})
tmp := NewConjunctionQuery([]Query{})
tmp.queryStringMode = q.queryStringMode
q.Must = tmp
}
for _, mq := range m {
q.Must.(*ConjunctionQuery).AddQuery(mq)
@ -72,7 +84,9 @@ func (q *BooleanQuery) AddMust(m ...Query) {
func (q *BooleanQuery) AddShould(m ...Query) {
if q.Should == nil {
q.Should = NewDisjunctionQuery([]Query{})
tmp := NewDisjunctionQuery([]Query{})
tmp.queryStringMode = q.queryStringMode
q.Should = tmp
}
for _, mq := range m {
q.Should.(*DisjunctionQuery).AddQuery(mq)
@ -81,7 +95,9 @@ func (q *BooleanQuery) AddShould(m ...Query) {
func (q *BooleanQuery) AddMustNot(m ...Query) {
if q.MustNot == nil {
q.MustNot = NewDisjunctionQuery([]Query{})
tmp := NewDisjunctionQuery([]Query{})
tmp.queryStringMode = q.queryStringMode
q.MustNot = tmp
}
for _, mq := range m {
q.MustNot.(*DisjunctionQuery).AddQuery(mq)
@ -105,8 +121,9 @@ func (q *BooleanQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, opt
if err != nil {
return nil, err
}
if q.Must == nil && q.Should == nil {
q.Must = NewMatchAllQuery()
// if must not is MatchNone, reset it to nil
if _, ok := mustNotSearcher.(*searcher.MatchNoneSearcher); ok {
mustNotSearcher = nil
}
}
@ -116,6 +133,10 @@ func (q *BooleanQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, opt
if err != nil {
return nil, err
}
// if must searcher is MatchNone, reset it to nil
if _, ok := mustSearcher.(*searcher.MatchNoneSearcher); ok {
mustSearcher = nil
}
}
var shouldSearcher search.Searcher
@ -124,8 +145,26 @@ func (q *BooleanQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, opt
if err != nil {
return nil, err
}
// if should searcher is MatchNone, reset it to nil
if _, ok := shouldSearcher.(*searcher.MatchNoneSearcher); ok {
shouldSearcher = nil
}
}
// if all 3 are nil, return MatchNone
if mustSearcher == nil && shouldSearcher == nil && mustNotSearcher == nil {
return searcher.NewMatchNoneSearcher(i)
}
// if only mustNotSearcher, start with MatchAll
if mustSearcher == nil && shouldSearcher == nil && mustNotSearcher != nil {
mustSearcher, err = searcher.NewMatchAllSearcher(i, 1.0, options)
if err != nil {
return nil, err
}
}
// optimization, if only should searcher, just return it instead
if mustSearcher == nil && shouldSearcher != nil && mustNotSearcher == nil {
return shouldSearcher, nil
}

View File

@ -24,8 +24,9 @@ import (
)
type ConjunctionQuery struct {
Conjuncts []Query `json:"conjuncts"`
BoostVal *Boost `json:"boost,omitempty"`
Conjuncts []Query `json:"conjuncts"`
BoostVal *Boost `json:"boost,omitempty"`
queryStringMode bool
}
// NewConjunctionQuery creates a new compound Query.
@ -52,10 +53,9 @@ func (q *ConjunctionQuery) AddQuery(aq ...Query) {
}
func (q *ConjunctionQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {
ss := make([]search.Searcher, len(q.Conjuncts))
for in, conjunct := range q.Conjuncts {
var err error
ss[in], err = conjunct.Searcher(i, m, options)
ss := make([]search.Searcher, 0, len(q.Conjuncts))
for _, conjunct := range q.Conjuncts {
sr, err := conjunct.Searcher(i, m, options)
if err != nil {
for _, searcher := range ss {
if searcher != nil {
@ -64,6 +64,14 @@ func (q *ConjunctionQuery) Searcher(i index.IndexReader, m mapping.IndexMapping,
}
return nil, err
}
if _, ok := sr.(*searcher.MatchNoneSearcher); ok && q.queryStringMode {
// in query string mode, skip match none
continue
}
ss = append(ss, sr)
}
if len(ss) < 1 {
return searcher.NewMatchNoneSearcher(i)
}
return searcher.NewConjunctionSearcher(i, ss, options)
}

View File

@ -25,9 +25,10 @@ import (
)
type DisjunctionQuery struct {
Disjuncts []Query `json:"disjuncts"`
BoostVal *Boost `json:"boost,omitempty"`
Min float64 `json:"min"`
Disjuncts []Query `json:"disjuncts"`
BoostVal *Boost `json:"boost,omitempty"`
Min float64 `json:"min"`
queryStringMode bool
}
// NewDisjunctionQuery creates a new compound Query.
@ -58,10 +59,9 @@ func (q *DisjunctionQuery) SetMin(m float64) {
}
func (q *DisjunctionQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {
ss := make([]search.Searcher, len(q.Disjuncts))
for in, disjunct := range q.Disjuncts {
var err error
ss[in], err = disjunct.Searcher(i, m, options)
ss := make([]search.Searcher, 0, len(q.Disjuncts))
for _, disjunct := range q.Disjuncts {
sr, err := disjunct.Searcher(i, m, options)
if err != nil {
for _, searcher := range ss {
if searcher != nil {
@ -70,6 +70,14 @@ func (q *DisjunctionQuery) Searcher(i index.IndexReader, m mapping.IndexMapping,
}
return nil, err
}
if _, ok := sr.(*searcher.MatchNoneSearcher); ok && q.queryStringMode {
// in query string mode, skip match none
continue
}
ss = append(ss, sr)
}
if len(ss) < 1 {
return searcher.NewMatchNoneSearcher(i)
}
return searcher.NewDisjunctionSearcher(i, ss, q.Min, options)
}

View File

@ -31,6 +31,9 @@ var debugParser bool
var debugLexer bool
func parseQuerySyntax(query string) (rq Query, err error) {
if query == "" {
return NewMatchNoneQuery(), nil
}
lex := newLexerWrapper(newQueryStringLex(strings.NewReader(query)))
doParse(lex)
@ -66,7 +69,7 @@ type lexerWrapper struct {
func newLexerWrapper(lex yyLexer) *lexerWrapper {
return &lexerWrapper{
lex: lex,
query: NewBooleanQuery(nil, nil, nil),
query: NewBooleanQueryForQueryString(nil, nil, nil),
}
}

View File

@ -39,7 +39,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "test",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
NewMatchQuery("test"),
@ -49,7 +49,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `"test phrase 1"`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
NewMatchPhraseQuery("test phrase 1"),
@ -59,7 +59,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "field:test",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -74,7 +74,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "field:t-est",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -89,7 +89,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "field:t+est",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -104,7 +104,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "field:t>est",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -119,7 +119,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "field:t<est",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -134,7 +134,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "field:t=est",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -148,7 +148,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "+field1:test1",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
[]Query{
func() Query {
q := NewMatchQuery("test1")
@ -162,7 +162,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "-field2:test2",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
nil,
[]Query{
@ -176,7 +176,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `field3:"test phrase 2"`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -190,7 +190,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `+field4:"test phrase 1"`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
[]Query{
func() Query {
q := NewMatchPhraseQuery("test phrase 1")
@ -204,7 +204,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `-field5:"test phrase 2"`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
nil,
[]Query{
@ -218,7 +218,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `+field6:test3 -field7:test4 field8:test5`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
[]Query{
func() Query {
q := NewMatchQuery("test3")
@ -244,7 +244,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "test^3",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -258,7 +258,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "test^3 other^6",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -277,7 +277,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "33",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
NewMatchQuery("33"),
@ -287,7 +287,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "field:33",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -301,7 +301,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "cat-dog",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
NewMatchQuery("cat-dog"),
@ -311,7 +311,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "watex~",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -325,7 +325,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "watex~2",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -339,7 +339,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "watex~ 2",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -354,7 +354,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "field:watex~",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -369,7 +369,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: "field:watex~2",
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -384,7 +384,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `field:555c3bb06f7a127cda000005`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -398,7 +398,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `field:>5`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -412,7 +412,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `field:>=5`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -426,7 +426,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `field:<5`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -440,7 +440,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `field:<=5`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -454,7 +454,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `field:>"2006-01-02T15:04:05Z"`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -468,7 +468,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `field:>="2006-01-02T15:04:05Z"`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -482,7 +482,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `field:<"2006-01-02T15:04:05Z"`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -496,7 +496,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `field:<="2006-01-02T15:04:05Z"`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -510,7 +510,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `/mar.*ty/`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
NewRegexpQuery("mar.*ty"),
@ -520,7 +520,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `name:/mar.*ty/`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -534,7 +534,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `mart*`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
NewWildcardQuery("mart*"),
@ -544,7 +544,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `name:mart*`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -562,7 +562,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `name\:marty`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
NewMatchQuery("name:marty"),
@ -573,7 +573,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `name:marty\:couchbase`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -588,7 +588,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `marty\ couchbase`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
NewMatchQuery("marty couchbase"),
@ -599,7 +599,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `\+marty`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
NewMatchQuery("+marty"),
@ -610,7 +610,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `\-marty`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
NewMatchQuery("-marty"),
@ -621,7 +621,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `"what does \"quote\" mean"`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
NewMatchPhraseQuery(`what does "quote" mean`),
@ -632,7 +632,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `can\ i\ escap\e`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
NewMatchQuery(`can i escap\e`),
@ -643,7 +643,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: ` what`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
NewMatchQuery(`what`),
@ -654,7 +654,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `term^`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
func() Query {
@ -670,7 +670,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `3.0\:`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
NewMatchQuery(`3.0:`),
@ -680,7 +680,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
{
input: `3.0\a`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
result: NewBooleanQueryForQueryString(
nil,
[]Query{
NewMatchQuery(`3.0\a`),
@ -702,7 +702,6 @@ func TestQuerySyntaxParserValid(t *testing.T) {
}
if !reflect.DeepEqual(q, test.result) {
t.Errorf("Expected %#v, got %#v: for %s", test.result, q, test.input)
t.Errorf("Expected %#v, got %#v: for %s", test.result.(*BooleanQuery).Should.(*DisjunctionQuery).Disjuncts[0], q.(*BooleanQuery).Should.(*DisjunctionQuery).Disjuncts[0], test.input)
}
}
}