From f6563ed9f5c1777112a804ab9ae7e38f462570f7 Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Fri, 24 Feb 2017 09:23:00 -0500 Subject: [PATCH 1/3] switch from go tool yacc to goyacc as of Go 1.8 does not imply need for Go 1.8 to use, just for developers to regenerate the query parser --- search/query/query_string_parser.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/search/query/query_string_parser.go b/search/query/query_string_parser.go index 423eba71..3fb7731b 100644 --- a/search/query/query_string_parser.go +++ b/search/query/query_string_parser.go @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:generate go tool yacc -o query_string.y.go query_string.y +// as of Go 1.8 this requires the goyacc external tool +// available from golang.org/x/tools/cmd/goyacc + +//go:generate goyacc -o query_string.y.go query_string.y //go:generate sed -i.tmp -e 1d query_string.y.go //go:generate rm query_string.y.go.tmp From 23bf9866322750164bc7e29976507d0d3b412266 Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Fri, 24 Feb 2017 13:41:59 -0500 Subject: [PATCH 2/3] allow for exact numeric matches with field:val syntax previously, the only way to get numeric matching was with the range operators :> :>= :< :<= now, when we encounter field:val if the val can be parsed as a number, then we do a disjunction search, which includes searching for val as a text term and as an exact numeric value --- search/query/query_string.y | 17 +++++++-- search/query/query_string.y.go | 45 +++++++++++++++--------- search/query/query_string_parser_test.go | 40 ++++++++++++++++++--- 3 files changed, 77 insertions(+), 25 deletions(-) diff --git a/search/query/query_string.y b/search/query/query_string.y index d703be28..e439db4e 100644 --- a/search/query/query_string.y +++ b/search/query/query_string.y @@ -127,7 +127,12 @@ tSTRING tCOLON tSTRING tTILDE { tNUMBER { str := $1 logDebugGrammar("STRING - %s", str) - q := NewMatchQuery(str) + q1 := NewMatchQuery(str) + val, _ := strconv.ParseFloat($1, 64) + inclusive := true + q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) + q := NewDisjunctionQuery([]Query{q1,q2}) + q.queryStringMode = true $$ = q } | @@ -158,8 +163,14 @@ tSTRING tCOLON tNUMBER { field := $1 str := $3 logDebugGrammar("FIELD - %s STRING - %s", field, str) - q := NewMatchQuery(str) - q.SetField(field) + q1 := NewMatchQuery(str) + q1.SetField(field) + val, _ := strconv.ParseFloat($3, 64) + inclusive := true + q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) + q2.SetField(field) + q := NewDisjunctionQuery([]Query{q1,q2}) + q.queryStringMode = true $$ = q } | diff --git a/search/query/query_string.y.go b/search/query/query_string.y.go index 74ea00fc..720350e1 100644 --- a/search/query/query_string.y.go +++ b/search/query/query_string.y.go @@ -581,12 +581,17 @@ yydefault: { str := yyDollar[1].s logDebugGrammar("STRING - %s", str) - q := NewMatchQuery(str) + q1 := NewMatchQuery(str) + val, _ := strconv.ParseFloat(yyDollar[1].s, 64) + inclusive := true + q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) + q := NewDisjunctionQuery([]Query{q1, q2}) + q.queryStringMode = true yyVAL.q = q } case 12: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:134 + //line query_string.y:139 { phrase := yyDollar[1].s logDebugGrammar("PHRASE - %s", phrase) @@ -595,7 +600,7 @@ yydefault: } case 13: yyDollar = yyS[yypt-3 : yypt+1] - //line query_string.y:141 + //line query_string.y:146 { field := yyDollar[1].s str := yyDollar[3].s @@ -613,18 +618,24 @@ yydefault: } case 14: yyDollar = yyS[yypt-3 : yypt+1] - //line query_string.y:157 + //line query_string.y:162 { field := yyDollar[1].s str := yyDollar[3].s logDebugGrammar("FIELD - %s STRING - %s", field, str) - q := NewMatchQuery(str) - q.SetField(field) + q1 := NewMatchQuery(str) + q1.SetField(field) + val, _ := strconv.ParseFloat(yyDollar[3].s, 64) + inclusive := true + q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) + q2.SetField(field) + q := NewDisjunctionQuery([]Query{q1, q2}) + q.queryStringMode = true yyVAL.q = q } case 15: yyDollar = yyS[yypt-3 : yypt+1] - //line query_string.y:166 + //line query_string.y:177 { field := yyDollar[1].s phrase := yyDollar[3].s @@ -635,7 +646,7 @@ yydefault: } case 16: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:175 + //line query_string.y:186 { field := yyDollar[1].s min, _ := strconv.ParseFloat(yyDollar[4].s, 64) @@ -647,7 +658,7 @@ yydefault: } case 17: yyDollar = yyS[yypt-5 : yypt+1] - //line query_string.y:185 + //line query_string.y:196 { field := yyDollar[1].s min, _ := strconv.ParseFloat(yyDollar[5].s, 64) @@ -659,7 +670,7 @@ yydefault: } case 18: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:195 + //line query_string.y:206 { field := yyDollar[1].s max, _ := strconv.ParseFloat(yyDollar[4].s, 64) @@ -671,7 +682,7 @@ yydefault: } case 19: yyDollar = yyS[yypt-5 : yypt+1] - //line query_string.y:205 + //line query_string.y:216 { field := yyDollar[1].s max, _ := strconv.ParseFloat(yyDollar[5].s, 64) @@ -683,7 +694,7 @@ yydefault: } case 20: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:215 + //line query_string.y:226 { field := yyDollar[1].s minInclusive := false @@ -700,7 +711,7 @@ yydefault: } case 21: yyDollar = yyS[yypt-5 : yypt+1] - //line query_string.y:230 + //line query_string.y:241 { field := yyDollar[1].s minInclusive := true @@ -717,7 +728,7 @@ yydefault: } case 22: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:245 + //line query_string.y:256 { field := yyDollar[1].s maxInclusive := false @@ -734,7 +745,7 @@ yydefault: } case 23: yyDollar = yyS[yypt-5 : yypt+1] - //line query_string.y:260 + //line query_string.y:271 { field := yyDollar[1].s maxInclusive := true @@ -751,13 +762,13 @@ yydefault: } case 24: yyDollar = yyS[yypt-0 : yypt+1] - //line query_string.y:276 + //line query_string.y:287 { yyVAL.pf = nil } case 25: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:280 + //line query_string.y:291 { yyVAL.pf = nil boost, err := strconv.ParseFloat(yyDollar[1].s, 64) diff --git a/search/query/query_string_parser_test.go b/search/query/query_string_parser_test.go index 8eb13602..e65738c0 100644 --- a/search/query/query_string_parser_test.go +++ b/search/query/query_string_parser_test.go @@ -24,6 +24,8 @@ import ( ) func TestQuerySyntaxParserValid(t *testing.T) { + thirtyThreePointOh := 33.0 + twoPointOh := 2.0 fivePointOh := 5.0 theTruth := true theFalsehood := false @@ -280,7 +282,15 @@ func TestQuerySyntaxParserValid(t *testing.T) { result: NewBooleanQueryForQueryString( nil, []Query{ - NewMatchQuery("33"), + func() Query { + qo := NewDisjunctionQuery( + []Query{ + NewMatchQuery("33"), + NewNumericRangeInclusiveQuery(&thirtyThreePointOh, &thirtyThreePointOh, &theTruth, &theTruth), + }) + qo.queryStringMode = true + return qo + }(), }, nil), }, @@ -291,9 +301,21 @@ func TestQuerySyntaxParserValid(t *testing.T) { nil, []Query{ func() Query { - q := NewMatchQuery("33") - q.SetField("field") - return q + qo := NewDisjunctionQuery( + []Query{ + func() Query { + q := NewMatchQuery("33") + q.SetField("field") + return q + }(), + func() Query { + q := NewNumericRangeInclusiveQuery(&thirtyThreePointOh, &thirtyThreePointOh, &theTruth, &theTruth) + q.SetField("field") + return q + }(), + }) + qo.queryStringMode = true + return qo }(), }, nil), @@ -347,7 +369,15 @@ func TestQuerySyntaxParserValid(t *testing.T) { q.SetFuzziness(1) return q }(), - NewMatchQuery("2"), + func() Query { + qo := NewDisjunctionQuery( + []Query{ + NewMatchQuery("2"), + NewNumericRangeInclusiveQuery(&twoPointOh, &twoPointOh, &theTruth, &theTruth), + }) + qo.queryStringMode = true + return qo + }(), }, nil), }, From bbab4d39ee54a14b01a0ae082ee9894604d1f981 Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Fri, 24 Feb 2017 16:30:09 -0500 Subject: [PATCH 3/3] improve error checking when parsing numbers --- search/query/query_string.y | 30 +++++++++--- search/query/query_string.y.go | 58 ++++++++++++++++-------- search/query/query_string_parser_test.go | 6 +++ 3 files changed, 68 insertions(+), 26 deletions(-) diff --git a/search/query/query_string.y b/search/query/query_string.y index e439db4e..a2f748f3 100644 --- a/search/query/query_string.y +++ b/search/query/query_string.y @@ -128,7 +128,10 @@ tNUMBER { str := $1 logDebugGrammar("STRING - %s", str) q1 := NewMatchQuery(str) - val, _ := strconv.ParseFloat($1, 64) + val, err := strconv.ParseFloat($1, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } inclusive := true q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) q := NewDisjunctionQuery([]Query{q1,q2}) @@ -165,7 +168,10 @@ tSTRING tCOLON tNUMBER { logDebugGrammar("FIELD - %s STRING - %s", field, str) q1 := NewMatchQuery(str) q1.SetField(field) - val, _ := strconv.ParseFloat($3, 64) + val, err := strconv.ParseFloat($3, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } inclusive := true q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) q2.SetField(field) @@ -185,7 +191,10 @@ tSTRING tCOLON tPHRASE { | tSTRING tCOLON tGREATER tNUMBER { field := $1 - min, _ := strconv.ParseFloat($4, 64) + min, err := strconv.ParseFloat($4, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } minInclusive := false logDebugGrammar("FIELD - GREATER THAN %f", min) q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil) @@ -195,7 +204,10 @@ tSTRING tCOLON tGREATER tNUMBER { | tSTRING tCOLON tGREATER tEQUAL tNUMBER { field := $1 - min, _ := strconv.ParseFloat($5, 64) + min, err := strconv.ParseFloat($5, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } minInclusive := true logDebugGrammar("FIELD - GREATER THAN OR EQUAL %f", min) q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil) @@ -205,7 +217,10 @@ tSTRING tCOLON tGREATER tEQUAL tNUMBER { | tSTRING tCOLON tLESS tNUMBER { field := $1 - max, _ := strconv.ParseFloat($4, 64) + max, err := strconv.ParseFloat($4, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } maxInclusive := false logDebugGrammar("FIELD - LESS THAN %f", max) q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive) @@ -215,7 +230,10 @@ tSTRING tCOLON tLESS tNUMBER { | tSTRING tCOLON tLESS tEQUAL tNUMBER { field := $1 - max, _ := strconv.ParseFloat($5, 64) + max, err := strconv.ParseFloat($5, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } maxInclusive := true logDebugGrammar("FIELD - LESS THAN OR EQUAL %f", max) q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive) diff --git a/search/query/query_string.y.go b/search/query/query_string.y.go index 720350e1..452ad6e6 100644 --- a/search/query/query_string.y.go +++ b/search/query/query_string.y.go @@ -582,7 +582,10 @@ yydefault: str := yyDollar[1].s logDebugGrammar("STRING - %s", str) q1 := NewMatchQuery(str) - val, _ := strconv.ParseFloat(yyDollar[1].s, 64) + val, err := strconv.ParseFloat(yyDollar[1].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } inclusive := true q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) q := NewDisjunctionQuery([]Query{q1, q2}) @@ -591,7 +594,7 @@ yydefault: } case 12: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:139 + //line query_string.y:142 { phrase := yyDollar[1].s logDebugGrammar("PHRASE - %s", phrase) @@ -600,7 +603,7 @@ yydefault: } case 13: yyDollar = yyS[yypt-3 : yypt+1] - //line query_string.y:146 + //line query_string.y:149 { field := yyDollar[1].s str := yyDollar[3].s @@ -618,14 +621,17 @@ yydefault: } case 14: yyDollar = yyS[yypt-3 : yypt+1] - //line query_string.y:162 + //line query_string.y:165 { field := yyDollar[1].s str := yyDollar[3].s logDebugGrammar("FIELD - %s STRING - %s", field, str) q1 := NewMatchQuery(str) q1.SetField(field) - val, _ := strconv.ParseFloat(yyDollar[3].s, 64) + val, err := strconv.ParseFloat(yyDollar[3].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } inclusive := true q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) q2.SetField(field) @@ -635,7 +641,7 @@ yydefault: } case 15: yyDollar = yyS[yypt-3 : yypt+1] - //line query_string.y:177 + //line query_string.y:183 { field := yyDollar[1].s phrase := yyDollar[3].s @@ -646,10 +652,13 @@ yydefault: } case 16: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:186 + //line query_string.y:192 { field := yyDollar[1].s - min, _ := strconv.ParseFloat(yyDollar[4].s, 64) + min, err := strconv.ParseFloat(yyDollar[4].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } minInclusive := false logDebugGrammar("FIELD - GREATER THAN %f", min) q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil) @@ -658,10 +667,13 @@ yydefault: } case 17: yyDollar = yyS[yypt-5 : yypt+1] - //line query_string.y:196 + //line query_string.y:205 { field := yyDollar[1].s - min, _ := strconv.ParseFloat(yyDollar[5].s, 64) + min, err := strconv.ParseFloat(yyDollar[5].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } minInclusive := true logDebugGrammar("FIELD - GREATER THAN OR EQUAL %f", min) q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil) @@ -670,10 +682,13 @@ yydefault: } case 18: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:206 + //line query_string.y:218 { field := yyDollar[1].s - max, _ := strconv.ParseFloat(yyDollar[4].s, 64) + max, err := strconv.ParseFloat(yyDollar[4].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } maxInclusive := false logDebugGrammar("FIELD - LESS THAN %f", max) q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive) @@ -682,10 +697,13 @@ yydefault: } case 19: yyDollar = yyS[yypt-5 : yypt+1] - //line query_string.y:216 + //line query_string.y:231 { field := yyDollar[1].s - max, _ := strconv.ParseFloat(yyDollar[5].s, 64) + max, err := strconv.ParseFloat(yyDollar[5].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } maxInclusive := true logDebugGrammar("FIELD - LESS THAN OR EQUAL %f", max) q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive) @@ -694,7 +712,7 @@ yydefault: } case 20: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:226 + //line query_string.y:244 { field := yyDollar[1].s minInclusive := false @@ -711,7 +729,7 @@ yydefault: } case 21: yyDollar = yyS[yypt-5 : yypt+1] - //line query_string.y:241 + //line query_string.y:259 { field := yyDollar[1].s minInclusive := true @@ -728,7 +746,7 @@ yydefault: } case 22: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:256 + //line query_string.y:274 { field := yyDollar[1].s maxInclusive := false @@ -745,7 +763,7 @@ yydefault: } case 23: yyDollar = yyS[yypt-5 : yypt+1] - //line query_string.y:271 + //line query_string.y:289 { field := yyDollar[1].s maxInclusive := true @@ -762,13 +780,13 @@ yydefault: } case 24: yyDollar = yyS[yypt-0 : yypt+1] - //line query_string.y:287 + //line query_string.y:305 { yyVAL.pf = nil } case 25: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:291 + //line query_string.y:309 { yyVAL.pf = nil boost, err := strconv.ParseFloat(yyDollar[1].s, 64) diff --git a/search/query/query_string_parser_test.go b/search/query/query_string_parser_test.go index e65738c0..aa5e133f 100644 --- a/search/query/query_string_parser_test.go +++ b/search/query/query_string_parser_test.go @@ -756,6 +756,12 @@ func TestQuerySyntaxParserInvalid(t *testing.T) { {`cat^3\0`}, {`cat~3\:`}, {`cat~3\0`}, + {`99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999`}, + {`field:99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999`}, + {`field:>99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999`}, + {`field:>=99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999`}, + {`field:<99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999`}, + {`field:<=99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999`}, } // turn on lexer debugging