gibheer
/
bleve
0
0
Fork 0
Code

simplify date parsing in queries, add date to query string 

parsing of date ranges in queries no longer consults the
index mapping.  it was deteremined that this wasn't very useful
and led to overly complicated query syntax/behavior.

instead, applications get set the datetime parser used for
date range queries with the top-level config QueryDateTimeParser

also, we now support querying date ranges in the query string,
the syntax is:

field:>"date"

>,>=,<,<= operators are supported
the date must be surrounded by quotes
and must parse in the configured date format
This commit is contained in:
Marty Schoch 2016-04-22 17:12:10 -04:00
parent d0c6dbc9cf
commit 8f8bb91439
10 changed files with 203 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
config.go
View File

@ -15,6 +15,7 @@ import (
"log"
"time"
"github.com/blevesearch/bleve/analysis/datetime_parsers/datetime_optional"
"github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store/boltdb"
"github.com/blevesearch/bleve/index/upside_down"
@ -31,6 +32,7 @@ type configuration struct {
DefaultHighlighter string
DefaultKVStore string
DefaultIndexType string
QueryDateTimeParser string
SlowSearchLogThreshold time.Duration
analysisQueue *index.AnalysisQueue
}
@ -64,6 +66,9 @@ func init() {
// default index
Config.DefaultIndexType = upside_down.Name
// default query date time parser
Config.QueryDateTimeParser = datetime_optional.Name
bootDuration := time.Since(bootStart)
bleveExpVar.Add("bootDuration", int64(bootDuration))
indexStats = NewIndexStats()

2
query.go
View File

@ -271,7 +271,7 @@ func expandQuery(m *IndexMapping, query Query) (Query, error) {
switch query.(type) {
case *queryStringQuery:
q := query.(*queryStringQuery)
parsed, err := parseQuerySyntax(q.Query, m)
parsed, err := parseQuerySyntax(q.Query)
if err != nil {
return nil, fmt.Errorf("could not parse '%s': %s", q.Query, err)
}

6
query_conjunction.go
View File

@ -58,6 +58,12 @@ func (q *conjunctionQuery) Searcher(i index.IndexReader, m *IndexMapping, explai
}
func (q *conjunctionQuery) Validate() error {
for _, q := range q.Conjuncts {
err := q.Validate()
if err != nil {
return err
}
}
return nil
}

32
query_date_range.go
View File

@ -26,7 +26,6 @@ type dateRangeQuery struct {
InclusiveEnd *bool `json:"inclusive_end,omitempty"`
FieldVal string `json:"field,omitempty"`
BoostVal float64 `json:"boost,omitempty"`
DateTimeParser *string `json:"datetime_parser,omitempty"`
}
// NewDateRangeQuery creates a new Query for ranges
@ -72,15 +71,9 @@ func (q *dateRangeQuery) SetField(f string) Query {
func (q *dateRangeQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) {
dateTimeParserName := ""
if q.DateTimeParser != nil {
dateTimeParserName = *q.DateTimeParser
} else {
dateTimeParserName = m.datetimeParserNameForPath(q.FieldVal)
}
dateTimeParser := m.dateTimeParserNamed(dateTimeParserName)
if dateTimeParser == nil {
return nil, fmt.Errorf("no datetime parser named '%s' registered", *q.DateTimeParser)
min, max, err := q.parseEndpoints()
if err != nil {
return nil, err
}
field := q.FieldVal
@ -88,30 +81,43 @@ func (q *dateRangeQuery) Searcher(i index.IndexReader, m *IndexMapping, explain
field = m.DefaultField
}
return searchers.NewNumericRangeSearcher(i, min, max, q.InclusiveStart, q.InclusiveEnd, field, q.BoostVal, explain)
}
func (q *dateRangeQuery) parseEndpoints() (*float64, *float64, error) {
dateTimeParser, err := Config.Cache.DateTimeParserNamed(Config.QueryDateTimeParser)
if err != nil {
return nil, nil, err
}
// now parse the endpoints
min := math.Inf(-1)
max := math.Inf(1)
if q.Start != nil && *q.Start != "" {
startTime, err := dateTimeParser.ParseDateTime(*q.Start)
if err != nil {
return nil, err
return nil, nil, err
}
min = numeric_util.Int64ToFloat64(startTime.UnixNano())
}
if q.End != nil && *q.End != "" {
endTime, err := dateTimeParser.ParseDateTime(*q.End)
if err != nil {
return nil, err
return nil, nil, err
}
max = numeric_util.Int64ToFloat64(endTime.UnixNano())
}
return searchers.NewNumericRangeSearcher(i, &min, &max, q.InclusiveStart, q.InclusiveEnd, field, q.BoostVal, explain)
return &min, &max, nil
}
func (q *dateRangeQuery) Validate() error {
if q.Start == nil && q.Start == q.End {
return fmt.Errorf("must specify start or end")
}
_, _, err := q.parseEndpoints()
if err != nil {
return err
}
return nil
}

6
query_disjunction.go
View File

@ -81,6 +81,12 @@ func (q *disjunctionQuery) Validate() error {
if int(q.MinVal) > len(q.Disjuncts) {
return ErrorDisjunctionFewerThanMinClauses
}
for _, q := range q.Disjuncts {
err := q.Validate()
if err != nil {
return err
}
}
return nil
}

8
query_string.go
View File

@ -39,7 +39,7 @@ func (q *queryStringQuery) SetBoost(b float64) Query {
}
func (q *queryStringQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) {
newQuery, err := parseQuerySyntax(q.Query, m)
newQuery, err := parseQuerySyntax(q.Query)
if err != nil {
return nil, err
}
@ -47,7 +47,11 @@ func (q *queryStringQuery) Searcher(i index.IndexReader, m *IndexMapping, explai
}
func (q *queryStringQuery) Validate() error {
return nil
newQuery, err := parseQuerySyntax(q.Query)
if err != nil {
return err
}
return newQuery.Validate()
}
func (q *queryStringQuery) Field() string {

40
query_string.y
View File

@ -232,6 +232,46 @@ tSTRING tCOLON tLESS tEQUAL tNUMBER {
logDebugGrammar("FIELD - LESS THAN OR EQUAL %f", max)
q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive).SetField(field)
$$ = q
}
|
tSTRING tCOLON tGREATER tPHRASE {
field := $1
minInclusive := false
phrase := $4
logDebugGrammar("FIELD - GREATER THAN DATE %s", phrase)
q := NewDateRangeInclusiveQuery(&phrase, nil, &minInclusive, nil).SetField(field)
$$ = q
}
|
tSTRING tCOLON tGREATER tEQUAL tPHRASE {
field := $1
minInclusive := true
phrase := $5
logDebugGrammar("FIELD - GREATER THAN OR EQUAL DATE %s", phrase)
q := NewDateRangeInclusiveQuery(&phrase, nil, &minInclusive, nil).SetField(field)
$$ = q
}
|
tSTRING tCOLON tLESS tPHRASE {
field := $1
maxInclusive := false
phrase := $4
logDebugGrammar("FIELD - LESS THAN DATE %s", phrase)
q := NewDateRangeInclusiveQuery(nil, &phrase, nil, &maxInclusive).SetField(field)
$$ = q
}
|
tSTRING tCOLON tLESS tEQUAL tPHRASE {
field := $1
maxInclusive := true
phrase := $5
logDebugGrammar("FIELD - LESS THAN OR EQUAL DATE %s", phrase)
q := NewDateRangeInclusiveQuery(nil, &phrase, nil, &maxInclusive).SetField(field)
$$ = q
};
searchBoost:

99
query_string.y.go
View File

@ -74,57 +74,62 @@ var yyExca = [...]int{
-2, 5,
}
const yyNprod = 30
const yyNprod = 34
const yyPrivate = 57344
var yyTokenNames []string
var yyStates []string
const yyLast = 36
const yyLast = 40
var yyAct = [...]int{
22, 26, 36, 10, 14, 29, 30, 35, 25, 27,
28, 13, 19, 33, 23, 24, 34, 11, 12, 31,
18, 20, 32, 21, 17, 6, 7, 2, 3, 1,
16, 8, 5, 4, 15, 9,
22, 26, 21, 10, 14, 29, 30, 17, 25, 27,
28, 13, 19, 3, 23, 24, 36, 11, 12, 1,
18, 20, 33, 34, 40, 16, 35, 38, 5, 31,
4, 39, 32, 2, 37, 6, 7, 8, 15, 9,
}
var yyPact = [...]int{
19, -1000, -1000, 19, -1, -1000, -1000, -1000, -1000, 15,
4, -1000, -1000, -1000, -1000, -1000, -1000, 11, -1000, -4,
-1000, -1000, -11, -1000, -1000, -1000, -1000, 7, 1, -1000,
-1000, -1000, -5, -1000, -10, -1000, -1000,
29, -1000, -1000, 29, -1, -1000, -1000, -1000, -1000, -2,
4, -1000, -1000, -1000, -1000, -1000, -1000, -10, -1000, -4,
-1000, -1000, -11, -1000, -1000, -1000, -1000, 17, 11, -1000,
-1000, -1000, 22, -1000, -1000, 19, -1000, -1000, -1000, -1000,
-1000,
}
var yyPgo = [...]int{
0, 35, 34, 33, 32, 30, 29, 27, 28,
0, 39, 38, 30, 28, 25, 19, 33, 13,
}
var yyR1 = [...]int{
0, 6, 7, 7, 8, 3, 3, 4, 4, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 5, 2, 2,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 5, 2, 2,
}
var yyR2 = [...]int{
0, 1, 2, 1, 3, 0, 1, 1, 1, 1,
1, 1, 2, 4, 2, 4, 3, 3, 1, 1,
3, 3, 3, 4, 5, 4, 5, 2, 0, 1,
3, 3, 3, 4, 5, 4, 5, 4, 5, 4,
5, 2, 0, 1,
}
var yyChk = [...]int{
-1000, -6, -7, -8, -3, -4, 6, 7, -7, -1,
4, 18, 19, 12, 5, -2, -5, 9, 16, 8,
17, 12, 4, 18, 19, 12, 5, 13, 14, 16,
17, 12, 15, 12, 15, 12, 12,
17, 12, 15, 5, 12, 15, 5, 12, 5, 12,
5,
}
var yyDef = [...]int{
5, -2, 1, -2, 0, 6, 7, 8, 2, 28,
9, 10, 11, 18, 19, 4, 29, 0, 12, 0,
14, 27, 20, 16, 17, 21, 22, 0, 0, 13,
15, 23, 0, 25, 0, 24, 26,
5, -2, 1, -2, 0, 6, 7, 8, 2, 32,
9, 10, 11, 18, 19, 4, 33, 0, 12, 0,
14, 31, 20, 16, 17, 21, 22, 0, 0, 13,
15, 23, 0, 27, 25, 0, 29, 24, 28, 26,
30,
}
var yyTok1 = [...]int{
@ -723,22 +728,70 @@ yydefault:
yyVAL.q = q
}
case 27:
yyDollar = yyS[yypt-4 : yypt+1]
//line query_string.y:237
{
field := yyDollar[1].s
minInclusive := false
phrase := yyDollar[4].s
logDebugGrammar("FIELD - GREATER THAN DATE %s", phrase)
q := NewDateRangeInclusiveQuery(&phrase, nil, &minInclusive, nil).SetField(field)
yyVAL.q = q
}
case 28:
yyDollar = yyS[yypt-5 : yypt+1]
//line query_string.y:247
{
field := yyDollar[1].s
minInclusive := true
phrase := yyDollar[5].s
logDebugGrammar("FIELD - GREATER THAN OR EQUAL DATE %s", phrase)
q := NewDateRangeInclusiveQuery(&phrase, nil, &minInclusive, nil).SetField(field)
yyVAL.q = q
}
case 29:
yyDollar = yyS[yypt-4 : yypt+1]
//line query_string.y:257
{
field := yyDollar[1].s
maxInclusive := false
phrase := yyDollar[4].s
logDebugGrammar("FIELD - LESS THAN DATE %s", phrase)
q := NewDateRangeInclusiveQuery(nil, &phrase, nil, &maxInclusive).SetField(field)
yyVAL.q = q
}
case 30:
yyDollar = yyS[yypt-5 : yypt+1]
//line query_string.y:267
{
field := yyDollar[1].s
maxInclusive := true
phrase := yyDollar[5].s
logDebugGrammar("FIELD - LESS THAN OR EQUAL DATE %s", phrase)
q := NewDateRangeInclusiveQuery(nil, &phrase, nil, &maxInclusive).SetField(field)
yyVAL.q = q
}
case 31:
yyDollar = yyS[yypt-2 : yypt+1]
//line query_string.y:238
//line query_string.y:278
{
boost, _ := strconv.ParseFloat(yyDollar[2].s, 64)
yyVAL.f = boost
logDebugGrammar("BOOST %f", boost)
}
case 28:
case 32:
yyDollar = yyS[yypt-0 : yypt+1]
//line query_string.y:245
//line query_string.y:285
{
yyVAL.f = 1.0
}
case 29:
case 33:
yyDollar = yyS[yypt-1 : yypt+1]
//line query_string.y:249
//line query_string.y:289
{
}

2
query_string_parser.go
View File

@ -25,7 +25,7 @@ import (
var debugParser bool
var debugLexer bool
func parseQuerySyntax(query string, mapping *IndexMapping) (rq Query, err error) {
func parseQuerySyntax(query string) (rq Query, err error) {
lex := newLexerWrapper(newLexer(strings.NewReader(query)))
doParse(lex)

45
query_string_parser_test.go
View File

@ -18,6 +18,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
fivePointOh := 5.0
theTruth := true
theFalsehood := false
theDate := "2006-01-02T15:04:05Z07:00"
tests := []struct {
input string
result Query
@ -324,6 +325,46 @@ func TestQuerySyntaxParserValid(t *testing.T) {
},
nil),
},
{
input: `field:>"2006-01-02T15:04:05Z07:00"`,
mapping: NewIndexMapping(),
result: NewBooleanQuery(
nil,
[]Query{
NewDateRangeInclusiveQuery(&theDate, nil, &theFalsehood, nil).SetField("field"),
},
nil),
},
{
input: `field:>="2006-01-02T15:04:05Z07:00"`,
mapping: NewIndexMapping(),
result: NewBooleanQuery(
nil,
[]Query{
NewDateRangeInclusiveQuery(&theDate, nil, &theTruth, nil).SetField("field"),
},
nil),
},
{
input: `field:<"2006-01-02T15:04:05Z07:00"`,
mapping: NewIndexMapping(),
result: NewBooleanQuery(
nil,
[]Query{
NewDateRangeInclusiveQuery(nil, &theDate, nil, &theFalsehood).SetField("field"),
},
nil),
},
{
input: `field:<="2006-01-02T15:04:05Z07:00"`,
mapping: NewIndexMapping(),
result: NewBooleanQuery(
nil,
[]Query{
NewDateRangeInclusiveQuery(nil, &theDate, nil, &theTruth).SetField("field"),
},
nil),
},
}
// turn on lexer debugging
@ -332,7 +373,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
for _, test := range tests {
q, err := parseQuerySyntax(test.input, test.mapping)
q, err := parseQuerySyntax(test.input)
if err != nil {
t.Error(err)
}
@ -365,7 +406,7 @@ func TestQuerySyntaxParserInvalid(t *testing.T) {
// logger = log.New(os.Stderr, "bleve", log.LstdFlags)
for _, test := range tests {
_, err := parseQuerySyntax(test.input, NewIndexMapping())
_, err := parseQuerySyntax(test.input)
if err == nil {
t.Errorf("expected error, got nil for `%s`", test.input)
}