gibheer
/
bleve
0
0
Fork 0
Code

switch DateRangeQuery to use time.Time instead of string 

as we are a Go library is this the much more natural way to
express such queries.

support for strings is still supported through json marshal
and unmarshal, as well as inside query string queries

as before we use the package level QueryDateTimeParser to
deterimine which date time parser to use for parsing

only serializing out to json, we consult a new package
variable: QueryDateTimeFormat

this addresses the longstanding PR #255
This commit is contained in:
Marty Schoch 2016-09-22 21:24:43 -04:00
parent a265218f76
commit ee17941f7f
7 changed files with 176 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
index_test.go
View File

@ -1280,9 +1280,9 @@ func TestDateTimeFieldMappingIssue287(t *testing.T) {
}
// search range across all docs
start := now.Add(-4 * time.Hour).Format(time.RFC3339)
end := now.Format(time.RFC3339)
sreq := NewSearchRequest(NewDateRangeQuery(&start, &end))
start := now.Add(-4 * time.Hour)
end := now
sreq := NewSearchRequest(NewDateRangeQuery(start, end))
sres, err := index.Search(sreq)
if err != nil {
t.Fatal(err)
@ -1292,9 +1292,9 @@ func TestDateTimeFieldMappingIssue287(t *testing.T) {
}
// search range includes only oldest
start = now.Add(-4 * time.Hour).Format(time.RFC3339)
end = now.Add(-121 * time.Minute).Format(time.RFC3339)
sreq = NewSearchRequest(NewDateRangeQuery(&start, &end))
start = now.Add(-4 * time.Hour)
end = now.Add(-121 * time.Minute)
sreq = NewSearchRequest(NewDateRangeQuery(start, end))
sres, err = index.Search(sreq)
if err != nil {
t.Fatal(err)
@ -1307,9 +1307,9 @@ func TestDateTimeFieldMappingIssue287(t *testing.T) {
}
// search range includes only newest
start = now.Add(-61 * time.Minute).Format(time.RFC3339)
end = now.Format(time.RFC3339)
sreq = NewSearchRequest(NewDateRangeQuery(&start, &end))
start = now.Add(-61 * time.Minute)
end = now
sreq = NewSearchRequest(NewDateRangeQuery(start, end))
sres, err = index.Search(sreq)
if err != nil {
t.Fatal(err)

10
query.go
View File

@ -9,7 +9,11 @@
package bleve
import "github.com/blevesearch/bleve/search/query"
import (
"time"
"github.com/blevesearch/bleve/search/query"
)
// NewBoolFieldQuery creates a new Query for boolean fields
func NewBoolFieldQuery(val bool) *query.BoolFieldQuery {
@ -41,7 +45,7 @@ func NewConjunctionQuery(conjuncts ...query.Query) *query.ConjunctionQuery {
// Date strings are parsed using the DateTimeParser configured in the
// top-level config.QueryDateTimeParser
// Either, but not both endpoints can be nil.
func NewDateRangeQuery(start, end *string) *query.DateRangeQuery {
func NewDateRangeQuery(start, end time.Time) *query.DateRangeQuery {
return query.NewDateRangeQuery(start, end)
}
@ -51,7 +55,7 @@ func NewDateRangeQuery(start, end *string) *query.DateRangeQuery {
// top-level config.QueryDateTimeParser
// Either, but not both endpoints can be nil.
// startInclusive and endInclusive control inclusion of the endpoints.
func NewDateRangeInclusiveQuery(start, end *string, startInclusive, endInclusive *bool) *query.DateRangeQuery {
func NewDateRangeInclusiveQuery(start, end time.Time, startInclusive, endInclusive *bool) *query.DateRangeQuery {
return query.NewDateRangeInclusiveQuery(start, end, startInclusive, endInclusive)
}

88
search/query/query_date_range.go
View File

@ -10,8 +10,10 @@
package query
import (
"encoding/json"
"fmt"
"math"
"time"
"github.com/blevesearch/bleve/analysis/datetime_parsers/datetime_optional"
"github.com/blevesearch/bleve/index"
@ -25,15 +27,56 @@ import (
// QueryDateTimeParser controls the default query date time parser
var QueryDateTimeParser = datetime_optional.Name
// QueryDateTimeFormat controls the format when Marshaling to JSON
var QueryDateTimeFormat = time.RFC3339
var cache = registry.NewCache()
type BleveQueryTime struct {
time.Time
}
func QueryTimeFromString(t string) (time.Time, error) {
dateTimeParser, err := cache.DateTimeParserNamed(QueryDateTimeParser)
if err != nil {
return time.Time{}, err
}
rv, err := dateTimeParser.ParseDateTime(t)
if err != nil {
return time.Time{}, err
}
return rv, nil
}
func (t *BleveQueryTime) MarshalJSON() ([]byte, error) {
tt := time.Time(t.Time)
return []byte(tt.Format(QueryDateTimeFormat)), nil
}
func (t *BleveQueryTime) UnmarshalJSON(data []byte) error {
var timeString string
err := json.Unmarshal(data, &timeString)
if err != nil {
return err
}
dateTimeParser, err := cache.DateTimeParserNamed(QueryDateTimeParser)
if err != nil {
return err
}
t.Time, err = dateTimeParser.ParseDateTime(timeString)
if err != nil {
return err
}
return nil
}
type DateRangeQuery struct {
Start *string `json:"start,omitempty"`
End *string `json:"end,omitempty"`
InclusiveStart *bool `json:"inclusive_start,omitempty"`
InclusiveEnd *bool `json:"inclusive_end,omitempty"`
Field string `json:"field,omitempty"`
Boost *Boost `json:"boost,omitempty"`
Start BleveQueryTime `json:"start,omitempty"`
End BleveQueryTime `json:"end,omitempty"`
InclusiveStart *bool `json:"inclusive_start,omitempty"`
InclusiveEnd *bool `json:"inclusive_end,omitempty"`
Field string `json:"field,omitempty"`
Boost *Boost `json:"boost,omitempty"`
}
// NewDateRangeQuery creates a new Query for ranges
@ -41,7 +84,7 @@ type DateRangeQuery struct {
// Date strings are parsed using the DateTimeParser configured in the
// top-level config.QueryDateTimeParser
// Either, but not both endpoints can be nil.
func NewDateRangeQuery(start, end *string) *DateRangeQuery {
func NewDateRangeQuery(start, end time.Time) *DateRangeQuery {
return NewDateRangeInclusiveQuery(start, end, nil, nil)
}
@ -51,10 +94,10 @@ func NewDateRangeQuery(start, end *string) *DateRangeQuery {
// top-level config.QueryDateTimeParser
// Either, but not both endpoints can be nil.
// startInclusive and endInclusive control inclusion of the endpoints.
func NewDateRangeInclusiveQuery(start, end *string, startInclusive, endInclusive *bool) *DateRangeQuery {
func NewDateRangeInclusiveQuery(start, end time.Time, startInclusive, endInclusive *bool) *DateRangeQuery {
return &DateRangeQuery{
Start: start,
End: end,
Start: BleveQueryTime{start},
End: BleveQueryTime{end},
InclusiveStart: startInclusive,
InclusiveEnd: endInclusive,
}
@ -70,7 +113,6 @@ func (q *DateRangeQuery) SetField(f string) {
}
func (q *DateRangeQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
min, max, err := q.parseEndpoints()
if err != nil {
return nil, err
@ -85,34 +127,20 @@ func (q *DateRangeQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, e
}
func (q *DateRangeQuery) parseEndpoints() (*float64, *float64, error) {
dateTimeParser, err := cache.DateTimeParserNamed(QueryDateTimeParser)
if err != nil {
return nil, nil, err
}
// now parse the endpoints
min := math.Inf(-1)
max := math.Inf(1)
if q.Start != nil && *q.Start != "" {
startTime, err := dateTimeParser.ParseDateTime(*q.Start)
if err != nil {
return nil, nil, err
}
min = numeric_util.Int64ToFloat64(startTime.UnixNano())
if !q.Start.IsZero() {
min = numeric_util.Int64ToFloat64(q.Start.UnixNano())
}
if q.End != nil && *q.End != "" {
endTime, err := dateTimeParser.ParseDateTime(*q.End)
if err != nil {
return nil, nil, err
}
max = numeric_util.Int64ToFloat64(endTime.UnixNano())
if !q.End.IsZero() {
max = numeric_util.Int64ToFloat64(q.End.UnixNano())
}
return &min, &max, nil
}
func (q *DateRangeQuery) Validate() error {
if q.Start == nil && q.Start == q.End {
if q.Start.IsZero() && q.End.IsZero() {
return fmt.Errorf("must specify start or end")
}
_, _, err := q.parseEndpoints()

25
search/query/query_string.y
View File

@ -4,6 +4,7 @@ import (
"fmt"
"strconv"
"strings"
"time"
)
func logDebugGrammar(format string, v ...interface{}) {
@ -217,7 +218,11 @@ tSTRING tCOLON tGREATER tPHRASE {
phrase := $4
logDebugGrammar("FIELD - GREATER THAN DATE %s", phrase)
q := NewDateRangeInclusiveQuery(&phrase, nil, &minInclusive, nil)
minTime, err := QueryTimeFromString(phrase)
if err != nil {
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err))
}
q := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil)
q.SetField(field)
$$ = q
}
@ -228,7 +233,11 @@ tSTRING tCOLON tGREATER tEQUAL tPHRASE {
phrase := $5
logDebugGrammar("FIELD - GREATER THAN OR EQUAL DATE %s", phrase)
q := NewDateRangeInclusiveQuery(&phrase, nil, &minInclusive, nil)
minTime, err := QueryTimeFromString(phrase)
if err != nil {
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err))
}
q := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil)
q.SetField(field)
$$ = q
}
@ -239,7 +248,11 @@ tSTRING tCOLON tLESS tPHRASE {
phrase := $4
logDebugGrammar("FIELD - LESS THAN DATE %s", phrase)
q := NewDateRangeInclusiveQuery(nil, &phrase, nil, &maxInclusive)
maxTime, err := QueryTimeFromString(phrase)
if err != nil {
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err))
}
q := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive)
q.SetField(field)
$$ = q
}
@ -250,7 +263,11 @@ tSTRING tCOLON tLESS tEQUAL tPHRASE {
phrase := $5
logDebugGrammar("FIELD - LESS THAN OR EQUAL DATE %s", phrase)
q := NewDateRangeInclusiveQuery(nil, &phrase, nil, &maxInclusive)
maxTime, err := QueryTimeFromString(phrase)
if err != nil {
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err))
}
q := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive)
q.SetField(field)
$$ = q
};

77
search/query/query_string.y.go
View File

@ -7,6 +7,7 @@ import (
"fmt"
"strconv"
"strings"
"time"
)
func logDebugGrammar(format string, v ...interface{}) {
@ -15,7 +16,7 @@ func logDebugGrammar(format string, v ...interface{}) {
}
}
//line query_string.y:16
//line query_string.y:17
type yySymType struct {
yys int
s string
@ -473,25 +474,25 @@ yydefault:
case 1:
yyDollar = yyS[yypt-1 : yypt+1]
//line query_string.y:38
//line query_string.y:39
{
logDebugGrammar("INPUT")
}
case 2:
yyDollar = yyS[yypt-2 : yypt+1]
//line query_string.y:43
//line query_string.y:44
{
logDebugGrammar("SEARCH PARTS")
}
case 3:
yyDollar = yyS[yypt-1 : yypt+1]
//line query_string.y:47
//line query_string.y:48
{
logDebugGrammar("SEARCH PART")
}
case 4:
yyDollar = yyS[yypt-3 : yypt+1]
//line query_string.y:52
//line query_string.y:53
{
query := yyDollar[2].q
if yyDollar[3].pf != nil {
@ -510,27 +511,27 @@ yydefault:
}
case 5:
yyDollar = yyS[yypt-0 : yypt+1]
//line query_string.y:71
//line query_string.y:72
{
yyVAL.n = queryShould
}
case 6:
yyDollar = yyS[yypt-1 : yypt+1]
//line query_string.y:75
//line query_string.y:76
{
logDebugGrammar("PLUS")
yyVAL.n = queryMust
}
case 7:
yyDollar = yyS[yypt-1 : yypt+1]
//line query_string.y:80
//line query_string.y:81
{
logDebugGrammar("MINUS")
yyVAL.n = queryMustNot
}
case 8:
yyDollar = yyS[yypt-1 : yypt+1]
//line query_string.y:86
//line query_string.y:87
{
str := yyDollar[1].s
logDebugGrammar("STRING - %s", str)
@ -546,7 +547,7 @@ yydefault:
}
case 9:
yyDollar = yyS[yypt-2 : yypt+1]
//line query_string.y:100
//line query_string.y:101
{
str := yyDollar[1].s
fuzziness, err := strconv.ParseFloat(yyDollar[2].s, 64)
@ -560,7 +561,7 @@ yydefault:
}
case 10:
yyDollar = yyS[yypt-4 : yypt+1]
//line query_string.y:112
//line query_string.y:113
{
field := yyDollar[1].s
str := yyDollar[3].s
@ -576,7 +577,7 @@ yydefault:
}
case 11:
yyDollar = yyS[yypt-1 : yypt+1]
//line query_string.y:126
//line query_string.y:127
{
str := yyDollar[1].s
logDebugGrammar("STRING - %s", str)
@ -585,7 +586,7 @@ yydefault:
}
case 12:
yyDollar = yyS[yypt-1 : yypt+1]
//line query_string.y:133
//line query_string.y:134
{
phrase := yyDollar[1].s
logDebugGrammar("PHRASE - %s", phrase)
@ -594,7 +595,7 @@ yydefault:
}
case 13:
yyDollar = yyS[yypt-3 : yypt+1]
//line query_string.y:140
//line query_string.y:141
{
field := yyDollar[1].s
str := yyDollar[3].s
@ -612,7 +613,7 @@ yydefault:
}
case 14:
yyDollar = yyS[yypt-3 : yypt+1]
//line query_string.y:156
//line query_string.y:157
{
field := yyDollar[1].s
str := yyDollar[3].s
@ -623,7 +624,7 @@ yydefault:
}
case 15:
yyDollar = yyS[yypt-3 : yypt+1]
//line query_string.y:165
//line query_string.y:166
{
field := yyDollar[1].s
phrase := yyDollar[3].s
@ -634,7 +635,7 @@ yydefault:
}
case 16:
yyDollar = yyS[yypt-4 : yypt+1]
//line query_string.y:174
//line query_string.y:175
{
field := yyDollar[1].s
min, _ := strconv.ParseFloat(yyDollar[4].s, 64)
@ -646,7 +647,7 @@ yydefault:
}
case 17:
yyDollar = yyS[yypt-5 : yypt+1]
//line query_string.y:184
//line query_string.y:185
{
field := yyDollar[1].s
min, _ := strconv.ParseFloat(yyDollar[5].s, 64)
@ -658,7 +659,7 @@ yydefault:
}
case 18:
yyDollar = yyS[yypt-4 : yypt+1]
//line query_string.y:194
//line query_string.y:195
{
field := yyDollar[1].s
max, _ := strconv.ParseFloat(yyDollar[4].s, 64)
@ -670,7 +671,7 @@ yydefault:
}
case 19:
yyDollar = yyS[yypt-5 : yypt+1]
//line query_string.y:204
//line query_string.y:205
{
field := yyDollar[1].s
max, _ := strconv.ParseFloat(yyDollar[5].s, 64)
@ -682,65 +683,81 @@ yydefault:
}
case 20:
yyDollar = yyS[yypt-4 : yypt+1]
//line query_string.y:214
//line query_string.y:215
{
field := yyDollar[1].s
minInclusive := false
phrase := yyDollar[4].s
logDebugGrammar("FIELD - GREATER THAN DATE %s", phrase)
q := NewDateRangeInclusiveQuery(&phrase, nil, &minInclusive, nil)
minTime, err := QueryTimeFromString(phrase)
if err != nil {
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err))
}
q := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil)
q.SetField(field)
yyVAL.q = q
}
case 21:
yyDollar = yyS[yypt-5 : yypt+1]
//line query_string.y:225
//line query_string.y:230
{
field := yyDollar[1].s
minInclusive := true
phrase := yyDollar[5].s
logDebugGrammar("FIELD - GREATER THAN OR EQUAL DATE %s", phrase)
q := NewDateRangeInclusiveQuery(&phrase, nil, &minInclusive, nil)
minTime, err := QueryTimeFromString(phrase)
if err != nil {
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err))
}
q := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil)
q.SetField(field)
yyVAL.q = q
}
case 22:
yyDollar = yyS[yypt-4 : yypt+1]
//line query_string.y:236
//line query_string.y:245
{
field := yyDollar[1].s
maxInclusive := false
phrase := yyDollar[4].s
logDebugGrammar("FIELD - LESS THAN DATE %s", phrase)
q := NewDateRangeInclusiveQuery(nil, &phrase, nil, &maxInclusive)
maxTime, err := QueryTimeFromString(phrase)
if err != nil {
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err))
}
q := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive)
q.SetField(field)
yyVAL.q = q
}
case 23:
yyDollar = yyS[yypt-5 : yypt+1]
//line query_string.y:247
//line query_string.y:260
{
field := yyDollar[1].s
maxInclusive := true
phrase := yyDollar[5].s
logDebugGrammar("FIELD - LESS THAN OR EQUAL DATE %s", phrase)
q := NewDateRangeInclusiveQuery(nil, &phrase, nil, &maxInclusive)
maxTime, err := QueryTimeFromString(phrase)
if err != nil {
yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err))
}
q := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive)
q.SetField(field)
yyVAL.q = q
}
case 24:
yyDollar = yyS[yypt-0 : yypt+1]
//line query_string.y:259
//line query_string.y:276
{
yyVAL.pf = nil
}
case 25:
yyDollar = yyS[yypt-1 : yypt+1]
//line query_string.y:263
//line query_string.y:280
{
yyVAL.pf = nil
boost, err := strconv.ParseFloat(yyDollar[1].s, 64)

22
search/query/query_string_parser_test.go
View File

@ -13,6 +13,7 @@ import (
"reflect"
"strings"
"testing"
"time"
"github.com/blevesearch/bleve/mapping"
)
@ -21,7 +22,10 @@ func TestQuerySyntaxParserValid(t *testing.T) {
fivePointOh := 5.0
theTruth := true
theFalsehood := false
theDate := "2006-01-02T15:04:05Z07:00"
theDate, err := time.Parse(time.RFC3339, "2006-01-02T15:04:05Z")
if err != nil {
t.Fatal(err)
}
tests := []struct {
input string
result Query
@ -443,13 +447,13 @@ func TestQuerySyntaxParserValid(t *testing.T) {
nil),
},
{
input: `field:>"2006-01-02T15:04:05Z07:00"`,
input: `field:>"2006-01-02T15:04:05Z"`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
nil,
[]Query{
func() Query {
q := NewDateRangeInclusiveQuery(&theDate, nil, &theFalsehood, nil)
q := NewDateRangeInclusiveQuery(theDate, time.Time{}, &theFalsehood, nil)
q.SetField("field")
return q
}(),
@ -457,13 +461,13 @@ func TestQuerySyntaxParserValid(t *testing.T) {
nil),
},
{
input: `field:>="2006-01-02T15:04:05Z07:00"`,
input: `field:>="2006-01-02T15:04:05Z"`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
nil,
[]Query{
func() Query {
q := NewDateRangeInclusiveQuery(&theDate, nil, &theTruth, nil)
q := NewDateRangeInclusiveQuery(theDate, time.Time{}, &theTruth, nil)
q.SetField("field")
return q
}(),
@ -471,13 +475,13 @@ func TestQuerySyntaxParserValid(t *testing.T) {
nil),
},
{
input: `field:<"2006-01-02T15:04:05Z07:00"`,
input: `field:<"2006-01-02T15:04:05Z"`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
nil,
[]Query{
func() Query {
q := NewDateRangeInclusiveQuery(nil, &theDate, nil, &theFalsehood)
q := NewDateRangeInclusiveQuery(time.Time{}, theDate, nil, &theFalsehood)
q.SetField("field")
return q
}(),
@ -485,13 +489,13 @@ func TestQuerySyntaxParserValid(t *testing.T) {
nil),
},
{
input: `field:<="2006-01-02T15:04:05Z07:00"`,
input: `field:<="2006-01-02T15:04:05Z"`,
mapping: mapping.NewIndexMapping(),
result: NewBooleanQuery(
nil,
[]Query{
func() Query {
q := NewDateRangeInclusiveQuery(nil, &theDate, nil, &theTruth)
q := NewDateRangeInclusiveQuery(time.Time{}, theDate, nil, &theTruth)
q.SetField("field")
return q
}(),

27
search/query/query_test.go
View File

@ -13,14 +13,29 @@ import (
"reflect"
"strings"
"testing"
"time"
"github.com/blevesearch/bleve/mapping"
)
var minNum = 5.1
var maxNum = 7.1
var startDate = "2011-01-01"
var endDate = "2012-01-01"
var startDateStr = "2011-01-01T00:00:00Z"
var endDateStr = "2012-01-01T00:00:00Z"
var startDate time.Time
var endDate time.Time
func init() {
var err error
startDate, err = time.Parse(time.RFC3339, startDateStr)
if err != nil {
panic(err)
}
endDate, err = time.Parse(time.RFC3339, endDateStr)
if err != nil {
panic(err)
}
}
func TestParseQuery(t *testing.T) {
tests := []struct {
@ -118,9 +133,9 @@ func TestParseQuery(t *testing.T) {
}(),
},
{
input: []byte(`{"start":"` + startDate + `","end":"` + endDate + `","field":"desc"}`),
input: []byte(`{"start":"` + startDateStr + `","end":"` + endDateStr + `","field":"desc"}`),
output: func() Query {
q := NewDateRangeQuery(&startDate, &endDate)
q := NewDateRangeQuery(startDate, endDate)
q.SetField("desc")
return q
}(),
@ -159,7 +174,7 @@ func TestParseQuery(t *testing.T) {
}
if !reflect.DeepEqual(test.output, actual) {
t.Errorf("expected: %#v, got: %#v", test.output, actual)
t.Errorf("expected: %#v, got: %#v for %s", test.output, actual, string(test.input))
}
}
}
@ -207,7 +222,7 @@ func TestQueryValidate(t *testing.T) {
},
{
query: func() Query {
q := NewDateRangeQuery(&startDate, &endDate)
q := NewDateRangeQuery(startDate, endDate)
q.SetField("desc")
return q
}(),