gibheer/bleve
0
0
Fork 0
Code

Merge pull request #531 from mschoch/losefloat

Browse Source 
remove use of float64 to represent int things
This commit is contained in:
Marty Schoch 2017-02-09 20:25:20 -05:00 committed by GitHub
commit 97a428f5b0
11 changed files with 67 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
index_test.go
View File

@ -1114,7 +1114,7 @@ func TestTermVectorArrayPositions(t *testing.T) {
t.Fatalf("expected at least one location array position")
}
if results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0] != 1 {
t.Fatalf("expected array position 1, got %f", results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0])
t.Fatalf("expected array position 1, got %d", results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0])
}
// repeat search for this document in Messages field
@ -1136,7 +1136,7 @@ func TestTermVectorArrayPositions(t *testing.T) {
t.Fatalf("expected at least one location array position")
}
if results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0] != 2 {
t.Fatalf("expected array position 2, got %f", results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0])
t.Fatalf("expected array position 2, got %d", results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0])
}
err = index.Close()

2
search/highlight/format/ansi/ansi.go
View File

@ -41,7 +41,7 @@ func (a *FragmentFormatter) Format(f *highlight.Fragment, orderedTermLocations h
continue
}
// make sure the array positions match
if !highlight.SameArrayPositions(f.ArrayPositions, termLocation.ArrayPositions) {
if !termLocation.ArrayPositions.Equals(f.ArrayPositions) {
continue
}
if termLocation.Start < curr {

2
search/highlight/format/html/html.go
View File

@ -44,7 +44,7 @@ func (a *FragmentFormatter) Format(f *highlight.Fragment, orderedTermLocations h
continue
}
// make sure the array positions match
if !highlight.SameArrayPositions(f.ArrayPositions, termLocation.ArrayPositions) {
if !termLocation.ArrayPositions.Equals(f.ArrayPositions) {
continue
}
if termLocation.Start < curr {

2
search/highlight/highlighter/simple/fragment_scorer_simple.go
View File

@ -37,7 +37,7 @@ func (s *FragmentScorer) Score(f *highlight.Fragment) {
OUTER:
for _, locations := range s.tlm {
for _, location := range locations {
if highlight.SameArrayPositions(f.ArrayPositions, location.ArrayPositions) && int(location.Start) >= f.Start && int(location.End) <= f.End {
if location.ArrayPositions.Equals(f.ArrayPositions) && int(location.Start) >= f.Start && int(location.End) <= f.End {
score += 1.0
// once we find a term in the fragment
// don't care about additional matches

2
search/highlight/highlighter/simple/highlighter_simple.go
View File

@ -87,7 +87,7 @@ func (s *Highlighter) BestFragmentsInField(dm *search.DocumentMatch, doc *docume
if ok {
termLocationsSameArrayPosition := make(highlight.TermLocations, 0)
for _, otl := range orderedTermLocations {
if highlight.SameArrayPositions(f.ArrayPositions(), otl.ArrayPositions) {
if otl.ArrayPositions.Equals(f.ArrayPositions()) {
termLocationsSameArrayPosition = append(termLocationsSameArrayPosition, otl)
}
}

14
search/highlight/term_locations.go
View File

@ -23,7 +23,7 @@ import (
type TermLocation struct {
Term string
ArrayPositions []float64
ArrayPositions search.ArrayPositions
Pos int
Start int
End int
@ -103,15 +103,3 @@ func OrderTermLocations(tlm search.TermLocationMap) TermLocations {
sort.Sort(rv)
return rv
}
func SameArrayPositions(fieldArrayPositions []uint64, termLocationArrayPositions []float64) bool {
if len(fieldArrayPositions) != len(termLocationArrayPositions) {
return false
}
for i := 0; i < len(fieldArrayPositions); i++ {
if fieldArrayPositions[i] != uint64(termLocationArrayPositions[i]) {
return false
}
}
return true
}

78
search/highlight/term_locations_test.go
View File

@ -64,12 +64,12 @@ func TestTermLocationOverlaps(t *testing.T) {
// with array positions
{
left: &TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
right: &TermLocation{
ArrayPositions: []float64{1},
ArrayPositions: search.ArrayPositions{1},
Start: 7,
End: 11,
},
@ -77,12 +77,12 @@ func TestTermLocationOverlaps(t *testing.T) {
},
{
left: &TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
right: &TermLocation{
ArrayPositions: []float64{1},
ArrayPositions: search.ArrayPositions{1},
Start: 3,
End: 11,
},
@ -90,12 +90,12 @@ func TestTermLocationOverlaps(t *testing.T) {
},
{
left: &TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
right: &TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 3,
End: 11,
},
@ -103,12 +103,12 @@ func TestTermLocationOverlaps(t *testing.T) {
},
{
left: &TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
right: &TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 7,
End: 11,
},
@ -235,24 +235,24 @@ func TestTermLocationsMergeOverlapping(t *testing.T) {
{
input: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
&TermLocation{
ArrayPositions: []float64{1},
ArrayPositions: search.ArrayPositions{1},
Start: 7,
End: 11,
},
},
output: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
&TermLocation{
ArrayPositions: []float64{1},
ArrayPositions: search.ArrayPositions{1},
Start: 7,
End: 11,
},
@ -261,24 +261,24 @@ func TestTermLocationsMergeOverlapping(t *testing.T) {
{
input: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 7,
End: 11,
},
},
output: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 7,
End: 11,
},
@ -287,19 +287,19 @@ func TestTermLocationsMergeOverlapping(t *testing.T) {
{
input: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 3,
End: 11,
},
},
output: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 11,
},
@ -309,24 +309,24 @@ func TestTermLocationsMergeOverlapping(t *testing.T) {
{
input: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
&TermLocation{
ArrayPositions: []float64{1},
ArrayPositions: search.ArrayPositions{1},
Start: 3,
End: 11,
},
},
output: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
&TermLocation{
ArrayPositions: []float64{1},
ArrayPositions: search.ArrayPositions{1},
Start: 3,
End: 11,
},
@ -401,23 +401,23 @@ func TestTermLocationsOrder(t *testing.T) {
input: search.TermLocationMap{
"term": []*search.Location{
{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
},
{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 5,
},
},
},
output: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Term: "term",
Start: 0,
},
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Term: "term",
Start: 5,
},
@ -427,23 +427,23 @@ func TestTermLocationsOrder(t *testing.T) {
input: search.TermLocationMap{
"term": []*search.Location{
{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 5,
},
{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
},
},
},
output: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Term: "term",
Start: 0,
},
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Term: "term",
Start: 5,
},
@ -453,23 +453,23 @@ func TestTermLocationsOrder(t *testing.T) {
input: search.TermLocationMap{
"term": []*search.Location{
{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 5,
},
{
ArrayPositions: []float64{1},
ArrayPositions: search.ArrayPositions{1},
Start: 0,
},
},
},
output: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Term: "term",
Start: 5,
},
&TermLocation{
ArrayPositions: []float64{1},
ArrayPositions: search.ArrayPositions{1},
Term: "term",
Start: 0,
},
@ -479,23 +479,23 @@ func TestTermLocationsOrder(t *testing.T) {
input: search.TermLocationMap{
"term": []*search.Location{
{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 5,
},
{
ArrayPositions: []float64{0, 1},
ArrayPositions: search.ArrayPositions{0, 1},
Start: 0,
},
},
},
output: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Term: "term",
Start: 5,
},
&TermLocation{
ArrayPositions: []float64{0, 1},
ArrayPositions: search.ArrayPositions{0, 1},
Term: "term",
Start: 0,
},

10
search/scorer/scorer_term.go
View File

@ -148,7 +148,7 @@ func (s *TermQueryScorer) Score(ctx *search.SearchContext, termMatch *index.Term
for _, v := range termMatch.Vectors {
totalPositions += len(v.ArrayPositions)
}
positions := make([]float64, totalPositions)
positions := make(search.ArrayPositions, totalPositions)
positionsUsed := 0
rv.Locations = make(search.FieldTermLocationMap)
@ -162,14 +162,14 @@ func (s *TermQueryScorer) Score(ctx *search.SearchContext, termMatch *index.Term
loc := &locs[locsUsed]
locsUsed++
loc.Pos = float64(v.Pos)
loc.Start = float64(v.Start)
loc.End = float64(v.End)
loc.Pos = v.Pos
loc.Start = v.Start
loc.End = v.End
if len(v.ArrayPositions) > 0 {
loc.ArrayPositions = positions[positionsUsed : positionsUsed+len(v.ArrayPositions)]
for i, ap := range v.ArrayPositions {
loc.ArrayPositions[i] = float64(ap)
loc.ArrayPositions[i] = ap
}
positionsUsed += len(v.ArrayPositions)
}

8
search/search.go
View File

@ -21,7 +21,7 @@ import (
"github.com/blevesearch/bleve/index"
)
type ArrayPositions []float64
type ArrayPositions []uint64
func (ap ArrayPositions) Equals(other ArrayPositions) bool {
if len(ap) != len(other) {
@ -36,9 +36,9 @@ func (ap ArrayPositions) Equals(other ArrayPositions) bool {
}
type Location struct {
Pos float64 `json:"pos"`
Start float64 `json:"start"`
End float64 `json:"end"`
Pos uint64 `json:"pos"`
Start uint64 `json:"start"`
End uint64 `json:"end"`
ArrayPositions ArrayPositions `json:"array_positions"`
}

24
search/searcher/search_phrase.go
View File

@ -185,7 +185,7 @@ func (p phrasePath) MergeInto(in search.TermLocationMap) {
// this is the primary state being built during the traversal
//
// returns slice of paths, or nil if invocation did not find any successul paths
func findPhrasePaths(prevPos float64, ap search.ArrayPositions, phraseTerms []string, tlm search.TermLocationMap, p phrasePath, remainingSlop int) []phrasePath {
func findPhrasePaths(prevPos uint64, ap search.ArrayPositions, phraseTerms []string, tlm search.TermLocationMap, p phrasePath, remainingSlop int) []phrasePath {
// no more terms
if len(phraseTerms) < 1 {
@ -197,10 +197,10 @@ func findPhrasePaths(prevPos float64, ap search.ArrayPositions, phraseTerms []st
// empty term is treated as match (continue)
if car == "" {
nextPos := prevPos + 1.0
if prevPos == 0.0 {
// if prevPos was 0.0, don't set it to 1 (as thats not a real abs pos)
nextPos = 0.0 // don't advance nextPos if prevPos was 0
nextPos := prevPos + 1
if prevPos == 0 {
// if prevPos was 0, don't set it to 1 (as thats not a real abs pos)
nextPos = 0 // don't advance nextPos if prevPos was 0
}
return findPhrasePaths(nextPos, ap, cdr, tlm, p, remainingSlop)
}
@ -209,19 +209,19 @@ func findPhrasePaths(prevPos float64, ap search.ArrayPositions, phraseTerms []st
locations := tlm[car]
var rv []phrasePath
for _, loc := range locations {
if prevPos != 0.0 && !loc.ArrayPositions.Equals(ap) {
if prevPos != 0 && !loc.ArrayPositions.Equals(ap) {
// if the array positions are wrong, can't match, try next location
continue
}
// compute distance from previous phrase term
dist := 0
if prevPos != 0.0 {
dist = editDistance(prevPos+1.0, loc.Pos)
if prevPos != 0 {
dist = editDistance(prevPos+1, loc.Pos)
}
// if enough slop reamining, continue recursively
if prevPos == 0.0 || (remainingSlop-dist) >= 0 {
if prevPos == 0 || (remainingSlop-dist) >= 0 {
// this location works, add it to the path (but not for empty term)
px := append(p, &phrasePart{term: car, loc: loc})
rv = append(rv, findPhrasePaths(loc.Pos, loc.ArrayPositions, cdr, tlm, px, remainingSlop-dist)...)
@ -230,10 +230,8 @@ func findPhrasePaths(prevPos float64, ap search.ArrayPositions, phraseTerms []st
return rv
}
func editDistance(p1, p2 float64) int {
i1 := int(p1)
i2 := int(p2)
dist := i1 - i2
func editDistance(p1, p2 uint64) int {
dist := int(p1 - p2)
if dist < 0 {
return -dist
}

2
search/searcher/search_phrase_test.go
View File

@ -68,7 +68,7 @@ func TestPhraseSearch(t *testing.T) {
Score: 1.0807601687084403,
},
},
locations: map[string]map[string][]search.Location{"desc": map[string][]search.Location{"beer": []search.Location{search.Location{Pos: 2, Start: 6, End: 10, ArrayPositions: []float64(nil)}}, "angst": []search.Location{search.Location{Pos: 1, Start: 0, End: 5, ArrayPositions: []float64(nil)}}}},
locations: map[string]map[string][]search.Location{"desc": map[string][]search.Location{"beer": []search.Location{search.Location{Pos: 2, Start: 6, End: 10}}, "angst": []search.Location{search.Location{Pos: 1, Start: 0, End: 5}}}},
fieldterms: [][2]string{[2]string{"desc", "beer"}, [2]string{"desc", "angst"}},
},
}