gibheer
/
bleve
0
0
Fork 0
Code

remove use of float64 to represent int things 

this originated from a misunderstanding of mine going back
several years.  the values need not be float64 just because
we plan to serialize them as json.

there are still larger questions about what the right type should
be, and where should any conversions go.  but, this commit
simply attempts to address the most egregious problems
This commit is contained in:
Marty Schoch 2017-02-09 20:02:12 -05:00
parent 0c87b7bff1
commit 8096d9fb90
11 changed files with 67 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
index_test.go
View File

@ -1114,7 +1114,7 @@ func TestTermVectorArrayPositions(t *testing.T) {
t.Fatalf("expected at least one location array position")
}
if results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0] != 1 {
t.Fatalf("expected array position 1, got %f", results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0])
t.Fatalf("expected array position 1, got %d", results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0])
}
// repeat search for this document in Messages field
@ -1136,7 +1136,7 @@ func TestTermVectorArrayPositions(t *testing.T) {
t.Fatalf("expected at least one location array position")
}
if results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0] != 2 {
t.Fatalf("expected array position 2, got %f", results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0])
t.Fatalf("expected array position 2, got %d", results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0])
}
err = index.Close()

2
search/highlight/format/ansi/ansi.go
View File

@ -41,7 +41,7 @@ func (a *FragmentFormatter) Format(f *highlight.Fragment, orderedTermLocations h
continue
}
// make sure the array positions match
if !highlight.SameArrayPositions(f.ArrayPositions, termLocation.ArrayPositions) {
if !termLocation.ArrayPositions.Equals(f.ArrayPositions) {
continue
}
if termLocation.Start < curr {

2
search/highlight/format/html/html.go
View File

@ -44,7 +44,7 @@ func (a *FragmentFormatter) Format(f *highlight.Fragment, orderedTermLocations h
continue
}
// make sure the array positions match
if !highlight.SameArrayPositions(f.ArrayPositions, termLocation.ArrayPositions) {
if !termLocation.ArrayPositions.Equals(f.ArrayPositions) {
continue
}
if termLocation.Start < curr {

2
search/highlight/highlighter/simple/fragment_scorer_simple.go
View File

@ -37,7 +37,7 @@ func (s *FragmentScorer) Score(f *highlight.Fragment) {
OUTER:
for _, locations := range s.tlm {
for _, location := range locations {
if highlight.SameArrayPositions(f.ArrayPositions, location.ArrayPositions) && int(location.Start) >= f.Start && int(location.End) <= f.End {
if location.ArrayPositions.Equals(f.ArrayPositions) && int(location.Start) >= f.Start && int(location.End) <= f.End {
score += 1.0
// once we find a term in the fragment
// don't care about additional matches

2
search/highlight/highlighter/simple/highlighter_simple.go
View File

@ -87,7 +87,7 @@ func (s *Highlighter) BestFragmentsInField(dm *search.DocumentMatch, doc *docume
if ok {
termLocationsSameArrayPosition := make(highlight.TermLocations, 0)
for _, otl := range orderedTermLocations {
if highlight.SameArrayPositions(f.ArrayPositions(), otl.ArrayPositions) {
if otl.ArrayPositions.Equals(f.ArrayPositions()) {
termLocationsSameArrayPosition = append(termLocationsSameArrayPosition, otl)
}
}

14
search/highlight/term_locations.go
View File

@ -23,7 +23,7 @@ import (
type TermLocation struct {
Term string
ArrayPositions []float64
ArrayPositions search.ArrayPositions
Pos int
Start int
End int
@ -103,15 +103,3 @@ func OrderTermLocations(tlm search.TermLocationMap) TermLocations {
sort.Sort(rv)
return rv
}
func SameArrayPositions(fieldArrayPositions []uint64, termLocationArrayPositions []float64) bool {
if len(fieldArrayPositions) != len(termLocationArrayPositions) {
return false
}
for i := 0; i < len(fieldArrayPositions); i++ {
if fieldArrayPositions[i] != uint64(termLocationArrayPositions[i]) {
return false
}
}
return true
}

78
search/highlight/term_locations_test.go
View File

@ -64,12 +64,12 @@ func TestTermLocationOverlaps(t *testing.T) {
// with array positions
{
left: &TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
right: &TermLocation{
ArrayPositions: []float64{1},
ArrayPositions: search.ArrayPositions{1},
Start: 7,
End: 11,
},
@ -77,12 +77,12 @@ func TestTermLocationOverlaps(t *testing.T) {
},
{
left: &TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
right: &TermLocation{
ArrayPositions: []float64{1},
ArrayPositions: search.ArrayPositions{1},
Start: 3,
End: 11,
},
@ -90,12 +90,12 @@ func TestTermLocationOverlaps(t *testing.T) {
},
{
left: &TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
right: &TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 3,
End: 11,
},
@ -103,12 +103,12 @@ func TestTermLocationOverlaps(t *testing.T) {
},
{
left: &TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
right: &TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 7,
End: 11,
},
@ -235,24 +235,24 @@ func TestTermLocationsMergeOverlapping(t *testing.T) {
{
input: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
&TermLocation{
ArrayPositions: []float64{1},
ArrayPositions: search.ArrayPositions{1},
Start: 7,
End: 11,
},
},
output: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
&TermLocation{
ArrayPositions: []float64{1},
ArrayPositions: search.ArrayPositions{1},
Start: 7,
End: 11,
},
@ -261,24 +261,24 @@ func TestTermLocationsMergeOverlapping(t *testing.T) {
{
input: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 7,
End: 11,
},
},
output: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 7,
End: 11,
},
@ -287,19 +287,19 @@ func TestTermLocationsMergeOverlapping(t *testing.T) {
{
input: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 3,
End: 11,
},
},
output: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 11,
},
@ -309,24 +309,24 @@ func TestTermLocationsMergeOverlapping(t *testing.T) {
{
input: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
&TermLocation{
ArrayPositions: []float64{1},
ArrayPositions: search.ArrayPositions{1},
Start: 3,
End: 11,
},
},
output: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
End: 5,
},
&TermLocation{
ArrayPositions: []float64{1},
ArrayPositions: search.ArrayPositions{1},
Start: 3,
End: 11,
},
@ -401,23 +401,23 @@ func TestTermLocationsOrder(t *testing.T) {
input: search.TermLocationMap{
"term": []*search.Location{
{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
},
{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 5,
},
},
},
output: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Term: "term",
Start: 0,
},
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Term: "term",
Start: 5,
},
@ -427,23 +427,23 @@ func TestTermLocationsOrder(t *testing.T) {
input: search.TermLocationMap{
"term": []*search.Location{
{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 5,
},
{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 0,
},
},
},
output: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Term: "term",
Start: 0,
},
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Term: "term",
Start: 5,
},
@ -453,23 +453,23 @@ func TestTermLocationsOrder(t *testing.T) {
input: search.TermLocationMap{
"term": []*search.Location{
{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 5,
},
{
ArrayPositions: []float64{1},
ArrayPositions: search.ArrayPositions{1},
Start: 0,
},
},
},
output: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Term: "term",
Start: 5,
},
&TermLocation{
ArrayPositions: []float64{1},
ArrayPositions: search.ArrayPositions{1},
Term: "term",
Start: 0,
},
@ -479,23 +479,23 @@ func TestTermLocationsOrder(t *testing.T) {
input: search.TermLocationMap{
"term": []*search.Location{
{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Start: 5,
},
{
ArrayPositions: []float64{0, 1},
ArrayPositions: search.ArrayPositions{0, 1},
Start: 0,
},
},
},
output: TermLocations{
&TermLocation{
ArrayPositions: []float64{0},
ArrayPositions: search.ArrayPositions{0},
Term: "term",
Start: 5,
},
&TermLocation{
ArrayPositions: []float64{0, 1},
ArrayPositions: search.ArrayPositions{0, 1},
Term: "term",
Start: 0,
},

10
search/scorer/scorer_term.go
View File

@ -148,7 +148,7 @@ func (s *TermQueryScorer) Score(ctx *search.SearchContext, termMatch *index.Term
for _, v := range termMatch.Vectors {
totalPositions += len(v.ArrayPositions)
}
positions := make([]float64, totalPositions)
positions := make(search.ArrayPositions, totalPositions)
positionsUsed := 0
rv.Locations = make(search.FieldTermLocationMap)
@ -162,14 +162,14 @@ func (s *TermQueryScorer) Score(ctx *search.SearchContext, termMatch *index.Term
loc := &locs[locsUsed]
locsUsed++
loc.Pos = float64(v.Pos)
loc.Start = float64(v.Start)
loc.End = float64(v.End)
loc.Pos = v.Pos
loc.Start = v.Start
loc.End = v.End
if len(v.ArrayPositions) > 0 {
loc.ArrayPositions = positions[positionsUsed : positionsUsed+len(v.ArrayPositions)]
for i, ap := range v.ArrayPositions {
loc.ArrayPositions[i] = float64(ap)
loc.ArrayPositions[i] = ap
}
positionsUsed += len(v.ArrayPositions)
}

8
search/search.go
View File

@ -21,7 +21,7 @@ import (
"github.com/blevesearch/bleve/index"
)
type ArrayPositions []float64
type ArrayPositions []uint64
func (ap ArrayPositions) Equals(other ArrayPositions) bool {
if len(ap) != len(other) {
@ -36,9 +36,9 @@ func (ap ArrayPositions) Equals(other ArrayPositions) bool {
}
type Location struct {
Pos float64 `json:"pos"`
Start float64 `json:"start"`
End float64 `json:"end"`
Pos uint64 `json:"pos"`
Start uint64 `json:"start"`
End uint64 `json:"end"`
ArrayPositions ArrayPositions `json:"array_positions"`
}

24
search/searcher/search_phrase.go
View File

@ -185,7 +185,7 @@ func (p phrasePath) MergeInto(in search.TermLocationMap) {
// this is the primary state being built during the traversal
//
// returns slice of paths, or nil if invocation did not find any successul paths
func findPhrasePaths(prevPos float64, ap search.ArrayPositions, phraseTerms []string, tlm search.TermLocationMap, p phrasePath, remainingSlop int) []phrasePath {
func findPhrasePaths(prevPos uint64, ap search.ArrayPositions, phraseTerms []string, tlm search.TermLocationMap, p phrasePath, remainingSlop int) []phrasePath {
// no more terms
if len(phraseTerms) < 1 {
@ -197,10 +197,10 @@ func findPhrasePaths(prevPos float64, ap search.ArrayPositions, phraseTerms []st
// empty term is treated as match (continue)
if car == "" {
nextPos := prevPos + 1.0
if prevPos == 0.0 {
// if prevPos was 0.0, don't set it to 1 (as thats not a real abs pos)
nextPos = 0.0 // don't advance nextPos if prevPos was 0
nextPos := prevPos + 1
if prevPos == 0 {
// if prevPos was 0, don't set it to 1 (as thats not a real abs pos)
nextPos = 0 // don't advance nextPos if prevPos was 0
}
return findPhrasePaths(nextPos, ap, cdr, tlm, p, remainingSlop)
}
@ -209,19 +209,19 @@ func findPhrasePaths(prevPos float64, ap search.ArrayPositions, phraseTerms []st
locations := tlm[car]
var rv []phrasePath
for _, loc := range locations {
if prevPos != 0.0 && !loc.ArrayPositions.Equals(ap) {
if prevPos != 0 && !loc.ArrayPositions.Equals(ap) {
// if the array positions are wrong, can't match, try next location
continue
}
// compute distance from previous phrase term
dist := 0
if prevPos != 0.0 {
dist = editDistance(prevPos+1.0, loc.Pos)
if prevPos != 0 {
dist = editDistance(prevPos+1, loc.Pos)
}
// if enough slop reamining, continue recursively
if prevPos == 0.0 || (remainingSlop-dist) >= 0 {
if prevPos == 0 || (remainingSlop-dist) >= 0 {
// this location works, add it to the path (but not for empty term)
px := append(p, &phrasePart{term: car, loc: loc})
rv = append(rv, findPhrasePaths(loc.Pos, loc.ArrayPositions, cdr, tlm, px, remainingSlop-dist)...)
@ -230,10 +230,8 @@ func findPhrasePaths(prevPos float64, ap search.ArrayPositions, phraseTerms []st
return rv
}
func editDistance(p1, p2 float64) int {
i1 := int(p1)
i2 := int(p2)
dist := i1 - i2
func editDistance(p1, p2 uint64) int {
dist := int(p1 - p2)
if dist < 0 {
return -dist
}

2
search/searcher/search_phrase_test.go
View File

@ -68,7 +68,7 @@ func TestPhraseSearch(t *testing.T) {
Score: 1.0807601687084403,
},
},
locations: map[string]map[string][]search.Location{"desc": map[string][]search.Location{"beer": []search.Location{search.Location{Pos: 2, Start: 6, End: 10, ArrayPositions: []float64(nil)}}, "angst": []search.Location{search.Location{Pos: 1, Start: 0, End: 5, ArrayPositions: []float64(nil)}}}},
locations: map[string]map[string][]search.Location{"desc": map[string][]search.Location{"beer": []search.Location{search.Location{Pos: 2, Start: 6, End: 10}}, "angst": []search.Location{search.Location{Pos: 1, Start: 0, End: 5}}}},
fieldterms: [][2]string{[2]string{"desc", "beer"}, [2]string{"desc", "angst"}},
},
}