Merge branch 'master' into newkvstore

index/upside_down/upside_down.go

@@ -345,9 +345,7 @@ func (udc *UpsideDownCouch) Update(doc *document.Document) (err error) {
 	resultChan := make(chan *index.AnalysisResult)
 	aw := index.NewAnalysisWork(udc, doc, resultChan)
 	// put the work on the queue
-	go func() {
-		udc.analysisQueue.Queue(aw)
-	}()
+	udc.analysisQueue.Queue(aw)
 
 	// wait for the result
 	result := <-resultChan
@@ -501,10 +499,10 @@ func encodeFieldType(f document.Field) byte {
 func (udc *UpsideDownCouch) indexField(docID string, field document.Field, fieldIndex uint16, fieldLength int, tokenFreqs analysis.TokenFrequencies) ([]index.IndexRow, []*BackIndexTermEntry) {
 
 	rows := make([]index.IndexRow, 0, 100)
-	backIndexTermEntries := make([]*BackIndexTermEntry, 0)
+	backIndexTermEntries := make([]*BackIndexTermEntry, 0, len(tokenFreqs))
 	fieldNorm := float32(1.0 / math.Sqrt(float64(fieldLength)))
 
-	for _, tf := range tokenFreqs {
+	for k, tf := range tokenFreqs {
 		var termFreqRow *TermFrequencyRow
 		if field.Options().IncludeTermVectors() {
 			tv, newFieldRows := udc.termVectorsFromTokenFreq(fieldIndex, tf)
@@ -515,7 +513,7 @@ func (udc *UpsideDownCouch) indexField(docID string, field document.Field, field
 		}
 
 		// record the back index entry
-		backIndexTermEntry := BackIndexTermEntry{Term: proto.String(string(tf.Term)), Field: proto.Uint32(uint32(fieldIndex))}
+		backIndexTermEntry := BackIndexTermEntry{Term: proto.String(k), Field: proto.Uint32(uint32(fieldIndex))}
 		backIndexTermEntries = append(backIndexTermEntries, &backIndexTermEntry)
 
 		rows = append(rows, termFreqRow)

utils/bleve_dump/main.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/blevesearch/bleve"
 	_ "github.com/blevesearch/bleve/config"
+	_ "github.com/blevesearch/bleve/index/store/metrics"
 	"github.com/blevesearch/bleve/index/upside_down"
 )
 

utils/bleve_index/main.go

@@ -20,9 +20,11 @@ import (
 	_ "github.com/blevesearch/bleve/config"
 )
 
-var indexPath = flag.String("index", "", "index path")
-var keepExt = flag.Bool("keepExt", false, "keep extension in doc id")
-var keepDir = flag.Bool("keepDir", false, "keep dir in doc id")
+var (
+	indexPath = flag.String("index", "", "index path")
+	keepExt   = flag.Bool("keepExt", false, "keep extension in doc id")
+	keepDir   = flag.Bool("keepDir", false, "keep dir in doc id")
+)
 
 func main() {
 
@@ -73,44 +75,35 @@ type file struct {
 
 func handleArgs(args []string) chan file {
 	rv := make(chan file)
-
-	go func() {
-		for _, arg := range args {
-			arg = filepath.Clean(arg)
-			handleArgRecursive(arg, rv)
-		}
-		close(rv)
-	}()
-
+	go getAllFiles(args, rv)
 	return rv
 }
 
-func handleArgRecursive(arg string, results chan file) {
-	stat, err := os.Stat(arg)
-	if err != nil {
-		log.Print(err)
-		return
-	}
-	if stat.IsDir() {
-		// open the directory
-		dirEntries, err := ioutil.ReadDir(arg)
+func getAllFiles(args []string, rv chan file) {
+	for _, arg := range args {
+		arg = filepath.Clean(arg)
+		err := filepath.Walk(arg, func(path string, finfo os.FileInfo, err error) error {
+			if err != nil {
+				log.Print(err)
+				return err
+			}
+			if finfo.IsDir() {
+				return nil
+			}
+
+			bytes, err := ioutil.ReadFile(path)
+			if err != nil {
+				log.Fatal(err)
+			}
+			rv <- file{
+				filename: filepath.Base(path),
+				contents: bytes,
+			}
+			return nil
+		})
 		if err != nil {
 			log.Fatal(err)
 		}
-
-		// walk the directory entries
-		for _, dirEntry := range dirEntries {
-			handleArgRecursive(arg+string(os.PathSeparator)+dirEntry.Name(), results)
-		}
-	} else {
-		bytes, err := ioutil.ReadFile(arg)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		results <- file{
-			filename: arg,
-			contents: bytes,
-		}
 	}
+	close(rv)
 }