Merge branch 'master' into newkvstore

analysis/freq.go

@@ -9,6 +9,10 @@
 
 package analysis
 
+// TokenLocation represents one occurrence of a term at a particular location in
+// a field. Start, End and Position have the same meaning as in analysis.Token.
+// Field and ArrayPositions identify the field value in the source document.
+// See document.Field for details.
 type TokenLocation struct {
 	Field          string
 	ArrayPositions []uint64
@@ -17,11 +21,15 @@ type TokenLocation struct {
 	Position       int
 }
 
+// TokenFreq represents all the occurrences of a term in all fields of a
+// document.
 type TokenFreq struct {
 	Term      []byte
 	Locations []*TokenLocation
 }
 
+// TokenFrequencies maps document terms to their combined frequencies from all
+// fields.
 type TokenFrequencies map[string]*TokenFreq
 
 func (tfs TokenFrequencies) MergeAll(remoteField string, other TokenFrequencies) {

analysis/type.go

@@ -30,10 +30,19 @@ const (
 	Double
 )
 
+// Token represents one occurrence of a term at a particular location in a
+// field.
 type Token struct {
-	Start    int       `json:"start"`
-	End      int       `json:"end"`
-	Term     []byte    `json:"term"`
+	// Start specifies the byte offset of the beginning of the term in the
+	// field.
+	Start int `json:"start"`
+
+	// End specifies the byte offset of the end of the term in the field.
+	End  int    `json:"end"`
+	Term []byte `json:"term"`
+
+	// Position specifies the 1-based index of the token in the sequence of
+	// occurrences of its term in the field.
 	Position int       `json:"position"`
 	Type     TokenType `json:"type"`
 	KeyWord  bool      `json:"keyword"`

document/field.go

@@ -14,7 +14,14 @@ import (
 )
 
 type Field interface {
+	// Name returns the path of the field from the root DocumentMapping.
+	// A root field path is "field", a subdocument field is "parent.field".
 	Name() string
+	// ArrayPositions returns the intermediate document and field indices
+	// required to resolve the field value in the document. For example, if the
+	// field path is "doc1.doc2.field" where doc1 and doc2 are slices or
+	// arrays, ArrayPositions returns 2 indices used to resolve "doc2" value in
+	// "doc1", then "field" in "doc2".
 	ArrayPositions() []uint64
 	Options() IndexingOptions
 	Analyze() (int, analysis.TokenFrequencies)

examples_test.go

@@ -253,7 +253,7 @@ func ExampleNewFacetRequest() {
 	fmt.Println(searchResults.Facets["facet name"].Total)
 	// numer of docs with no value for this field
 	fmt.Println(searchResults.Facets["facet name"].Missing)
-	// term with highest occurences in field name
+	// term with highest occurrences in field name
 	fmt.Println(searchResults.Facets["facet name"].Terms[0].Term)
 	// Output:
 	// 5
@@ -339,7 +339,7 @@ func ExampleSearchRequest_AddFacet() {
 	fmt.Println(searchResults.Facets["facet name"].Total)
 	// numer of docs with no value for this field
 	fmt.Println(searchResults.Facets["facet name"].Missing)
-	// term with highest occurences in field name
+	// term with highest occurrences in field name
 	fmt.Println(searchResults.Facets["facet name"].Terms[0].Term)
 	// Output:
 	// 5

index/store/boltdb/store.go

@@ -7,6 +7,14 @@
 //  either express or implied. See the License for the specific language governing permissions
 //  and limitations under the License.
 
+// Package boltdb implements a store.KVStore on top of BoltDB. It supports the
+// following options:
+//
+// "bucket" (string): the name of BoltDB bucket to use, defaults to "bleve".
+//
+// "nosync" (bool): if true, set boltdb.DB.NoSync to true. It speeds up index
+// operations in exchange of losing integrity guarantees if indexation aborts
+// without closing the index. Use it when rebuilding indexes from zero.
 package boltdb
 
 import (
@@ -23,6 +31,7 @@ type Store struct {
 	path   string
 	bucket string
 	db     *bolt.DB
+	noSync bool
 	mo     store.MergeOperator
 }
 
@@ -37,10 +46,13 @@ func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore,
 		bucket = "bleve"
 	}
 
+	noSync, _ := config["nosync"].(bool)
+
 	db, err := bolt.Open(path, 0600, nil)
 	if err != nil {
 		return nil, err
 	}
+	db.NoSync = noSync
 
 	err = db.Update(func(tx *bolt.Tx) error {
 		_, err := tx.CreateBucketIfNotExists([]byte(bucket))
@@ -56,6 +68,7 @@ func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore,
 		bucket: bucket,
 		db:     db,
 		mo:     mo,
+		noSync: noSync,
 	}
 	return &rv, nil
 }

index/upside_down/upside_down.go

@@ -220,11 +220,7 @@ func (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRows []UpsideDow
 	}
 
 	// write out the batch
-	err = writer.ExecuteBatch(wb)
-	if err != nil {
-		return
-	}
-	return
+	return writer.ExecuteBatch(wb)
 }
 
 func (udc *UpsideDownCouch) DocCount() (uint64, error) {

mapping_field.go

@@ -28,6 +28,8 @@ type FieldMapping struct {
 	// the IndexMapping.DefaultAnalyzer.
 	Analyzer string `json:"analyzer,omitempty"`
 
+	// Store indicates whether to store field values in the index. Stored
+	// values can be retrieved from search results using SearchRequest.Fields.
 	Store              bool   `json:"store,omitempty"`
 	Index              bool   `json:"index,omitempty"`
 	IncludeTermVectors bool   `json:"include_term_vectors,omitempty"`

search.go

@@ -152,7 +152,8 @@ func (h *HighlightRequest) AddField(field string) {
 // Highlight describes optional search result
 // highlighting.
 // Fields describes a list of field values which
-// should be retrieved for result documents.
+// should be retrieved for result documents, provided they
+// were stored while indexing.
 // Facets describe the set of facets to be computed.
 // Explain triggers inclusion of additional search
 // result score explanations.

search/search.go

@@ -37,12 +37,16 @@ type FieldTermLocationMap map[string]TermLocationMap
 type FieldFragmentMap map[string][]string
 
 type DocumentMatch struct {
-	ID        string                 `json:"id"`
-	Score     float64                `json:"score"`
-	Expl      *Explanation           `json:"explanation,omitempty"`
-	Locations FieldTermLocationMap   `json:"locations,omitempty"`
-	Fragments FieldFragmentMap       `json:"fragments,omitempty"`
-	Fields    map[string]interface{} `json:"fields,omitempty"`
+	ID        string               `json:"id"`
+	Score     float64              `json:"score"`
+	Expl      *Explanation         `json:"explanation,omitempty"`
+	Locations FieldTermLocationMap `json:"locations,omitempty"`
+	Fragments FieldFragmentMap     `json:"fragments,omitempty"`
+
+	// Fields contains the values for document fields listed in
+	// SearchRequest.Fields. Text fields are returned as strings, numeric
+	// fields as float64s and date fields as time.RFC3339 formatted strings.
+	Fields map[string]interface{} `json:"fields,omitempty"`
 }
 
 func (dm *DocumentMatch) AddFieldValue(name string, value interface{}) {