improve performance of regular expression and wildcard queries

While researching an observed performance issue with wildcard
queries, it was observed that the LiteralPrefix() method on
the regexp.Regexp struct did not always behave as expected.

In particular, when the pattern starts with ^, AND involves
some backtracking, the LiteralPrefix() seems to always be the
empty string.

The side-effect of this is that we rely on having a helpful
prefix, to reduce the number of terms in the term dictionary
that need to be visited.

This change now makes the searcher enforce start/end on the term
directly, by using FindStringIndex() instead of Match().
Next, we also modified WildcardQuery and RegexpQuery to no
longer include the ^ and $ modifiers.

Documentation was also udpated to instruct users that they should
not include the ^ and $ modifiers in their patterns.

search/query/regexp.go

@@ -33,7 +33,9 @@ type RegexpQuery struct {
 
 // NewRegexpQuery creates a new Query which finds
 // documents containing terms that match the
-// specified regular expression.
+// specified regular expression.  The regexp pattern
+// SHOULD NOT include ^ or $ modifiers, the search
+// will only match entire terms even without them.
 func NewRegexpQuery(regexp string) *RegexpQuery {
 	return &RegexpQuery{
 		Regexp: regexp,
@@ -76,14 +78,14 @@ func (q *RegexpQuery) Validate() error {
 
 func (q *RegexpQuery) compile() error {
 	if q.compiled == nil {
-		// require that pattern be anchored to start and end of term
+		// require that pattern NOT be anchored to start and end of term
 		actualRegexp := q.Regexp
-		if !strings.HasPrefix(actualRegexp, "^") {
-			actualRegexp = "^" + actualRegexp
-		}
-		if !strings.HasSuffix(actualRegexp, "$") {
-			actualRegexp = actualRegexp + "$"
+		if strings.HasPrefix(actualRegexp, "^") {
+			actualRegexp = actualRegexp[1:] // remove leading ^
 		}
+		// do not attempt to remove trailing $, it's presence is not
+		// known to interfere with LiteralPrefix() the way ^ does
+		// and removing $ introduces possible ambiguities with escaped \$, \\$, etc
 		var err error
 		q.compiled, err = regexp.Compile(actualRegexp)
 		if err != nil {

search/query/wildcard.go

@@ -101,6 +101,6 @@ func (q *WildcardQuery) Validate() error {
 }
 
 func (q *WildcardQuery) convertToRegexp() (*regexp.Regexp, error) {
-	regexpString := "^" + wildcardRegexpReplacer.Replace(q.Wildcard) + "$"
+	regexpString := wildcardRegexpReplacer.Replace(q.Wildcard)
 	return regexp.Compile(regexpString)
 }

search/searcher/search_regexp.go

@@ -21,6 +21,11 @@ import (
 	"github.com/blevesearch/bleve/search"
 )
 
+// NewRegexpSearcher creates a searcher which will match documents that
+// contain terms which match the pattern regexp.  The match must be EXACT
+// matching the entire term.  The provided regexp SHOULD NOT start with ^
+// or end with $ as this can intefere with the implementation.  Separately,
+// matches will be checked to ensure they match the entire term.
 func NewRegexpSearcher(indexReader index.IndexReader, pattern *regexp.Regexp, field string, boost float64, options search.SearcherOptions) (search.Searcher, error) {
 
 	prefixTerm, complete := pattern.LiteralPrefix()
@@ -79,7 +84,8 @@ func findRegexpCandidateTerms(indexReader index.IndexReader, pattern *regexp.Reg
 	// enumerate the terms and check against regexp
 	tfd, err := fieldDict.Next()
 	for err == nil && tfd != nil {
-		if pattern.MatchString(tfd.Term) {
+		matchPos := pattern.FindStringIndex(tfd.Term)
+		if matchPos != nil && matchPos[0] == 0 && matchPos[1] == len(tfd.Term) {
 			rv = append(rv, tfd.Term)
 			if tooManyClauses(len(rv)) {
 				return rv, tooManyClausesErr()