added test case clarifying whitespace tokenizer on empty input

2014-08-19 10:43:52 -04:00 · 2014-08-19 10:43:52 -04:00 · b48dc87afa
commit b48dc87afa
parent a08a7f5b2a
1 changed files with 4 additions and 0 deletions
--- a/analysis/tokenizers/regexp_tokenizer/regexp_tokenizer_test.go
+++ b/analysis/tokenizers/regexp_tokenizer/regexp_tokenizer_test.go
@ -43,6 +43,10 @@ func TestBoundary(t *testing.T) {
 				},
 			},
 		},
+		{
+			[]byte(""),
+			analysis.TokenStream{},
+		},
 	}

 	for _, test := range tests {