From b48dc87afaa0b08fce7772c613d3d893b71713ee Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Tue, 19 Aug 2014 10:43:52 -0400 Subject: [PATCH] added test case clarifying whitespace tokenizer on empty input --- analysis/tokenizers/regexp_tokenizer/regexp_tokenizer_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/analysis/tokenizers/regexp_tokenizer/regexp_tokenizer_test.go b/analysis/tokenizers/regexp_tokenizer/regexp_tokenizer_test.go index eef0d77f..f6bc1a3c 100644 --- a/analysis/tokenizers/regexp_tokenizer/regexp_tokenizer_test.go +++ b/analysis/tokenizers/regexp_tokenizer/regexp_tokenizer_test.go @@ -43,6 +43,10 @@ func TestBoundary(t *testing.T) { }, }, }, + { + []byte(""), + analysis.TokenStream{}, + }, } for _, test := range tests {