added test case clarifying whitespace tokenizer on empty input
This commit is contained in:
parent
a08a7f5b2a
commit
b48dc87afa
|
@ -43,6 +43,10 @@ func TestBoundary(t *testing.T) {
|
|||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
[]byte(""),
|
||||
analysis.TokenStream{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
|
|
Loading…
Reference in New Issue
Block a user