added test case clarifying whitespace tokenizer on empty input
This commit is contained in:
parent
a08a7f5b2a
commit
b48dc87afa
|
@ -43,6 +43,10 @@ func TestBoundary(t *testing.T) {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
[]byte(""),
|
||||||
|
analysis.TokenStream{},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
|
|
Loading…
Reference in New Issue
Block a user