From ff03874f19862eef24431c717e476822f085bef4 Mon Sep 17 00:00:00 2001
From: Patrick Mezard <patrick@mezard.eu>
Date: Thu, 5 Nov 2015 14:07:54 +0100
Subject: [PATCH] token_map: document it along with stop_token_filter

---
 .../stop_tokens_filter/stop_tokens_filter.go          |  7 +++++++
 analysis/token_map/custom.go                          | 11 +++++++++++
 analysis/type.go                                      |  1 +
 3 files changed, 19 insertions(+)

diff --git a/analysis/token_filters/stop_tokens_filter/stop_tokens_filter.go b/analysis/token_filters/stop_tokens_filter/stop_tokens_filter.go
index de3970da..ed9890c1 100644
--- a/analysis/token_filters/stop_tokens_filter/stop_tokens_filter.go
+++ b/analysis/token_filters/stop_tokens_filter/stop_tokens_filter.go
@@ -7,6 +7,13 @@
 //  either express or implied. See the License for the specific language governing permissions
 //  and limitations under the License.
 
+// package stop_tokens_filter implements a TokenFilter removing tokens found in
+// a TokenMap.
+//
+// It constructor takes the following arguments:
+//
+// "stop_token_map" (string): the name of the token map identifying tokens to
+// remove.
 package stop_tokens_filter
 
 import (
diff --git a/analysis/token_map/custom.go b/analysis/token_map/custom.go
index eb5ccb68..89c7c61d 100644
--- a/analysis/token_map/custom.go
+++ b/analysis/token_map/custom.go
@@ -7,6 +7,17 @@
 //  either express or implied. See the License for the specific language governing permissions
 //  and limitations under the License.
 
+// package token_map implements a generic TokenMap, often used in conjunction
+// with filters to remove or process specific tokens.
+//
+// Its constructor takes the following arguments:
+//
+// "filename" (string): the path of a file listing the tokens. Each line may
+// contain one or more whitespace separated tokens, followed by an optional
+// comment starting with a "#" or "|" character.
+//
+// "tokens" ([]interface{}): if "filename" is not specified, tokens can be
+// passed directly as a sequence of strings wrapped in a []interface{}.
 package token_map
 
 import (
diff --git a/analysis/type.go b/analysis/type.go
index a8feeabd..c5f88ffd 100644
--- a/analysis/type.go
+++ b/analysis/type.go
@@ -60,6 +60,7 @@ type Tokenizer interface {
 	Tokenize([]byte) TokenStream
 }
 
+// A TokenFilter adds, transforms or removes tokens from a token stream.
 type TokenFilter interface {
 	Filter(TokenStream) TokenStream
 }