Docstring for accents

google-research · Aug 1, 2020 · ce4b869 · ce4b869
1 parent 9dd0028
commit ce4b869
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/model/tokenization.py b/model/tokenization.py
@@ -113,6 +113,13 @@ class FullTokenizer(object):
   """Runs end-to-end tokenziation."""
 
   def __init__(self, vocab_file, do_lower_case=True, strip_accents=True):
+    """Constructs a FullTokenizer.
+
+    Args:
+      vocab_file: The vocabulary file.
+      do_lower_case: Whether to lower case the input.
+      strip_accents: Whether to strip the accents.
+    """
     self.vocab = load_vocab(vocab_file)
     self.inv_vocab = {v: k for k, v in self.vocab.items()}
     self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case, strip_accents=strip_accents)