diff --git a/bert/bert-large-japanese-v2/tokenizer_config.json b/bert/bert-large-japanese-v2/tokenizer_config.json new file mode 100644 index 000000000..dfbcc4690 --- /dev/null +++ b/bert/bert-large-japanese-v2/tokenizer_config.json @@ -0,0 +1,10 @@ +{ + "tokenizer_class": "BertJapaneseTokenizer", + "model_max_length": 512, + "do_lower_case": false, + "word_tokenizer_type": "mecab", + "subword_tokenizer_type": "wordpiece", + "mecab_kwargs": { + "mecab_dic": "unidic_lite" + } +}