count_text_tokens.yaml

# example call:
# python src/evaluate_documents.py dataset=conll2003 metric=count_text_tokens

_target_: pytorch_ie.metrics.statistics.TokenCountCollector
title: text length (tokens)
text_field: text
tokenizer: bert-base-uncased
tokenizer_kwargs:
  add_special_tokens: false
  # strict_span_conversion: false
show_histogram: true
show_as_markdown: true
document_type: pytorch_ie.documents.TextBasedDocument