Add support for ICU tokenizer #36

	name: Benchmark

	on:
	push:
	branches:
	- main
	pull_request:

	permissions:
	# deployments permission to deploy GitHub pages website
	deployments: write
	# contents permission to update benchmark contents in gh-pages branch
	contents: write

	jobs:
	benchmark:
	name: Performance regression check
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- uses: actions/setup-python@v3
	- name: Install dependencies
	run: pip install .
	- name: Run benchmark
	run: python contrib/benchmark.py > benchmark.json
	# Run `github-action-benchmark` action
	- name: Store benchmark result
	uses: benchmark-action/github-action-benchmark@v1
	with:
	# What benchmark tool the output.txt came from
	tool: 'customSmallerIsBetter'
	# Where the output from the benchmark tool is stored
	output-file-path: benchmark.json
	# Workflow will fail when an alert happens
	fail-on-alert: true
	# GitHub API token to make a commit comment
	github-token: ${{ secrets.GITHUB_TOKEN }}
	# Enable alert commit comment
	comment-on-alert: true
	# Enable Job Summary for PRs
	summary-always: true
	# Only push benchmark if this is the main branch
	auto-push: ${{ github.ref == 'refs/heads/main' }}

Provide feedback