XLM-R tokenizer, return correct unk id for corrupted input #1073

	name: Rust

	on: [push, pull_request]

	jobs:
	fmt:
	name: Rustfmt
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2
	- uses: actions-rs/toolchain@v1
	with:
	profile: minimal
	toolchain: stable
	override: true
	components: rustfmt
	- uses: actions-rs/cargo@v1
	with:
	command: fmt
	args: --all -- --check

	clippy:
	name: Clippy
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v1
	- uses: actions/cache@v2
	with:
	path: \|
	~/.cargo/registry
	~/.cargo/git
	target
	key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }}
	- uses: actions-rs/toolchain@v1
	with:
	profile: minimal
	toolchain: stable
	override: true
	components: clippy
	- uses: tensordot/[email protected]
	- uses: actions-rs/cargo@v1
	with:
	command: clippy
	args: -- -D warnings

	test:
	name: Test
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v1
	- uses: actions/cache@v2
	with:
	path: \|
	~/.cargo/registry
	~/.cargo/git
	target
	key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
	- uses: actions-rs/toolchain@v1
	with:
	profile: minimal
	toolchain: stable
	override: true
	- uses: tensordot/[email protected]
	- uses: actions-rs/cargo@v1
	with:
	command: test

Provide feedback