Add `ignore_merges` option to BPE tokenizers (#716) #297

	# Simple workflow for deploying site to GitHub Pages
	name: Build and Deploy Demo Website

	on:
	# Runs on pushes targeting the default branch
	push:
	branches: ["main"]

	# Allows you to run this workflow manually from the Actions tab
	workflow_dispatch:

	# Also trigger after the publish workflow has completed
	workflow_run:
	workflows: ["Publish Package to npmjs"]
	types:
	- completed

	# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
	permissions:
	contents: read
	pages: write
	id-token: write

	# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
	# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
	concurrency:
	group: "pages"
	cancel-in-progress: false

	# Set base path
	env:
	BASE_PATH: "/transformers.js/"

	jobs:
	# Single deploy job since we're just deploying
	deploy:
	environment:
	name: github-pages
	url: ${{ steps.deployment.outputs.page_url }}
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: ./examples/demo-site/
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Setup Pages
	uses: actions/configure-pages@v3

	- name: Build website
	run: \|
	npm install
	npm run build
	- name: Upload artifact
	uses: actions/upload-pages-artifact@v1
	with:
	# Upload built files
	path: './examples/demo-site/dist/'
	- name: Deploy to GitHub Pages
	id: deployment
	uses: actions/deploy-pages@v2

Provide feedback