Support large dataset preprocessing #49

	name: Pre-Commit Checks

	on:
	pull_request:
	push:
	branches: [main]

	jobs:
	pre-commit:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- uses: actions/setup-python@v4
	with:
	python-version: "3.10"
	cache: "pip"
	- name: Install requirements
	run: \|
	pip install -U pip
	pip install pylint
	pip install -U black
	pip install .[dev]
	pip install wandb
	pip install tqdm
	- name: Run black
	run: \|
	python -m black .
	- uses: pre-commit/[email protected]

Provide feedback