Add batched decorator #31

	name: Test Python package
	on:
	push:
	branches:
	- main
	pull_request:
	workflow_dispatch:

	# When this workflow is queued, automatically cancel any previous running
	# or pending jobs from the same branch
	concurrency:
	group: test-${{ github.ref }}
	cancel-in-progress: true

	jobs:
	build_and_test:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest]
	python-version: ["3.10"]
	steps:
	- uses: actions/checkout@v4
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install NeMo-Curator and pytest
	# TODO: Remove pytest when optional test dependencies are added to setup.py

	# Installing wheel beforehand due to fasttext issue:
	# https://github.com/facebookresearch/fastText/issues/512#issuecomment-1837367666
	# Explicitly install cython: https://github.com/VKCOM/YouTokenToMe/issues/94
	run: \|
	pip install wheel cython
	pip install --no-cache-dir --extra-index-url https://pypi.nvidia.com .
	pip install pytest
	- name: Run tests
	# TODO: Remove env variable when gpu dependencies are optional
	run: \|
	RAPIDS_NO_INITIALIZE=1 python -m pytest -v --cpu

Provide feedback