Add support for Transformers and TikToken #5

	name: Python application

	on:
	push:
	branches: [ "main" ]
	pull_request:
	branches: [ "main" ]

	permissions:
	contents: read

	jobs:
	build:
	runs-on: ubuntu-latest

	steps:
	- uses: actions/checkout@v4

	- name: Set up Python 3.10
	uses: actions/setup-python@v3
	with:
	python-version: "3.10"

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install flake8 pytest
	# Install the package in editable mode with all extras
	pip install -e .[all,dev]
	# If you have any test-specific requirements
	pip install pytest-cov pytest-mock

	- name: Lint with flake8
	run: \|
	# stop the build if there are Python syntax errors or undefined names
	flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
	# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
	flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics

	- name: Test with pytest
	run: \|
	pytest -v --cov=chonkie tests/

Provide feedback