pyproject.toml

# SPDX-License-Identifier: Apache-2.0

[build-system]
requires = ["setuptools>=64", "setuptools_scm>=8"]
build-backend = "setuptools.build_meta"

[project]
name = "instructlab-eval"
authors = [
    { name="InstructLab", email="dev@instructlab.ai" },
]
description = "Evaluation"
readme = "README.md"
license = {text = "Apache-2.0"}
requires-python = ">=3.9"
classifiers = [
    "Development Status :: 3 - Alpha",
    "Environment :: Console",
    "License :: OSI Approved :: Apache Software License",
    "License :: OSI Approved :: MIT License",
    "Operating System :: MacOS :: MacOS X",
    "Operating System :: POSIX :: Linux",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: Implementation :: CPython",
]
dynamic = ["dependencies", "optional-dependencies", "version"]

[project.scripts]

[project.urls]
homepage = "https://instructlab.ai"
source = "https://github.com/instructlab/eval"
issues = "https://github.com/instructlab/eval/issues"

[project.entry-points."instructlab.eval.evaluator"]
"mmlu" = "instructlab.eval.mmlu:MMLUEvaluator"
"mmlu_branch" = "instructlab.eval.mmlu:MMLUBranchEvaluator"
"mt_bench" = "instructlab.eval.mt_bench:MTBenchEvaluator"
"mt_bench_branch" = "instructlab.eval.mt_bench:MTBenchBranchEvaluator"

[tool.setuptools_scm]
version_file = "src/instructlab/eval/_version.py"
# do not include +gREV local version, required for Test PyPI upload
local_scheme = "no-local-version"

[tool.setuptools]
package-dir = {"" = "src"}

[tool.setuptools.dynamic]
dependencies = {file = ["requirements.txt"]}

[tool.setuptools.packages.find]
where = ["src"]
include = ["instructlab.eval"]

[tool.ruff]
target-version = "py39"
# same as black's default line length
line-length = 88

[tool.ruff.lint]
# Allow fix for all enabled rules (when `--fix`) is provided.
fixable = ["ALL"]
unfixable = []

# Fixers will be enabled gradually.
select = [
    # "B",  # flake8-bugbear
    # "E",  # pycodestyle
    # "F",  # Pyflakes
    "Q",  # flake8-quotes
    # Ruff does not support isort's import_headings feature, yet.
    # "I",  # isort
    # "UP",  # pyupgrade
    # "SIM",  # flake8-simplify
    "TID",  # flake8-tidy-imports
]
ignore = [
    # some embedded strings are longer than 88 characters
    "E501",  # line too long
    "TID252",  # Prefer absolute imports over relative imports from parent modules
]

[tool.ruff.lint.isort]
# same as .isort.cfg
from-first = true
# not supported yet
# import-heading-future=Future
# import-heading-stdlib=Standard
# import-heading-thirdparty=Third Party
# import-heading-firstparty=First Party
# import-heading-localfolder=Local
known-local-folder = ["tuning"]

[tool.mypy]
ignore_missing_imports = true