diff --git a/pyproject.toml b/pyproject.toml index ed8e8cd..9753c36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "poodle" description = "Mutation Testing Tool" -version = "1.2.2" +version = "1.3.0" license = { file = "LICENSE" } keywords = [ "test", diff --git a/src/poodle/__init__.py b/src/poodle/__init__.py index 4c73286..086d1e2 100644 --- a/src/poodle/__init__.py +++ b/src/poodle/__init__.py @@ -8,7 +8,15 @@ from pathlib import Path from typing import Any -__version__ = "1.2.2" +__version__ = "1.3.0" + + +class PoodleTestingFailedError(Exception): + """Poodle testing failed.""" + + +class PoodleNoMutantsFoundError(Exception): + """Poodle could not find any mutants to test.""" class PoodleInputError(ValueError): diff --git a/src/poodle/cli.py b/src/poodle/cli.py index a5e8499..c113ef4 100644 --- a/src/poodle/cli.py +++ b/src/poodle/cli.py @@ -8,7 +8,14 @@ import click -from . import PoodleInputError, core +from . import ( + PoodleInputError, + PoodleNoMutantsFoundError, + PoodleTestingFailedError, + PoodleTrialRunError, + __version__, + core, +) from .config import build_config CONTEXT_SETTINGS = { @@ -27,7 +34,9 @@ @click.option("--report", help="Enable reporter by name. Multiple allowed.", multiple=True) @click.option("--html", help="Folder name to store HTML report in.", type=click.Path(path_type=Path)) @click.option("--json", help="File to create with JSON report.", type=click.Path(path_type=Path)) -def main( +@click.option("--fail_under", help="Fail if mutation score is under this value.", type=float) +@click.version_option(version=__version__) +def main( # noqa: C901, PLR0912 sources: tuple[Path], config_file: Path | None, quiet: int, @@ -38,19 +47,39 @@ def main( report: tuple[str], html: Path | None, json: Path | None, + fail_under: float | None, ) -> None: """Poodle Mutation Test Tool.""" try: - config = build_config(sources, config_file, quiet, verbose, workers, exclude, only, report, html, json) + config = build_config( + sources, config_file, quiet, verbose, workers, exclude, only, report, html, json, fail_under + ) except PoodleInputError as err: - click.echo(err.args) + for arg in err.args: + click.echo(arg) sys.exit(4) try: core.main_process(config) + except PoodleTestingFailedError as err: + for arg in err.args: + click.echo(arg) + sys.exit(1) except KeyboardInterrupt: click.echo("Aborted due to Keyboard Interrupt!") sys.exit(2) + except PoodleTrialRunError as err: + for arg in err.args: + click.echo(arg) + sys.exit(3) + except PoodleInputError as err: + for arg in err.args: + click.echo(arg) + sys.exit(4) + except PoodleNoMutantsFoundError as err: + for arg in err.args: + click.echo(arg) + sys.exit(5) except: # noqa: E722 click.echo("Aborted due to Internal Error!") click.echo(traceback.format_exc()) @@ -58,15 +87,6 @@ def main( sys.exit(0) -# pytest return codes -# Exit code 0: All tests were collected and passed successfully -# Exit code 1: Tests were collected and run but some of the tests failed -# Exit code 2: Test execution was interrupted by the user -# Exit code 3: Internal error happened while executing tests -# Exit code 4: pytest command line usage error -# Exit code 5: No tests were collected - - # nomut: start if __name__ == "__main__": main() diff --git a/src/poodle/config.py b/src/poodle/config.py index 8b2b952..14902f5 100644 --- a/src/poodle/config.py +++ b/src/poodle/config.py @@ -53,6 +53,7 @@ def build_config( # noqa: PLR0913 cmd_report: tuple[str], cmd_html: Path | None, cmd_json: Path | None, + cmd_fail_under: float | None, ) -> PoodleConfig: """Build PoodleConfig object.""" config_file_path = get_config_file_path(cmd_config_file) @@ -118,6 +119,7 @@ def build_config( # noqa: PLR0913 runner_opts=get_dict_from_config("runner_opts", config_file_data), reporters=get_reporters(config_file_data, cmd_report, cmd_html, cmd_json), reporter_opts=get_dict_from_config("reporter_opts", config_file_data, command_line=cmd_reporter_opts), + fail_under=get_float_from_config("fail_under", config_file_data, command_line=cmd_fail_under), ) @@ -414,6 +416,28 @@ def get_int_from_config( raise PoodleInputError(msg) from None +def get_float_from_config( + option_name: str, + config_data: dict, + default: float | None = None, + command_line: float | None = None, +) -> float | None: + """Retrieve Config Option that should be an float or None. + + Retrieve highest priority value from config sources. + """ + value, source = get_option_from_config(option_name=option_name, config_data=config_data, command_line=command_line) + + if value is None: + return default + + try: + return float(value) + except ValueError: + msg = f"{option_name} from {source} must be a valid float" + raise PoodleInputError(msg) from None + + def get_str_from_config( option_name: str, config_data: dict, diff --git a/src/poodle/core.py b/src/poodle/core.py index 8b577db..01fb681 100644 --- a/src/poodle/core.py +++ b/src/poodle/core.py @@ -5,9 +5,7 @@ import logging import shutil -import click - -from . import PoodleInputError, PoodleTrialRunError, __version__ +from . import PoodleNoMutantsFoundError, PoodleTestingFailedError, __version__ from .data_types import PoodleConfig, PoodleWork from .mutate import create_mutants_for_all_mutators, initialize_mutators from .report import generate_reporters @@ -19,42 +17,41 @@ def main_process(config: PoodleConfig) -> None: """Poodle core run process.""" - try: - work = PoodleWork(config) # sets logging defaults - print_header(work) - logger.info("\n%s", pprint_str(config)) + work = PoodleWork(config) # sets logging defaults + print_header(work) + logger.info("\n%s", pprint_str(config)) + + if config.work_folder.exists(): + logger.info("delete %s", config.work_folder) + shutil.rmtree(config.work_folder) - if config.work_folder.exists(): - logger.info("delete %s", config.work_folder) - shutil.rmtree(config.work_folder) + create_temp_zips(work) - create_temp_zips(work) + work.mutators = initialize_mutators(work) + work.runner = get_runner(config) + work.reporters = list(generate_reporters(config)) - work.mutators = initialize_mutators(work) - work.runner = get_runner(config) - work.reporters = list(generate_reporters(config)) + mutants = create_mutants_for_all_mutators(work) + if not mutants: + raise PoodleNoMutantsFoundError("No mutants were found to test!") + work.echo(f"Identified {len(mutants)} mutants") - mutants = create_mutants_for_all_mutators(work) - work.echo(f"Identified {len(mutants)} mutants") + clean_run_results = clean_run_each_source_folder(work) + timeout = calc_timeout(config, clean_run_results) + results = run_mutant_trails(work, mutants, timeout) - clean_run_results = clean_run_each_source_folder(work) - timeout = calc_timeout(config, clean_run_results) - results = run_mutant_trails(work, mutants, timeout) + for trial in results.mutant_trials: + trial.mutant.unified_diff = create_unified_diff(trial.mutant) - for trial in results.mutant_trials: - trial.mutant.unified_diff = create_unified_diff(trial.mutant) + for reporter in work.reporters: + reporter(config=config, echo=work.echo, testing_results=results) - for reporter in work.reporters: - reporter(config=config, echo=work.echo, testing_results=results) + logger.info("delete %s", config.work_folder) + shutil.rmtree(config.work_folder) - logger.info("delete %s", config.work_folder) - shutil.rmtree(config.work_folder) - except PoodleInputError as err: - for arg in err.args: - click.echo(arg) - except PoodleTrialRunError as err: - for arg in err.args: - click.echo(arg) + if config.fail_under and results.summary.success_rate < config.fail_under / 100: + msg = f"Mutation score {results.summary.coverage_display} is below {config.fail_under:.2f}%" + raise PoodleTestingFailedError(msg) poodle_header_str = r""" diff --git a/src/poodle/data_types/data.py b/src/poodle/data_types/data.py index 90936b9..e3b1ea0 100644 --- a/src/poodle/data_types/data.py +++ b/src/poodle/data_types/data.py @@ -61,6 +61,8 @@ class PoodleConfig: reporters: list[str] reporter_opts: dict + fail_under: float | None + @dataclass class FileMutation: diff --git a/tests/data_types/test_data.py b/tests/data_types/test_data.py index 0b98863..681dac8 100644 --- a/tests/data_types/test_data.py +++ b/tests/data_types/test_data.py @@ -52,6 +52,8 @@ class PoodleConfigStub(PoodleConfig): reporters: list[str] = None # type: ignore [assignment] reporter_opts: dict = None # type: ignore [assignment] + fail_under: float | None = None + class TestPoodleConfig: @staticmethod @@ -81,6 +83,7 @@ def create_poodle_config(): runner_opts={"command_line": "pytest tests"}, reporters=["summary"], reporter_opts={"summary": "value"}, + fail_under=95.0, ) def test_poodle_config(self): @@ -119,6 +122,8 @@ def test_poodle_config(self): assert config.reporters == ["summary"] assert config.reporter_opts == {"summary": "value"} + assert config.fail_under == 95.0 + class TestFileMutation: @staticmethod diff --git a/tests/test_cli.py b/tests/test_cli.py index 94b8866..9ebe6f1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -8,33 +8,38 @@ import pytest from click.testing import CliRunner -from poodle import PoodleInputError, cli +from poodle import PoodleInputError, PoodleNoMutantsFoundError, PoodleTestingFailedError, PoodleTrialRunError, cli -class TestCli: - @pytest.fixture(autouse=True) - def _setup(self): - importlib.reload(cli) +@pytest.fixture(autouse=True) +def _setup(): + importlib.reload(cli) - @pytest.fixture() - def main_process(self): - with mock.patch("poodle.cli.core.main_process") as main_process: - yield main_process - @pytest.fixture() - def build_config(self): - with mock.patch("poodle.cli.build_config") as build_config: - yield build_config +@pytest.fixture() +def main_process(): + with mock.patch("poodle.cli.core.main_process") as main_process: + yield main_process - @pytest.fixture() - def echo(self): - with mock.patch("poodle.cli.click.echo") as echo: - yield echo - @pytest.fixture() - def runner(self): - return CliRunner() +@pytest.fixture() +def build_config(): + with mock.patch("poodle.cli.build_config") as build_config: + yield build_config + +@pytest.fixture() +def echo(): + with mock.patch("poodle.cli.click.echo") as echo: + yield echo + + +@pytest.fixture() +def runner(): + return CliRunner() + + +class TestCliHelp: def test_cli_help(self, main_process: mock.MagicMock, build_config: mock.MagicMock, runner: CliRunner): result = runner.invoke(cli.main, ["--help"]) assert result.exit_code == 0 @@ -136,6 +141,20 @@ def test_cli_help_json(self, runner: CliRunner): is not None ) + def test_cli_help_fail_under(self, runner: CliRunner): + result = runner.invoke(cli.main, ["--help"]) + assert result.exit_code == 0 + assert ( + re.match( + r".*--fail_under FLOAT\s+Fail if mutation score is under this value\..*", + result.output, + flags=re.DOTALL, + ) + is not None + ) + + +class TestInputs: def assert_build_config_called_with( self, build_config: mock.MagicMock, @@ -149,6 +168,7 @@ def assert_build_config_called_with( report: tuple[str] = (), # type: ignore [assignment] html: Path | None = None, json: Path | None = None, + fail_under: float | None = None, ): build_config.assert_called_with( sources, @@ -161,6 +181,7 @@ def assert_build_config_called_with( report, html, json, + fail_under, ) def test_cli(self, main_process: mock.MagicMock, build_config: mock.MagicMock, runner: CliRunner): @@ -263,21 +284,53 @@ def test_main_json(self, main_process: mock.MagicMock, build_config: mock.MagicM self.assert_build_config_called_with(build_config, json=Path("summary.json")) main_process.assert_called_with(build_config.return_value) - def test_main_input_error( + def test_main_fail_under(self, main_process: mock.MagicMock, build_config: mock.MagicMock, runner: CliRunner): + result = runner.invoke(cli.main, ["--fail_under", "80"]) + assert result.exit_code == 0 + self.assert_build_config_called_with(build_config, fail_under=80) + main_process.assert_called_with(build_config.return_value) + + +class TestErrors: + def test_main_build_config_input_error( self, main_process: mock.MagicMock, echo: mock.MagicMock, build_config: mock.MagicMock, runner: CliRunner, ): - build_config.side_effect = PoodleInputError("bad input") + build_config.side_effect = PoodleInputError("bad input", "input error") result = runner.invoke(cli.main, []) assert result.exit_code == 4 build_config.assert_called() - echo.assert_called_with(("bad input",)) + echo.assert_has_calls( + [ + mock.call("bad input"), + mock.call("input error"), + ] + ) main_process.assert_not_called() - def test_main_keyboard_interrupt( + def test_main_process_testing_failed( + self, + main_process: mock.MagicMock, + echo: mock.MagicMock, + build_config: mock.MagicMock, + runner: CliRunner, + ): + main_process.side_effect = PoodleTestingFailedError("testing failed", "error message") + result = runner.invoke(cli.main, []) + assert result.exit_code == 1 + build_config.assert_called() + echo.assert_has_calls( + [ + mock.call("testing failed"), + mock.call("error message"), + ] + ) + main_process.assert_called_with(build_config.return_value) + + def test_main_process_keyboard_interrupt( self, main_process: mock.MagicMock, echo: mock.MagicMock, @@ -288,11 +341,68 @@ def test_main_keyboard_interrupt( result = runner.invoke(cli.main, []) assert result.exit_code == 2 build_config.assert_called() - echo.assert_called_with("Aborted due to Keyboard Interrupt!") + echo.assert_called_once_with("Aborted due to Keyboard Interrupt!") + main_process.assert_called_with(build_config.return_value) + + def test_main_process_trial_run_error( + self, + main_process: mock.MagicMock, + echo: mock.MagicMock, + build_config: mock.MagicMock, + runner: CliRunner, + ): + main_process.side_effect = PoodleTrialRunError("testing failed", "error message") + result = runner.invoke(cli.main, []) + assert result.exit_code == 3 + build_config.assert_called() + echo.assert_has_calls( + [ + mock.call("testing failed"), + mock.call("error message"), + ] + ) + main_process.assert_called_with(build_config.return_value) + + def test_main_process_input_error( + self, + main_process: mock.MagicMock, + echo: mock.MagicMock, + build_config: mock.MagicMock, + runner: CliRunner, + ): + main_process.side_effect = PoodleInputError("testing failed", "error message") + result = runner.invoke(cli.main, []) + assert result.exit_code == 4 + build_config.assert_called() + echo.assert_has_calls( + [ + mock.call("testing failed"), + mock.call("error message"), + ] + ) + main_process.assert_called_with(build_config.return_value) + + def test_main_process_no_mutants_error( + self, + main_process: mock.MagicMock, + echo: mock.MagicMock, + build_config: mock.MagicMock, + runner: CliRunner, + ): + main_process.side_effect = PoodleNoMutantsFoundError("testing failed", "error message") + result = runner.invoke(cli.main, []) + assert result.exit_code == 5 + build_config.assert_called() + echo.assert_has_calls( + [ + mock.call("testing failed"), + mock.call("error message"), + ] + ) main_process.assert_called_with(build_config.return_value) @mock.patch("poodle.cli.traceback") - def test_main_other_error( + def test_main_process_other_error( self, traceback: mock.MagicMock, main_process: mock.MagicMock, diff --git a/tests/test_config.py b/tests/test_config.py index 109f3b7..0e98181 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -152,6 +152,11 @@ def get_int_from_config(self): with mock.patch("poodle.config.get_int_from_config") as get_int_from_config: yield get_int_from_config + @pytest.fixture() + def get_float_from_config(self): + with mock.patch("poodle.config.get_float_from_config") as get_float_from_config: + yield get_float_from_config + @pytest.fixture() def get_any_from_config(self): with mock.patch("poodle.config.get_any_from_config") as get_any_from_config: @@ -223,6 +228,7 @@ def build_config_with( cmd_report: tuple[str] = (), # type: ignore[assignment] cmd_html: Path | None = None, cmd_json: Path | None = None, + cmd_fail_under: float | None = None, ): return config.build_config( cmd_sources, @@ -235,6 +241,7 @@ def build_config_with( cmd_report, cmd_html, cmd_json, + cmd_fail_under, ) def test_build_config_project_info(self, get_project_info: mock.MagicMock): @@ -446,6 +453,13 @@ def test_build_config_reporter_opts(self, get_dict_from_config, get_config_file_ command_line={"json_report_file": Path("json"), "html": {"report_folder": Path("html")}}, ) + @pytest.mark.usefixtures("_setup_build_config_mocks") + def test_build_config_fail_under(self, get_float_from_config, get_config_file_data): + config_file_data = get_config_file_data.return_value + config_data = self.build_config_with(cmd_fail_under=50) + assert config_data.fail_under == get_float_from_config.return_value + get_float_from_config.assert_any_call("fail_under", config_file_data, command_line=50) + @mock.patch("poodle.config.get_config_file_data") @mock.patch("poodle.config.get_project_info") def test_build_config_defaults(self, get_project_info, get_config_file_data): @@ -463,6 +477,7 @@ def test_build_config_defaults(self, get_project_info, get_config_file_data): cmd_report=(), cmd_html=None, cmd_json=None, + cmd_fail_under=None, ) == config.PoodleConfig( project_name=None, project_version=None, @@ -488,6 +503,7 @@ def test_build_config_defaults(self, get_project_info, get_config_file_data): runner_opts={}, reporters=["summary", "not_found"], reporter_opts={}, + fail_under=None, ) @@ -1191,6 +1207,68 @@ def test_convert_error(self, get_option_from_config): ) +class TestGetFloatFromConfig: + def test_default(self, get_option_from_config): + get_option_from_config.return_value = (None, None) + + assert ( + config.get_float_from_config( + option_name="test_option", + config_data={"test_option": 3}, + command_line=4, + default=5, + ) + == 5.0 + ) + + get_option_from_config.assert_called_with( + option_name="test_option", + config_data={"test_option": 3}, + command_line=4, + ) + + def test_default_inputs(self, get_option_from_config): + get_option_from_config.return_value = (None, None) + + assert ( + config.get_float_from_config( + option_name="test_option", + config_data={"test_option": 3}, + ) + is None + ) + + get_option_from_config.assert_called_with( + option_name="test_option", + config_data={"test_option": 3}, + command_line=None, + ) + + def test_str_to_float(self, get_option_from_config): + get_option_from_config.return_value = ("5", "Source Name") + + assert ( + config.get_float_from_config( + option_name="test_option", + config_data={"test_option": "3"}, + command_line="4", + default="5", + ) + == 5 + ) + + def test_convert_error(self, get_option_from_config): + get_option_from_config.return_value = ("a", "Source Name") + + with pytest.raises(ValueError, match="^test_option from Source Name must be a valid float$"): + config.get_float_from_config( + option_name="test_option", + config_data={"test_option": "3"}, + command_line="4", + default="5", + ) + + class TestGetStrFromConfig: def test_default(self, get_option_from_config): get_option_from_config.return_value = (None, None) diff --git a/tests/test_core.py b/tests/test_core.py index 752733c..1249bac 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -2,7 +2,7 @@ import pytest -from poodle import PoodleInputError, PoodleTrialRunError, core +from poodle import PoodleNoMutantsFoundError, PoodleTestingFailedError, core from poodle.data_types import MutantTrial, PoodleWork from tests.data_types.test_data import PoodleConfigStub @@ -28,12 +28,10 @@ class TestMain: @mock.patch("poodle.core.create_mutants_for_all_mutators") @mock.patch("poodle.core.clean_run_each_source_folder") @mock.patch("poodle.core.run_mutant_trails") - @mock.patch("poodle.core.click") @mock.patch("poodle.core.print_header") def test_main( self, print_header: mock.MagicMock, - core_click: mock.MagicMock, run_mutant_trails: mock.MagicMock, clean_run_each_source_folder: mock.MagicMock, create_mutants_for_all_mutators: mock.MagicMock, @@ -113,10 +111,8 @@ def test_main( @mock.patch("poodle.core.create_mutants_for_all_mutators") @mock.patch("poodle.core.clean_run_each_source_folder") @mock.patch("poodle.core.run_mutant_trails") - @mock.patch("poodle.core.click") def test_main_not_exists( self, - core_click: mock.MagicMock, run_mutant_trails: mock.MagicMock, clean_run_each_source_folder: mock.MagicMock, create_mutants_for_all_mutators: mock.MagicMock, @@ -166,10 +162,8 @@ def test_main_not_exists( @mock.patch("poodle.core.create_mutants_for_all_mutators") @mock.patch("poodle.core.clean_run_each_source_folder") @mock.patch("poodle.core.run_mutant_trails") - @mock.patch("poodle.core.click") def test_input_error( self, - core_click: mock.MagicMock, run_mutant_trails: mock.MagicMock, clean_run_each_source_folder: mock.MagicMock, create_mutants_for_all_mutators: mock.MagicMock, @@ -182,12 +176,13 @@ def test_input_error( poodle_work_class: mock.MagicMock, logger_mock: mock.MagicMock, ): - poodle_work_class.side_effect = [PoodleInputError("Input Error", "Bad Input")] + work_folder = mock.MagicMock() + work_folder.exists.return_value = False - core.main_process(PoodleConfigStub()) + create_mutants_for_all_mutators.return_value = [] - core_click.echo.assert_any_call("Input Error") - core_click.echo.assert_any_call("Bad Input") + with pytest.raises(PoodleNoMutantsFoundError, match="^No mutants were found to test!$"): + core.main_process(PoodleConfigStub(work_folder=work_folder)) @mock.patch("poodle.core.PoodleWork") @mock.patch("poodle.core.pprint_str") @@ -199,10 +194,8 @@ def test_input_error( @mock.patch("poodle.core.create_mutants_for_all_mutators") @mock.patch("poodle.core.clean_run_each_source_folder") @mock.patch("poodle.core.run_mutant_trails") - @mock.patch("poodle.core.click") - def test_trial_error( + def test_fail_under_error( self, - core_click: mock.MagicMock, run_mutant_trails: mock.MagicMock, clean_run_each_source_folder: mock.MagicMock, create_mutants_for_all_mutators: mock.MagicMock, @@ -215,12 +208,25 @@ def test_trial_error( poodle_work_class: mock.MagicMock, logger_mock: mock.MagicMock, ): - poodle_work_class.side_effect = [PoodleTrialRunError("Trial Error", "Execution Failed")] + work_folder = mock.MagicMock() + work_folder.exists.return_value = False - core.main_process(PoodleConfigStub()) + results = run_mutant_trails.return_value - core_click.echo.assert_any_call("Trial Error") - core_click.echo.assert_any_call("Execution Failed") + results.summary.coverage_display = "80.00%" + results.summary.success_rate = 0.8 + + clean_run_each_source_folder.return_value = {"folder": MutantTrial(mutant=None, result=None, duration=1.0)} # type: ignore [arg-type] + + with pytest.raises(PoodleTestingFailedError, match=r"^Mutation score 80.00% is below 99.00%$"): + core.main_process( + PoodleConfigStub( + work_folder=work_folder, + fail_under=99, + min_timeout=10, + timeout_multiplier=10, + ) + ) poodle_header_str = r""" diff --git a/tools.ipynb b/tools.ipynb index 1ad979a..71cf68e 100644 --- a/tools.ipynb +++ b/tools.ipynb @@ -36,7 +36,8 @@ "metadata": {}, "outputs": [], "source": [ - "!poodle --help" + "!poodle --help\n", + "!poodle --version" ] }, {