diff --git a/.gitignore b/.gitignore index 119453dd5..4c2aee4e3 100644 --- a/.gitignore +++ b/.gitignore @@ -92,6 +92,7 @@ venv/ ENV/ env.bak/ venv.bak/ +.envrc # mkdocs documentation /site @@ -107,4 +108,4 @@ tech-support/* .*report.html clab-atd-anta/* -clab-atd-anta/ \ No newline at end of file +clab-atd-anta/ diff --git a/.vscode/settings.json b/.vscode/settings.json index 3b3ba1da7..d688a64d6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,23 +1,18 @@ { - "python.linting.enabled": true, - "python.linting.flake8Enabled": true, - "python.linting.mypyEnabled": true, - "python.formatting.provider": "black", + "black-formatter.importStrategy": "fromEnvironment", "pylint.importStrategy": "fromEnvironment", + "pylint.args": [ + "--rcfile=pylintrc" + ], "flake8.importStrategy": "fromEnvironment", - "black-formatter.importStrategy": "fromEnvironment", - "python.linting.flake8Args": [ + "flake8.args": [ "--config=/dev/null", "--max-line-length=165" ], - "python.linting.mypyArgs": [ + "mypy-type-checker.importStrategy": "fromEnvironment", + "mypy-type-checker.args": [ "--config-file=pyproject.toml" ], - "pylint.severity": { - "refactor": "Warning" - }, - "pylint.args": [ - "--load-plugins pylint_pydantic", - "--rcfile=pylintrc" - ] + "isort.importStrategy": "fromEnvironment", + "isort.check": true, } \ No newline at end of file diff --git a/anta/catalog.py b/anta/catalog.py new file mode 100644 index 000000000..ad0670bb8 --- /dev/null +++ b/anta/catalog.py @@ -0,0 +1,284 @@ +# Copyright (c) 2023 Arista Networks, Inc. +# Use of this source code is governed by the Apache License 2.0 +# that can be found in the LICENSE file. +""" +Catalog related functions +""" +from __future__ import annotations + +import importlib +import logging +from inspect import isclass +from pathlib import Path +from types import ModuleType +from typing import Any, Dict, List, Optional, Tuple, Type, Union + +from pydantic import BaseModel, ConfigDict, RootModel, ValidationError, ValidationInfo, field_validator, model_validator +from pydantic.types import ImportString +from yaml import YAMLError, safe_load + +from anta.models import AntaTest +from anta.tools.misc import anta_log_exception + +logger = logging.getLogger(__name__) + +# { : [ { : }, ... ] } +RawCatalogInput = Dict[str, List[Dict[str, Optional[Dict[str, Any]]]]] + +# [ ( , ), ... ] +ListAntaTestTuples = List[Tuple[Type[AntaTest], Optional[Union[AntaTest.Input, Dict[str, Any]]]]] + + +class AntaTestDefinition(BaseModel): + """ + Define a test with its associated inputs. + + test: An AntaTest concrete subclass + inputs: The associated AntaTest.Input subclass instance + """ + + model_config = ConfigDict(frozen=True) + + test: Type[AntaTest] + inputs: AntaTest.Input + + def __init__(self, **data: Any) -> None: + """ + Inject test in the context to allow to instantiate Input in the BeforeValidator + https://docs.pydantic.dev/2.0/usage/validators/#using-validation-context-with-basemodel-initialization + """ + self.__pydantic_validator__.validate_python( + data, + self_instance=self, + context={"test": data["test"]}, + ) + super(BaseModel, self).__init__() + + @field_validator("inputs", mode="before") + @classmethod + def instantiate_inputs(cls, data: AntaTest.Input | dict[str, Any] | None, info: ValidationInfo) -> AntaTest.Input: + """ + If the test has no inputs, allow the user to omit providing the `inputs` field. + If the test has inputs, allow the user to provide a valid dictionary of the input fields. + This model validator will instantiate an Input class from the `test` class field. + """ + if info.context is None: + raise ValueError("Could not validate inputs as no test class could be identified") + # Pydantic guarantees at this stage that test_class is a subclass of AntaTest because of the ordering + # of fields in the class definition - so no need to check for this + test_class = info.context["test"] + if not (isclass(test_class) and issubclass(test_class, AntaTest)): + raise ValueError(f"Could not validate inputs as no test class {test_class} is not a subclass of AntaTest") + + if data is None: + return test_class.Input() + if isinstance(data, AntaTest.Input): + return data + if isinstance(data, dict): + return test_class.Input(**data) + raise ValueError(f"Coud not instantiate inputs as type {type(data)} is not valid") + + @model_validator(mode="after") + def check_inputs(self) -> "AntaTestDefinition": + """ + The `inputs` class attribute needs to be an instance of the AntaTest.Input subclass defined in the class `test`. + """ + if not isinstance(self.inputs, self.test.Input): + raise ValueError(f"Test input has type {self.inputs.__class__.__qualname__} but expected type {self.test.Input.__qualname__}") + return self + + +class AntaCatalogFile(RootModel[Dict[ImportString[Any], List[AntaTestDefinition]]]): # pylint: disable=too-few-public-methods + """ + This model represents an ANTA Test Catalog File. + + A valid test catalog file must have the following structure: + : + - : + + """ + + root: Dict[ImportString[Any], List[AntaTestDefinition]] + + @model_validator(mode="before") + @classmethod + def check_tests(cls, data: Any) -> Any: + """ + Allow the user to provide a Python data structure that only has string values. + This validator will try to flatten and import Python modules, check if the tests classes + are actually defined in their respective Python module and instantiate Input instances + with provided value to validate test inputs. + """ + + def flatten_modules(data: dict[str, Any], package: str | None = None) -> dict[ModuleType, list[Any]]: + """ + Allow the user to provide a data structure with nested Python modules. + + Example: + ``` + anta.tests.routing: + generic: + - + bgp: + - + ``` + `anta.tests.routing.generic` and `anta.tests.routing.bgp` are importable Python modules. + """ + modules: dict[ModuleType, list[Any]] = {} + for module_name, tests in data.items(): + if package and not module_name.startswith("."): + module_name = f".{module_name}" + try: + module: ModuleType = importlib.import_module(name=module_name, package=package) + except ModuleNotFoundError as e: + module_str = module_name[1:] if module_name.startswith(".") else module_name + if package: + module_str += f" from package {package}" + raise ValueError(f"Module named {module_str} cannot be imported") from e + if isinstance(tests, dict): + # This is an inner Python module + modules.update(flatten_modules(data=tests, package=module.__name__)) + else: + if not isinstance(tests, list): + raise ValueError(f"{tests} must be a list of AntaTestDefinition") + # This is a list of AntaTestDefinition + modules[module] = tests + return modules + + if isinstance(data, dict): + typed_data: dict[ModuleType, list[Any]] = flatten_modules(data) + for module, tests in typed_data.items(): + test_definitions: list[AntaTestDefinition] = [] + for test_definition in tests: + if not isinstance(test_definition, dict): + raise ValueError("AntaTestDefinition must be a dictionary") + if len(test_definition) != 1: + raise ValueError("AntaTestDefinition must be a dictionary with a single entry") + for test_name, test_inputs in test_definition.copy().items(): + test: type[AntaTest] | None = getattr(module, test_name, None) + if test is None: + raise ValueError(f"{test_name} is not defined in Python module {module}") + test_definitions.append(AntaTestDefinition(test=test, inputs=test_inputs)) + typed_data[module] = test_definitions + return typed_data + + +class AntaCatalog: + """ + Class representing an ANTA Catalog. + + It can be instantiated using its contructor or one of the static methods: `parse()`, `from_list()` or `from_dict()` + """ + + def __init__(self, tests: list[AntaTestDefinition] | None = None, filename: str | Path | None = None) -> None: + """ + Constructor of AntaCatalog. + + Args: + tests: A list of AntaTestDefinition instances. + filename: The path from which the catalog is loaded. + """ + self._tests: list[AntaTestDefinition] = [] + if tests is not None: + self._tests = tests + self._filename: Path | None = None + if filename is not None: + if isinstance(filename, Path): + self._filename = filename + else: + self._filename = Path(filename) + + @property + def filename(self) -> Path | None: + """Path of the file used to create this AntaCatalog instance""" + return self._filename + + @property + def tests(self) -> list[AntaTestDefinition]: + """List of AntaTestDefinition in this catalog""" + return self._tests + + @tests.setter + def tests(self, value: list[AntaTestDefinition]) -> None: + if not isinstance(value, list): + raise ValueError("The catalog must contain a list of tests") + for t in value: + if not isinstance(t, AntaTestDefinition): + raise ValueError("A test in the catalog must be an AntaTestDefinition instance") + self._tests = value + + @staticmethod + def parse(filename: str | Path) -> AntaCatalog: + """ + Create an AntaCatalog instance from a test catalog file. + + Args: + filename: Path to test catalog YAML file + """ + try: + with open(file=filename, mode="r", encoding="UTF-8") as file: + data = safe_load(file) + except (YAMLError, OSError) as e: + message = f"Unable to parse ANTA Test Catalog file '{filename}'" + anta_log_exception(e, message, logger) + raise + try: + catalog_data = AntaCatalogFile(**data) + except ValidationError as e: + anta_log_exception(e, f"Test catalog '{filename}' is invalid!", logger) + raise + tests: list[AntaTestDefinition] = [] + for t in catalog_data.root.values(): + tests.extend(t) + return AntaCatalog(tests, filename=filename) + + @staticmethod + def from_dict(data: RawCatalogInput) -> AntaCatalog: + """ + Create an AntaCatalog instance from a dictionary data structure. + See RawCatalogInput type alias for details. + It is the data structure returned by `yaml.load()` function of a valid + YAML Test Catalog file. + + Args: + data: Python dictionary used to instantiate the AntaCatalog instance + """ + tests: list[AntaTestDefinition] = [] + try: + catalog_data = AntaCatalogFile(**data) # type: ignore[arg-type] + except ValidationError as e: + anta_log_exception(e, "Test catalog is invalid!", logger) + raise + for t in catalog_data.root.values(): + tests.extend(t) + return AntaCatalog(tests) + + @staticmethod + def from_list(data: ListAntaTestTuples) -> AntaCatalog: + """ + Create an AntaCatalog instance from a list data structure. + See ListAntaTestTuples type alias for details. + + Args: + data: Python list used to instantiate the AntaCatalog instance + """ + tests: list[AntaTestDefinition] = [] + try: + tests.extend(AntaTestDefinition(test=test, inputs=inputs) for test, inputs in data) + except ValidationError as e: + anta_log_exception(e, "Test catalog is invalid!", logger) + raise + return AntaCatalog(tests) + + def get_tests_by_tags(self, tags: list[str], strict: bool = False) -> list[AntaTestDefinition]: + """ + Return all the tests that have matching tags in their input filters. + If strict=True, returns only tests that match all the tags provided as input. + If strict=False, return all the tests that match at least one tag provided as input. + """ + result: list[AntaTestDefinition] = [] + for test in self.tests: + if test.inputs.filters and (f := test.inputs.filters.tags): + if (strict and all(t in tags for t in f)) or (not strict and any(t in tags for t in f)): + result.append(test) + return result diff --git a/anta/cli/__init__.py b/anta/cli/__init__.py index b725d56ca..1349c15ff 100644 --- a/anta/cli/__init__.py +++ b/anta/cli/__init__.py @@ -10,19 +10,20 @@ import logging import pathlib -from typing import Any, Callable, Literal +from typing import Any, Literal import click from anta import __version__ +from anta.catalog import AntaCatalog +from anta.cli.check import commands as check_commands from anta.cli.debug import commands as debug_commands from anta.cli.exec import commands as exec_commands from anta.cli.get import commands as get_commands -from anta.cli.nrfu import commands as check_commands +from anta.cli.nrfu import commands as nrfu_commands from anta.cli.utils import AliasedGroup, IgnoreRequiredWithHelp, parse_catalog, parse_inventory -from anta.loader import setup_logging +from anta.logger import setup_logging from anta.result_manager import ResultManager -from anta.result_manager.models import TestResult @click.group(cls=IgnoreRequiredWithHelp) @@ -133,6 +134,7 @@ def anta( ctx.ensure_object(dict) ctx.obj["inventory"] = parse_inventory(ctx, inventory) + ctx.obj["inventory_path"] = ctx.params["inventory"] @anta.group("nrfu", cls=IgnoreRequiredWithHelp) @@ -140,18 +142,24 @@ def anta( @click.option( "--catalog", "-c", + envvar="ANTA_CATALOG", show_envvar=True, - help="Path to the tests catalog YAML file", + help="Path to the test catalog YAML file", type=click.Path(file_okay=True, dir_okay=False, exists=True, readable=True), required=True, callback=parse_catalog, ) -def _nrfu(ctx: click.Context, catalog: list[tuple[Callable[..., TestResult], dict[Any, Any]]]) -> None: +def _nrfu(ctx: click.Context, catalog: AntaCatalog) -> None: """Run NRFU against inventory devices""" ctx.obj["catalog"] = catalog ctx.obj["result_manager"] = ResultManager() +@anta.group("check", cls=AliasedGroup) +def _check() -> None: + """Check commands for building ANTA""" + + @anta.group("exec", cls=AliasedGroup) def _exec() -> None: """Execute commands to inventory devices""" @@ -170,11 +178,14 @@ def _debug() -> None: # Load group commands # Prefixing with `_` for avoiding the confusion when importing anta.cli.debug.commands as otherwise the debug group has # a commands attribute. +_check.add_command(check_commands.catalog) +# Inventory cannot be implemented for now as main 'anta' CLI is already parsing it +# _check.add_command(check_commands.inventory) + _exec.add_command(exec_commands.clear_counters) _exec.add_command(exec_commands.snapshot) _exec.add_command(exec_commands.collect_tech_support) - _get.add_command(get_commands.from_cvp) _get.add_command(get_commands.from_ansible) _get.add_command(get_commands.inventory) @@ -183,10 +194,10 @@ def _debug() -> None: _debug.add_command(debug_commands.run_cmd) _debug.add_command(debug_commands.run_template) -_nrfu.add_command(check_commands.table) -_nrfu.add_command(check_commands.json) -_nrfu.add_command(check_commands.text) -_nrfu.add_command(check_commands.tpl_report) +_nrfu.add_command(nrfu_commands.table) +_nrfu.add_command(nrfu_commands.json) +_nrfu.add_command(nrfu_commands.text) +_nrfu.add_command(nrfu_commands.tpl_report) # ANTA CLI Execution diff --git a/anta/cli/check/__init__.py b/anta/cli/check/__init__.py new file mode 100644 index 000000000..c460d5493 --- /dev/null +++ b/anta/cli/check/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2023 Arista Networks, Inc. +# Use of this source code is governed by the Apache License 2.0 +# that can be found in the LICENSE file. diff --git a/anta/cli/check/commands.py b/anta/cli/check/commands.py new file mode 100644 index 000000000..1b16a6689 --- /dev/null +++ b/anta/cli/check/commands.py @@ -0,0 +1,38 @@ +# Copyright (c) 2023 Arista Networks, Inc. +# Use of this source code is governed by the Apache License 2.0 +# that can be found in the LICENSE file. +# pylint: disable = redefined-outer-name +""" +Commands for Anta CLI to run check commands. +""" +from __future__ import annotations + +import logging + +import click +from rich.pretty import pretty_repr + +from anta.catalog import AntaCatalog +from anta.cli.console import console +from anta.cli.utils import parse_catalog + +logger = logging.getLogger(__name__) + + +@click.command() +@click.option( + "--catalog", + "-c", + envvar="ANTA_CATALOG", + show_envvar=True, + help="Path to the test catalog YAML file", + type=click.Path(file_okay=True, dir_okay=False, exists=True, readable=True, resolve_path=True), + required=True, + callback=parse_catalog, +) +def catalog(catalog: AntaCatalog) -> None: + """ + Check that the catalog is valid + """ + console.print(f"[bold][green]Catalog {catalog} is valid") + console.print(pretty_repr(catalog.tests)) diff --git a/anta/cli/exec/commands.py b/anta/cli/exec/commands.py index ca61d7e34..765aa5906 100644 --- a/anta/cli/exec/commands.py +++ b/anta/cli/exec/commands.py @@ -66,7 +66,7 @@ def snapshot(ctx: click.Context, tags: list[str] | None, commands_list: Path, ou @click.command() @click.pass_context -@click.option("--output", "-o", default="./tech-support", show_default=True, help="Path for tests catalog", type=click.Path(path_type=Path), required=False) +@click.option("--output", "-o", default="./tech-support", show_default=True, help="Path for test catalog", type=click.Path(path_type=Path), required=False) @click.option("--latest", help="Number of scheduled show-tech to retrieve", type=int, required=False) @click.option( "--configure", diff --git a/anta/cli/get/commands.py b/anta/cli/get/commands.py index 86ca1b354..5dddfbaaf 100644 --- a/anta/cli/get/commands.py +++ b/anta/cli/get/commands.py @@ -4,7 +4,7 @@ # pylint: disable = redefined-outer-name """ -Commands for Anta CLI to run check commands. +Commands for Anta CLI to get information / build inventories.. """ from __future__ import annotations diff --git a/anta/cli/nrfu/utils.py b/anta/cli/nrfu/utils.py index 7275530d0..17aa4352d 100644 --- a/anta/cli/nrfu/utils.py +++ b/anta/cli/nrfu/utils.py @@ -2,7 +2,7 @@ # Use of this source code is governed by the Apache License 2.0 # that can be found in the LICENSE file. """ -Utils functions to use with anta.cli.check.commands module. +Utils functions to use with anta.cli.nrfu.commands module. """ from __future__ import annotations @@ -26,7 +26,7 @@ def print_settings(context: click.Context, report_template: pathlib.Path | None = None, report_output: pathlib.Path | None = None) -> None: """Print ANTA settings before running tests""" - message = f"Running ANTA tests:\n- {context.obj['inventory']}\n- Tests catalog contains {len(context.obj['catalog'])} tests" + message = f"Running ANTA tests:\n- {context.obj['inventory']}\n- Tests catalog contains {len(context.obj['catalog'].tests)} tests" if report_template: message += f"\n- Report template: {report_template}" if report_output: diff --git a/anta/cli/utils.py b/anta/cli/utils.py index 4c16793b2..43a8ba564 100644 --- a/anta/cli/utils.py +++ b/anta/cli/utils.py @@ -14,9 +14,10 @@ from typing import TYPE_CHECKING, Any import click -from yaml import safe_load +from pydantic import ValidationError +from yaml import YAMLError -import anta.loader +from anta.catalog import AntaCatalog from anta.inventory import AntaInventory from anta.tools.misc import anta_log_exception @@ -25,8 +26,6 @@ if TYPE_CHECKING: from click import Option - from anta.models import AntaTest - class ExitCode(enum.IntEnum): """ @@ -82,26 +81,22 @@ def parse_tags(ctx: click.Context, param: Option, value: str) -> list[str] | Non return None -def parse_catalog(ctx: click.Context, param: Option, value: str) -> list[tuple[AntaTest, dict[str, Any] | None]]: +def parse_catalog(ctx: click.Context, param: Option, value: Path) -> AntaCatalog: # pylint: disable=unused-argument """ - Click option callback to parse an ANTA tests catalog YAML file + Click option callback to parse an ANTA test catalog YAML file + + Store the orignal value (catalog path) in the ctx.obj """ if ctx.obj.get("_anta_help"): # Currently looking for help for a subcommand so no - # need to parse the Catalog - return an empty list - return [] + # need to parse the Catalog - return an empty catalog + return AntaCatalog() try: - with open(value, "r", encoding="UTF-8") as file: - data = safe_load(file) - # TODO catch proper exception - # pylint: disable-next=broad-exception-caught - except Exception as e: - message = f"Unable to parse ANTA Tests Catalog file '{value}'" - anta_log_exception(e, message, logger) - ctx.fail(message) - - return anta.loader.parse_catalog(data) + catalog: AntaCatalog = AntaCatalog.parse(value) + except (ValidationError, YAMLError, OSError): + ctx.fail("Unable to load ANTA Test Catalog") + return catalog def exit_with_code(ctx: click.Context) -> None: diff --git a/anta/custom_types.py b/anta/custom_types.py index 79fd4e8f7..bb265a0bf 100644 --- a/anta/custom_types.py +++ b/anta/custom_types.py @@ -17,10 +17,13 @@ def aaa_group_prefix(v: str) -> str: return f"group {v}" if v not in built_in_methods and not v.startswith("group ") else v +# ANTA framework +TestStatus = Literal["unset", "success", "failure", "error", "skipped"] + +# AntaTest.Input types AAAAuthMethod = Annotated[str, AfterValidator(aaa_group_prefix)] Vlan = Annotated[int, Field(ge=0, le=4094)] Vni = Annotated[int, Field(ge=1, le=16777215)] -TestStatus = Literal["unset", "success", "failure", "error", "skipped"] Interface = Annotated[str, Field(pattern=r"^(Ethernet|Fabric|Loopback|Management|Port-Channel|Tunnel|Vlan|Vxlan)[0-9]+(\/[0-9]+)*(\.[0-9]+)?$")] Afi = Literal["ipv4", "ipv6", "vpn-ipv4", "vpn-ipv6", "evpn", "rt-membership"] Safi = Literal["unicast", "multicast", "labeled-unicast"] diff --git a/anta/device.py b/anta/device.py index a0e6fc45e..0a13e27fd 100644 --- a/anta/device.py +++ b/anta/device.py @@ -56,6 +56,8 @@ def __init__(self, name: str, tags: Optional[list[str]] = None, disable_cache: b self.name: str = name self.hw_model: Optional[str] = None self.tags: list[str] = tags if tags is not None else [] + # A device always has its own name as tag + self.tags.append(self.name) self.is_online: bool = False self.established: bool = False self.cache: Optional[Cache] = None @@ -65,6 +67,25 @@ def __init__(self, name: str, tags: Optional[list[str]] = None, disable_cache: b if not disable_cache: self._init_cache() + @property + @abstractmethod + def _keys(self) -> tuple[Any, ...]: + """ + Read-only property to implement hashing and equality for AntaDevice classes. + """ + + def __eq__(self, other: object) -> bool: + """ + Implement equality for AntaDevice objects. + """ + return self._keys == other._keys if isinstance(other, self.__class__) else False + + def __hash__(self) -> int: + """ + Implement hashing for AntaDevice objects. + """ + return hash(self._keys) + def _init_cache(self) -> None: """ Initialize cache for the device, can be overriden by subclasses to manipulate how it works @@ -96,12 +117,6 @@ def __rich_repr__(self) -> Iterator[tuple[str, Any]]: yield "established", self.established yield "disable_cache", self.cache is None - @abstractmethod - def __eq__(self, other: object) -> bool: - """ - AntaDevice equality depends on the class implementation. - """ - @abstractmethod async def _collect(self, command: AntaCommand) -> None: """ @@ -237,7 +252,7 @@ def __init__( # pylint: disable=R0913 self._session: Device = Device(host=host, port=port, username=username, password=password, proto=proto, timeout=timeout) ssh_params: dict[str, Any] = {} if insecure: - ssh_params.update({"known_hosts": None}) + ssh_params["known_hosts"] = None self._ssh_opts: SSHClientConnectionOptions = SSHClientConnectionOptions(host=host, port=ssh_port, username=username, password=password, **ssh_params) def __rich_repr__(self) -> Iterator[tuple[str, Any]]: @@ -245,30 +260,27 @@ def __rich_repr__(self) -> Iterator[tuple[str, Any]]: Implements Rich Repr Protocol https://rich.readthedocs.io/en/stable/pretty.html#rich-repr-protocol """ - - PASSWORD_VALUE = "" - yield from super().__rich_repr__() - yield "host", self._session.host - yield "eapi_port", self._session.port - yield "username", self._ssh_opts.username - yield "enable", self.enable - yield "insecure", self._ssh_opts.known_hosts is None + yield ("host", self._session.host) + yield ("eapi_port", self._session.port) + yield ("username", self._ssh_opts.username) + yield ("enable", self.enable) + yield ("insecure", self._ssh_opts.known_hosts is None) if __DEBUG__: _ssh_opts = vars(self._ssh_opts).copy() + PASSWORD_VALUE = "" _ssh_opts["password"] = PASSWORD_VALUE _ssh_opts["kwargs"]["password"] = PASSWORD_VALUE - yield "_session", vars(self._session) - yield "_ssh_opts", _ssh_opts + yield ("_session", vars(self._session)) + yield ("_ssh_opts", _ssh_opts) - def __eq__(self, other: object) -> bool: + @property + def _keys(self) -> tuple[Any, ...]: """ Two AsyncEOSDevice objects are equal if the hostname and the port are the same. This covers the use case of port forwarding when the host is localhost and the devices have different ports. """ - if not isinstance(other, AsyncEOSDevice): - return False - return self._session.host == other._session.host and self._session.port == other._session.port + return (self._session.host, self._session.port) async def _collect(self, command: AntaCommand) -> None: """ @@ -333,26 +345,28 @@ async def refresh(self) -> None: - established: When a command execution succeeds - hw_model: The hardware model of the device """ - # Refresh command - COMMAND: str = "show version" - # Hardware model definition in show version - HW_MODEL_KEY: str = "modelName" logger.debug(f"Refreshing device {self.name}") self.is_online = await self._session.check_connection() if self.is_online: + COMMAND: str = "show version" + HW_MODEL_KEY: str = "modelName" try: response = await self._session.cli(command=COMMAND) except EapiCommandError as e: logger.warning(f"Cannot get hardware information from device {self.name}: {e.errmsg}") + except (HTTPError, ConnectError) as e: logger.warning(f"Cannot get hardware information from device {self.name}: {exc_to_str(e)}") + else: if HW_MODEL_KEY in response: self.hw_model = response[HW_MODEL_KEY] else: logger.warning(f"Cannot get hardware information from device {self.name}: cannot parse '{COMMAND}'") + else: logger.warning(f"Could not connect to device {self.name}: cannot open eAPI port") + self.established = bool(self.is_online and self.hw_model) async def copy(self, sources: list[Path], destination: Path, direction: Literal["to", "from"] = "from") -> None: @@ -379,12 +393,15 @@ async def copy(self, sources: list[Path], destination: Path, direction: Literal[ dst = destination for file in sources: logger.info(f"Copying '{file}' from device {self.name} to '{destination}' locally") + elif direction == "to": src = sources - dst = (conn, destination) - for file in sources: + dst = conn, destination + for file in src: logger.info(f"Copying '{file}' to device {self.name} to '{destination}' remotely") + else: logger.critical(f"'direction' argument to copy() fonction is invalid: {direction}") + return await asyncssh.scp(src, dst) diff --git a/anta/loader.py b/anta/loader.py deleted file mode 100644 index 95d4e9347..000000000 --- a/anta/loader.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright (c) 2023 Arista Networks, Inc. -# Use of this source code is governed by the Apache License 2.0 -# that can be found in the LICENSE file. -""" -Loader that parses a YAML test catalog and imports corresponding Python functions -""" -from __future__ import annotations - -import importlib -import logging -import sys -from pathlib import Path -from typing import Any - -from rich.logging import RichHandler - -from anta import __DEBUG__ -from anta.models import AntaTest - -logger = logging.getLogger(__name__) - - -def setup_logging(level: str = logging.getLevelName(logging.INFO), file: Path | None = None) -> None: - """ - Configure logging for ANTA. - By default, the logging level is INFO for all loggers except for httpx and asyncssh which are too verbose: - their logging level is WARNING. - - If logging level DEBUG is selected, all loggers will be configured with this level. - - In ANTA Debug Mode (environment variable `ANTA_DEBUG=true`), Python tracebacks are logged and logging level is - overwritten to be DEBUG. - - If a file is provided, logs will also be sent to the file in addition to stdout. - If a file is provided and logging level is DEBUG, only the logging level INFO and higher will - be logged to stdout while all levels will be logged in the file. - - Args: - level: ANTA logging level - file: Send logs to a file - """ - # Init root logger - root = logging.getLogger() - # In ANTA debug mode, level is overriden to DEBUG - loglevel = getattr(logging, level.upper()) if not __DEBUG__ else logging.DEBUG - root.setLevel(loglevel) - # Silence the logging of chatty Python modules when level is INFO - if loglevel == logging.INFO: - # asyncssh is really chatty - logging.getLogger("asyncssh").setLevel(logging.WARNING) - # httpx as well - logging.getLogger("httpx").setLevel(logging.WARNING) - - # Add RichHandler for stdout - richHandler = RichHandler(markup=True, rich_tracebacks=True, tracebacks_show_locals=True) - # In ANTA debug mode, show Python module in stdout - if __DEBUG__: - fmt_string = r"[grey58]\[%(name)s][/grey58] %(message)s" - else: - fmt_string = "%(message)s" - formatter = logging.Formatter(fmt=fmt_string, datefmt="[%X]") - richHandler.setFormatter(formatter) - root.addHandler(richHandler) - # Add FileHandler if file is provided - if file: - fileHandler = logging.FileHandler(file) - formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") - fileHandler.setFormatter(formatter) - root.addHandler(fileHandler) - # If level is DEBUG and file is provided, do not send DEBUG level to stdout - if loglevel == logging.DEBUG: - richHandler.setLevel(logging.INFO) - - if __DEBUG__: - logger.debug("ANTA Debug Mode enabled") - - -def parse_catalog(test_catalog: dict[str, Any], package: str | None = None) -> list[tuple[AntaTest, dict[str, Any] | None]]: - """ - Function to parse the catalog and return a list of tests with their inputs - - A valid test catalog must follow the following structure: - : - - : - - - Example: - anta.tests.connectivity: - - VerifyReachability: - hosts: - - dst: 8.8.8.8 - src: 172.16.0.1 - - dst: 1.1.1.1 - src: 172.16.0.1 - result_overwrite: - categories: - - "Overwritten category 1" - description: "Test with overwritten description" - custom_field: "Test run by John Doe" - - Also supports nesting for Python module definition: - anta.tests: - connectivity: - - VerifyReachability: - hosts: - - dst: 8.8.8.8 - src: 172.16.0.1 - - dst: 1.1.1.1 - src: 172.16.0.1 - result_overwrite: - categories: - - "Overwritten category 1" - description: "Test with overwritten description" - custom_field: "Test run by John Doe" - - Args: - test_catalog: Python dictionary representing the test catalog YAML file - - Returns: - tests: List of tuples (test, inputs) where test is a reference of an AntaTest subclass - and inputs is a dictionary - """ - tests: list[tuple[AntaTest, dict[str, Any] | None]] = [] - if not test_catalog: - return tests - for key, value in test_catalog.items(): - # Required to manage iteration within a tests module - if package is not None: - key = ".".join([package, key]) - try: - module = importlib.import_module(f"{key}") - except ModuleNotFoundError: - logger.critical(f"No test module named '{key}'") - sys.exit(1) - if isinstance(value, list): - # This is a list of tests - for test in value: - for test_name, inputs in test.items(): - # A test must be a subclass of AntaTest as defined in the Python module - try: - test = getattr(module, test_name) - except AttributeError: - logger.critical(f"Wrong test name '{test_name}' in '{module.__name__}'") - sys.exit(1) - if not issubclass(test, AntaTest): - logger.critical(f"'{test.__module__}.{test.__name__}' is not an AntaTest subclass") - sys.exit(1) - # Test inputs can be either None or a dictionary - if inputs is None or isinstance(inputs, dict): - tests.append((test, inputs)) - else: - logger.critical(f"'{test.__module__}.{test.__name__}' inputs must be a dictionary") - sys.exit(1) - if isinstance(value, dict): - # This is an inner Python module - tests.extend(parse_catalog(value, package=module.__name__)) - return tests diff --git a/anta/logger.py b/anta/logger.py new file mode 100644 index 000000000..f8cabdc79 --- /dev/null +++ b/anta/logger.py @@ -0,0 +1,71 @@ +# Copyright (c) 2023 Arista Networks, Inc. +# Use of this source code is governed by the Apache License 2.0 +# that can be found in the LICENSE file. +""" +Configure logging for ANTA +""" +from __future__ import annotations + +import logging +from pathlib import Path + +from rich.logging import RichHandler + +from anta import __DEBUG__ + +logger = logging.getLogger(__name__) + + +def setup_logging(level: str = logging.getLevelName(logging.INFO), file: Path | None = None) -> None: + """ + Configure logging for ANTA. + By default, the logging level is INFO for all loggers except for httpx and asyncssh which are too verbose: + their logging level is WARNING. + + If logging level DEBUG is selected, all loggers will be configured with this level. + + In ANTA Debug Mode (environment variable `ANTA_DEBUG=true`), Python tracebacks are logged and logging level is + overwritten to be DEBUG. + + If a file is provided, logs will also be sent to the file in addition to stdout. + If a file is provided and logging level is DEBUG, only the logging level INFO and higher will + be logged to stdout while all levels will be logged in the file. + + Args: + level: ANTA logging level + file: Send logs to a file + """ + # Init root logger + root = logging.getLogger() + # In ANTA debug mode, level is overriden to DEBUG + loglevel = logging.DEBUG if __DEBUG__ else getattr(logging, level.upper()) + root.setLevel(loglevel) + # Silence the logging of chatty Python modules when level is INFO + if loglevel == logging.INFO: + # asyncssh is really chatty + logging.getLogger("asyncssh").setLevel(logging.WARNING) + # httpx as well + logging.getLogger("httpx").setLevel(logging.WARNING) + + # Add RichHandler for stdout + richHandler = RichHandler(markup=True, rich_tracebacks=True, tracebacks_show_locals=True) + # In ANTA debug mode, show Python module in stdout + if __DEBUG__: + fmt_string = r"[grey58]\[%(name)s][/grey58] %(message)s" + else: + fmt_string = "%(message)s" + formatter = logging.Formatter(fmt=fmt_string, datefmt="[%X]") + richHandler.setFormatter(formatter) + root.addHandler(richHandler) + # Add FileHandler if file is provided + if file: + fileHandler = logging.FileHandler(file) + formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + fileHandler.setFormatter(formatter) + root.addHandler(fileHandler) + # If level is DEBUG and file is provided, do not send DEBUG level to stdout + if loglevel == logging.DEBUG: + richHandler.setLevel(logging.INFO) + + if __DEBUG__: + logger.debug("ANTA Debug Mode enabled") diff --git a/anta/models.py b/anta/models.py index 13823496a..59cfa144f 100644 --- a/anta/models.py +++ b/anta/models.py @@ -290,6 +290,13 @@ class Input(BaseModel): result_overwrite: Optional[ResultOverwrite] = None filters: Optional[Filters] = None + def __hash__(self) -> int: + """ + Implement generic hashing for AntaTest.Input. + This will work in most cases but this does not consider 2 lists with different ordering as equal. + """ + return hash(self.model_dump_json()) + class ResultOverwrite(BaseModel): """Test inputs model to overwrite result fields @@ -299,6 +306,7 @@ class ResultOverwrite(BaseModel): custom_field: a free string that will be included in the TestResult object """ + model_config = ConfigDict(extra="forbid") description: Optional[str] = None categories: Optional[List[str]] = None custom_field: Optional[str] = None @@ -307,18 +315,17 @@ class Filters(BaseModel): """Runtime filters to map tests with list of tags or devices Attributes: - devices: List of devices for the test. tags: List of device's tags for the test. """ - devices: Optional[List[str]] = None + model_config = ConfigDict(extra="forbid") tags: Optional[List[str]] = None def __init__( self, device: AntaDevice, - inputs: Optional[dict[str, Any]], - eos_data: Optional[list[dict[Any, Any] | str]] = None, + inputs: dict[str, Any] | AntaTest.Input | None = None, + eos_data: list[dict[Any, Any] | str] | None = None, ): """AntaTest Constructor @@ -337,14 +344,19 @@ def __init__( if self.result.result == "unset": self._init_commands(eos_data) - def _init_inputs(self, inputs: Optional[dict[str, Any]]) -> None: + def _init_inputs(self, inputs: dict[str, Any] | AntaTest.Input | None) -> None: """Instantiate the `inputs` instance attribute with an `AntaTest.Input` instance to validate test inputs from defined model. Overwrite result fields based on `ResultOverwrite` input definition. Any input validation error will set this test result status as 'error'.""" try: - self.inputs = self.Input(**inputs) if inputs is not None else self.Input() + if inputs is None: + self.inputs = self.Input() + elif isinstance(inputs, AntaTest.Input): + self.inputs = inputs + elif isinstance(inputs, dict): + self.inputs = self.Input(**inputs) except ValidationError as e: message = f"{self.__module__}.{self.__class__.__name__}: Inputs are not valid\n{e}" self.logger.error(message) @@ -467,7 +479,7 @@ def anta_test(function: F) -> Callable[..., Coroutine[Any, Any, TestResult]]: @wraps(function) async def wrapper( self: AntaTest, - eos_data: Optional[list[dict[Any, Any] | str]] = None, + eos_data: list[dict[Any, Any] | str] | None = None, **kwargs: Any, ) -> TestResult: """ diff --git a/anta/runner.py b/anta/runner.py index d9f9e961c..448bc1059 100644 --- a/anta/runner.py +++ b/anta/runner.py @@ -8,10 +8,11 @@ from __future__ import annotations import asyncio -import itertools import logging -from typing import Union +from typing import Tuple +from anta.catalog import AntaCatalog, AntaTestDefinition +from anta.device import AntaDevice from anta.inventory import AntaInventory from anta.models import AntaTest from anta.result_manager import ResultManager @@ -19,19 +20,10 @@ logger = logging.getLogger(__name__) +AntaTestRunner = Tuple[AntaTestDefinition, AntaDevice] -def filter_tags(tags_cli: Union[list[str], None], tags_device: list[str], tags_test: list[str]) -> bool: - """Implement filtering logic for tags""" - return (tags_cli is None or any(t for t in tags_cli if t in tags_device)) and any(t for t in tags_device if t in tags_test) - -async def main( - manager: ResultManager, - inventory: AntaInventory, - tests: list[tuple[type[AntaTest], AntaTest.Input]], - tags: list[str], - established_only: bool = True, -) -> None: +async def main(manager: ResultManager, inventory: AntaInventory, catalog: AntaCatalog, tags: list[str] | None = None, established_only: bool = True) -> None: """ Main coroutine to run ANTA. Use this as an entrypoint to the test framwork in your script. @@ -39,51 +31,57 @@ async def main( Args: manager: ResultManager object to populate with the test results. inventory: AntaInventory object that includes the device(s). - tests: ANTA test catalog. Output of anta.loader.parse_catalog(). + catalog: AntaCatalog object that includes the list of tests. tags: List of tags to filter devices from the inventory. Defaults to None. established_only: Include only established device(s). Defaults to True. Returns: any: ResultManager object gets updated with the test results. """ - - if not tests: + if not catalog.tests: logger.info("The list of tests is empty, exiting") return - if len(inventory) == 0: logger.info("The inventory is empty, exiting") return - await inventory.connect_inventory() + devices: list[AntaDevice] = list(inventory.get_inventory(established_only=established_only, tags=tags).values()) - # asyncio.gather takes an iterator of the function to run concurrently. - # we get the cross product of the devices and tests to build that iterator. - devices = inventory.get_inventory(established_only=established_only, tags=tags).values() - - if len(devices) == 0: + if not devices: logger.info( f"No device in the established state '{established_only}' " f"{f'matching the tags {tags} ' if tags else ''}was found. There is no device to run tests against, exiting" ) - return + return coros = [] + # Using a set to avoid inserting duplicate tests + tests_set: set[AntaTestRunner] = set() + for device in devices: + if tags: + # If there are CLI tags, only execute tests with matching tags + tests_set.update((test, device) for test in catalog.get_tests_by_tags(tags)) + else: + # If there is no CLI tags, execute all tests without filters + tests_set.update((t, device) for t in catalog.tests if t.inputs.filters is None or t.inputs.filters.tags is None) + + # Then add the tests with matching tags from device tags + tests_set.update((t, device) for t in catalog.get_tests_by_tags(device.tags)) + + tests: list[AntaTestRunner] = list(tests_set) - for device, test in itertools.product(devices, tests): - test_class = test[0] - test_inputs = test[1] - test_filters = test[1].get("filters", None) if test[1] is not None else None - test_tags = test_filters.get("tags", []) if test_filters is not None else [] - if len(test_tags) == 0 or filter_tags(tags_cli=tags, tags_device=device.tags, tags_test=test_tags): - try: - # Instantiate AntaTest object - test_instance = test_class(device=device, inputs=test_inputs) - coros.append(test_instance.test(eos_data=None)) - except Exception as e: # pylint: disable=broad-exception-caught - message = "Error when creating ANTA tests" - anta_log_exception(e, message, logger) + if not tests: + logger.info(f"There is no tests{f' matching the tags {tags} ' if tags else ' '}to run on current inventory. " "Exiting...") + return + for test_definition, device in tests: + try: + test_instance = test_definition.test(device=device, inputs=test_definition.inputs) + + coros.append(test_instance.test()) + except Exception as e: # pylint: disable=broad-exception-caught + message = "Error when creating ANTA tests" + anta_log_exception(e, message, logger) if AntaTest.progress is not None: AntaTest.nrfu_task = AntaTest.progress.add_task("Running NRFU Tests...", total=len(coros)) @@ -96,8 +94,6 @@ async def main( anta_log_exception(r, message, logger) else: manager.add_test_result(r) - - # Get each device statistics for device in devices: if device.cache_statistics is not None: logger.info(f"Cache statistics for {device.name}: {device.cache_statistics}") diff --git a/docs/api/catalog.md b/docs/api/catalog.md new file mode 100644 index 000000000..8b91fff9d --- /dev/null +++ b/docs/api/catalog.md @@ -0,0 +1,13 @@ + + +### ::: anta.catalog.AntaCatalog + options: + filters: ["!^_[^_]", "!__str__"] + +### ::: anta.catalog.AntaTestDefinition + +### ::: anta.catalog.AntaCatalogFile diff --git a/docs/cli/exec.md b/docs/cli/exec.md index be94b4bc5..2fe726193 100644 --- a/docs/cli/exec.md +++ b/docs/cli/exec.md @@ -161,7 +161,7 @@ Usage: anta exec collect-tech-support [OPTIONS] Collect scheduled tech-support from EOS devices Options: - -o, --output PATH Path for tests catalog [default: ./tech- + -o, --output PATH Path for test catalog [default: ./tech- support] --latest INTEGER Number of scheduled show-tech to retrieve --configure Ensure devices have 'aaa authorization exec default diff --git a/docs/cli/nrfu.md b/docs/cli/nrfu.md index 6b05cb246..90617d086 100644 --- a/docs/cli/nrfu.md +++ b/docs/cli/nrfu.md @@ -22,8 +22,8 @@ Usage: anta nrfu [OPTIONS] COMMAND [ARGS]... Run NRFU against inventory devices Options: - -c, --catalog FILE Path to the tests catalog YAML file [env var: - ANTA_NRFU_CATALOG; required] + -c, --catalog FILE Path to the test catalog YAML file [env var: + ANTA_CATALOG; required] --help Show this message and exit. Commands: @@ -41,7 +41,7 @@ The `--tags` option can be used to target specific devices in your inventory and | Command | Description | | ------- | ----------- | -| `none` | Run all tests on all devices according `tag` definition in your inventory and tests catalog. And tests with no tag are executed on all devices| +| `none` | Run all tests on all devices according `tag` definition in your inventory and test catalog. And tests with no tag are executed on all devices| | `--tags leaf` | Run all tests marked with `leaf` tag on all devices configured with `leaf` tag.
All other tags are ignored | | `--tags leaf,spine` | Run all tests marked with `leaf` tag on all devices configured with `leaf` tag.
Run all tests marked with `spine` tag on all devices configured with `spine` tag.
All other tags are ignored | diff --git a/docs/cli/tag-management.md b/docs/cli/tag-management.md index 61a2c227a..3ec24f276 100644 --- a/docs/cli/tag-management.md +++ b/docs/cli/tag-management.md @@ -8,7 +8,7 @@ ## Overview -ANTA nrfu command comes with a `--tags` option. This allows users to specify a set of tests, marked with a given tag, to be run on devices marked with the same tag. For instance, you can run tests dedicated to leaf devices on your leaf devices only and not on other devices. +The `anta nrfu` command comes with a `--tags` option. This allows users to specify a set of tests, marked with a given tag, to be run on devices marked with the same tag. For instance, you can run tests dedicated to leaf devices on your leaf devices only and not on other devices. Tags are string defined by the user and can be anything considered as a string by Python. A [default one](#default-tags) is present for all tests and devices. @@ -16,13 +16,13 @@ The next table provides a short summary of the scope of tags using CLI | Command | Description | | ------- | ----------- | -| `none` | Run all tests on all devices according `tag` definition in your inventory and tests catalog. And tests with no tag are executed on all devices| +| `none` | Run all tests on all devices according `tag` definition in your inventory and test catalog. And tests with no tag are executed on all devices| | `--tags leaf` | Run all tests marked with `leaf` tag on all devices configured with `leaf` tag.
All other tags are ignored | | `--tags leaf,spine` | Run all tests marked with `leaf` tag on all devices configured with `leaf` tag.
Run all tests marked with `spine` tag on all devices configured with `spine` tag.
All other tags are ignored | ## Inventory and Catalog for tests -All commands in this page are based on the following inventory and tests catalog. +All commands in this page are based on the following inventory and test catalog. === "Inventory" @@ -50,7 +50,7 @@ All commands in this page are based on the following inventory and tests catalog tags: ['fabric', 'leaf' ``` -=== "Tests Catalog" +=== "Test Catalog" ```yaml anta.tests.system: @@ -135,7 +135,7 @@ leaf04 :: VerifyReloadCause :: SUCCESS leaf04 :: VerifyCPUUtilization :: SUCCESS ``` -In this case, only `leaf` devices defined in your [inventory](#inventory-and-catalog-for-tests) are used to run tests marked with `leaf` in your [tests catalog](#inventory-and-catalog-for-tests) +In this case, only `leaf` devices defined in your [inventory](#inventory-and-catalog-for-tests) are used to run tests marked with `leaf` in your [test catalog](#inventory-and-catalog-for-tests) ## Use multiple tags in CLI diff --git a/docs/getting-started.md b/docs/getting-started.md index ab7c4337f..63442034f 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -139,8 +139,8 @@ Usage: anta nrfu [OPTIONS] COMMAND [ARGS]... Run NRFU against inventory devices Options: - -c, --catalog FILE Path to the tests catalog YAML file [env var: - ANTA_NRFU_CATALOG; required] + -c, --catalog FILE Path to the test catalog YAML file [env var: + ANTA_CATALOG; required] --help Show this message and exit. Commands: diff --git a/docs/usage-inventory-catalog.md b/docs/usage-inventory-catalog.md index c1ebf9b23..a4d13b68d 100644 --- a/docs/usage-inventory-catalog.md +++ b/docs/usage-inventory-catalog.md @@ -4,13 +4,20 @@ ~ that can be found in the LICENSE file. --> -# Inventory and Catalog definition +# Inventory and Catalog -This page describes how to create an inventory and a tests catalog. +The ANTA framework needs 2 important inputs from the user to run: a **device inventory** and a **test catalog**. -## Create an inventory file +Both inputs can be defined in a file or programmatically. -`anta` cli needs an inventory file to list all devices to tests. This inventory is a YAML file with the folowing keys: +## Device Inventory + +A device inventory is an instance of the [AntaInventory](../api/inventory.md#anta.inventory.AntaInventory) class. + +### Device Inventory File + +The ANTA device inventory can easily be defined as a YAML file. +The file must comply with the following structure: ```yaml anta_inventory: @@ -31,12 +38,18 @@ anta_inventory: disable_cache: < Disable cache per range. Default is False. > ``` -Your inventory file can be based on any of these 3 keys and MUST start with `anta_inventory` key. A full description of the inventory model is available in [API documentation](api/inventory.models.input.md) +The inventory file must start with the `anta_inventory` key then define one or multiple methods: + +- `hosts`: define each device individually +- `networks`: scan a network for devices accesible via eAPI +- `ranges`: scan a range for devices accesible via eAPI + +A full description of the inventory model is available in [API documentation](api/inventory.models.input.md) !!! info Caching can be disabled per device, network or range by setting the `disable_cache` key to `True` in the inventory file. For more details about how caching is implemented in ANTA, please refer to [Caching in ANTA](advanced_usages/caching.md). -An inventory example: +### Example ```yaml --- @@ -59,19 +72,100 @@ anta_inventory: ## Test Catalog -In addition to your inventory file, you also have to define a catalog of tests to execute against all your devices. This catalog list all your tests and their parameters. -Its format is a YAML file and keys are tests functions inherited from the python path. +A test catalog is an instance of the [AntaCatalog](../api/catalog.md#anta.catalog.AntaCatalog) class. -### Default tests catalog +### Test Catalog File -All tests are located under `anta.tests` module and are categorised per family (one submodule). So to run test for software version, you can do: +In addition to the inventory file, you also have to define a catalog of tests to execute against your devices. This catalog list all your tests, their inputs and their tags. + +A valid test catalog file must have the following structure: +```yaml +--- +: + - : + +``` + +### Example + +```yaml +--- +anta.tests.connectivity: + - VerifyReachability: + hosts: + - source: Management0 + destination: 1.1.1.1 + vrf: MGMT + - source: Management0 + destination: 8.8.8.8 + vrf: MGMT + filters: + tags: ['leaf'] + result_overwrite: + categories: + - "Overwritten category 1" + description: "Test with overwritten description" + custom_field: "Test run by John Doe" +``` + +It is also possible to nest Python module definition: +```yaml +anta.tests: + connectivity: + - VerifyReachability: + hosts: + - source: Management0 + destination: 1.1.1.1 + vrf: MGMT + - source: Management0 + destination: 8.8.8.8 + vrf: MGMT + filters: + tags: ['leaf'] + result_overwrite: + categories: + - "Overwritten category 1" + description: "Test with overwritten description" + custom_field: "Test run by John Doe" +``` + +[This test catalog example](https://github.com/arista-netdevops-community/anta/blob/main/examples/tests.yaml) is maintained with all the tests defined in the `anta.tests` Python module. + +### Test tags + +All tests can be defined with a list of user defined tags. These tags will be mapped with device tags: when at least one tag is defined for a test, this test will only be executed on devices with the same tag. If a test is defined in the catalog without any tags, the test will be executed on all devices. + +```yaml +anta.tests.system: + - VerifyUptime: + minimum: 10 + filters: + tags: ['demo', 'leaf'] + - VerifyReloadCause: + - VerifyCoredump: + - VerifyAgentLogs: + - VerifyCPUUtilization: + filters: + tags: ['leaf'] +``` + +!!! info + When using the CLI, you can filter the NRFU execution using tags. Refer to [this section](cli/tag-management.md) of the CLI documentation. + +### Tests available in ANTA + +All tests available as part of the ANTA framework are defined under the `anta.tests` Python module and are categorised per family (Python submodule). +The complete list of the tests and their respective inputs is available at the [tests section](api/tests.md) of this website. + + +To run test to verify the EOS software version, you can do: ```yaml anta.tests.software: - VerifyEosVersion: ``` -It will load the test `VerifyEosVersion` located in `anta.tests.software`. But since this function has parameters, we will create a catalog with the following structure: +It will load the test `VerifyEosVersion` located in `anta.tests.software`. But since this test has mandatory inputs, we need to provide them as a dictionary in the YAML file: ```yaml anta.tests.software: @@ -82,9 +176,7 @@ anta.tests.software: - 4.26.1F ``` -To get a list of all available tests and their respective parameters, you can read the [tests section](./api/tests.md) of this website. - -The following example gives a very minimal tests catalog you can use in almost any situation +The following example is a very minimal test catalog: ```yaml --- @@ -110,35 +202,10 @@ anta.tests.configuration: - VerifyRunningConfigDiffs: ``` -All tests can be configured with a list of user defined tags. These tags will be mapped with device tags when `nrfu` cli is called with the `--tags` option. If a test is configured in the catalog without a list of tags, the test will be executed for all devices. When at least one tag is defined for a test, this test will be only executed on devices with the same tag. - -```yaml -anta.tests.system: - - VerifyUptime: - minimum: 10 - filters: - tags: ['demo', 'leaf'] - - VerifyReloadCause: - - VerifyCoredump: - - VerifyAgentLogs: - - VerifyCPUUtilization: - filters: - tags: ['leaf'] -``` - -!!! info - The `tag` field must be equal to values configured for `tags` under your device inventory. - -### Custom tests catalog - -In case you want to leverage your own tests collection, you can use the following syntax: - -```yaml -: - - : -``` +### Catalog with custom tests -So for instance, it could be: +In case you want to leverage your own tests collection, use your own Python package in the test catalog. +So for instance, if my custom tests are defined in the `titom73.tests.system` Python module, the test catalog will be: ```yaml titom73.tests.system: @@ -147,7 +214,7 @@ titom73.tests.system: ``` !!! tip "How to create custom tests" - To create your custom tests, you should refer to this [following documentation](advanced_usages/custom-tests.md) + To create your custom tests, you should refer to this [documentation](advanced_usages/custom-tests.md) ### Customize test description and categories diff --git a/mkdocs.yml b/mkdocs.yml index 9bffeb69c..3fd647743 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -170,7 +170,7 @@ nav: - Caching in ANTA: advanced_usages/caching.md - Developing ANTA tests: advanced_usages/custom-tests.md - ANTA as a Python Library: advanced_usages/as-python-lib.md - - Tests Catalog Documentation: + - Test Catalog Documentation: - Overview: api/tests.md - AAA: api/tests.aaa.md - Configuration: api/tests.configuration.md @@ -196,8 +196,8 @@ nav: - Inventory: - Inventory module: api/inventory.md - Inventory models: api/inventory.models.input.md - - Device: - - Device models: api/device.md + - Test Catalog: api/catalog.md + - Device: api/device.md - Test: - Test models: api/models.md - Input Types: api/types.md diff --git a/tests/data/json_data.py b/tests/data/json_data.py index 95760949c..9ff2a10d0 100644 --- a/tests/data/json_data.py +++ b/tests/data/json_data.py @@ -8,7 +8,6 @@ { "name": "validIPv6", "input": "fe80::cc62:a9ff:feef:932a", - "expected_result": "valid", }, ] @@ -16,18 +15,15 @@ { "name": "invalidIPv4_with_netmask", "input": "1.1.1.1/32", - "expected_result": "invalid", }, { "name": "invalidIPv6_with_netmask", "input": "fe80::cc62:a9ff:feef:932a/128", - "expected_result": "invalid", }, {"name": "invalidHost_format", "input": "@", "expected_result": "invalid"}, { "name": "invalidIPv6_format", "input": "fe80::cc62:a9ff:feef:", - "expected_result": "invalid", }, ] @@ -253,55 +249,10 @@ }, ] -TEST_RESULT_UNIT = [ - { - "name": "valid_with_host_ip_only", - "input": {"host": "1.1.1.1", "test": "pytest_unit_test"}, - "expected_result": "valid", - }, - { - "name": "valid_with_host_ip_and_success_result", - "input": {"host": "1.1.1.1", "test": "pytest_unit_test", "result": "success"}, - "expected_result": "valid", - }, - { - "name": "valid_with_host_ip_and_skipped_result", - "input": {"host": "1.1.1.1", "test": "pytest_unit_test", "result": "success"}, - "expected_result": "valid", - }, - { - "name": "valid_with_host_ip_and_failure_result", - "input": {"host": "1.1.1.1", "test": "pytest_unit_test", "result": "failure"}, - "expected_result": "valid", - }, - { - "name": "valid_with_host_ip_and_error_result", - "input": {"host": "1.1.1.1", "test": "pytest_unit_test", "result": "error"}, - "expected_result": "valid", - }, - { - "name": "valid_full", - "input": { - "host": "1.1.1.1", - "test": "pytest_unit_test", - "result": "success", - "messages": ["test"], - }, - "expected_result": "valid", - }, - { - "name": "invalid_by_host", - "input": {"host": "demo.arista.com", "test": "pytest_unit_test"}, - "expected_result": "invalid", - }, - { - "name": "invalid_by_test", - "input": {"host": "1.1.1.1"}, - "expected_result": "invalid", - }, - { - "name": "invelid_by_result", - "input": {"host": "1.1.1.1", "test": "pytest_unit_test", "result": "ok"}, - "expected_result": "invalid", - }, +TEST_RESULT_SET_STATUS = [ + {"name": "set_success", "target": "success", "message": "success"}, + {"name": "set_error", "target": "error", "message": "error"}, + {"name": "set_failure", "target": "failure", "message": "failure"}, + {"name": "set_skipped", "target": "skipped", "message": "skipped"}, + {"name": "set_unset", "target": "unset", "message": "unset"}, ] diff --git a/tests/data/test_catalog_not_a_list.yml b/tests/data/test_catalog_not_a_list.yml new file mode 100644 index 000000000..d8c42976d --- /dev/null +++ b/tests/data/test_catalog_not_a_list.yml @@ -0,0 +1,2 @@ +--- +anta.tests.configuration: true diff --git a/tests/data/test_catalog_test_definition_multiple_dicts.yml b/tests/data/test_catalog_test_definition_multiple_dicts.yml new file mode 100644 index 000000000..9287ee6d3 --- /dev/null +++ b/tests/data/test_catalog_test_definition_multiple_dicts.yml @@ -0,0 +1,9 @@ +--- +anta.tests.software: + - VerifyEOSVersion: + versions: + - 4.25.4M + - 4.26.1F + VerifyTerminAttrVersion: + versions: + - 4.25.4M diff --git a/tests/data/test_catalog_test_definition_not_a_dict.yml b/tests/data/test_catalog_test_definition_not_a_dict.yml new file mode 100644 index 000000000..052ad267a --- /dev/null +++ b/tests/data/test_catalog_test_definition_not_a_dict.yml @@ -0,0 +1,3 @@ +--- +anta.tests.software: + - VerifyEOSVersion diff --git a/tests/data/test_catalog_with_tags.yml b/tests/data/test_catalog_with_tags.yml new file mode 100644 index 000000000..0c8f5f60c --- /dev/null +++ b/tests/data/test_catalog_with_tags.yml @@ -0,0 +1,28 @@ +--- +anta.tests.system: + - VerifyUptime: + minimum: 10 + filters: + tags: ['fabric'] + - VerifyReloadCause: + filters: + tags: ['leaf', 'spine'] + - VerifyCoredump: + - VerifyAgentLogs: + - VerifyCPUUtilization: + filters: + tags: ['leaf'] + - VerifyMemoryUtilization: + filters: + tags: ['testdevice'] + - VerifyFileSystemUtilization: + - VerifyNTP: + +anta.tests.mlag: + - VerifyMlagStatus: + +anta.tests.interfaces: + - VerifyL3MTU: + mtu: 1500 + filters: + tags: ['demo'] diff --git a/tests/data/test_catalog_with_undefined_module.yml b/tests/data/test_catalog_with_undefined_module.yml new file mode 100644 index 000000000..f2e116b6e --- /dev/null +++ b/tests/data/test_catalog_with_undefined_module.yml @@ -0,0 +1,3 @@ +--- +anta.tests.undefined: + - MyTest: diff --git a/tests/data/test_catalog_with_undefined_module_nested.yml b/tests/data/test_catalog_with_undefined_module_nested.yml new file mode 100644 index 000000000..cf0f393ad --- /dev/null +++ b/tests/data/test_catalog_with_undefined_module_nested.yml @@ -0,0 +1,4 @@ +--- +anta.tests: + undefined: + - MyTest: diff --git a/tests/data/test_catalog_with_undefined_tests.yml b/tests/data/test_catalog_with_undefined_tests.yml new file mode 100644 index 000000000..8282714f2 --- /dev/null +++ b/tests/data/test_catalog_with_undefined_tests.yml @@ -0,0 +1,3 @@ +--- +anta.tests.software: + - FakeTest: diff --git a/tests/lib/fixture.py b/tests/lib/fixture.py index ef15d0f41..6a333a8c4 100644 --- a/tests/lib/fixture.py +++ b/tests/lib/fixture.py @@ -4,6 +4,7 @@ """Fixture for Anta Testing""" from __future__ import annotations +from os import environ from typing import Callable from unittest.mock import MagicMock, create_autospec @@ -117,3 +118,14 @@ def click_runner() -> CliRunner: Convenience fixture to return a click.CliRunner for cli testing """ return CliRunner() + + +@pytest.fixture(autouse=True) +def clean_anta_env_variables() -> None: + """ + Autouse fixture that cleans the various ANTA_FOO env variables + that could come from the user environment and make some tests fail. + """ + for envvar in environ: + if envvar.startswith("ANTA_"): + environ.pop(envvar) diff --git a/tests/lib/utils.py b/tests/lib/utils.py index a97da7995..1d6e70907 100644 --- a/tests/lib/utils.py +++ b/tests/lib/utils.py @@ -45,7 +45,7 @@ def default_anta_env() -> dict[str, str]: "ANTA_USERNAME": "anta", "ANTA_PASSWORD": "formica", "ANTA_INVENTORY": str(Path(__file__).parent.parent / "data" / "test_inventory.yml"), - "ANTA_NRFU_CATALOG": str(Path(__file__).parent.parent / "data" / "test_catalog.yml"), + "ANTA_CATALOG": str(Path(__file__).parent.parent / "data" / "test_catalog.yml"), } diff --git a/tests/units/result_manager/test_models.py b/tests/units/result_manager/test_models.py index 54e802745..20815b66f 100644 --- a/tests/units/result_manager/test_models.py +++ b/tests/units/result_manager/test_models.py @@ -4,197 +4,54 @@ """ANTA Result Manager models unit tests.""" from __future__ import annotations -import logging -from typing import Any +from typing import Any, Callable import pytest from anta.result_manager.models import TestResult -from tests.data.json_data import TEST_RESULT_UNIT +from tests.data.json_data import TEST_RESULT_SET_STATUS from tests.lib.utils import generate_test_ids_dict -pytest.skip(reason="Not yet ready for CI", allow_module_level=True) +# pytest.skip(reason="Not yet ready for CI", allow_module_level=True) -class Test_InventoryUnitModels: +class TestTestResultModels: """Test components of anta.result_manager.models.""" - @pytest.mark.parametrize("test_definition", TEST_RESULT_UNIT, ids=generate_test_ids_dict) - def test_anta_result_init_valid(self, test_definition: dict[str, Any]) -> None: - """Test Result model. - - Test structure: - --------------- - { - 'name': 'valid_full', - 'input': {"host": '1.1.1.1', 'test': 'pytest_unit_test', 'result': 'success', 'messages': ['test']}, - 'expected_result': 'valid', - } - - """ - if test_definition["expected_result"] == "invalid": - pytest.skip("Not concerned by the test") - try: - result = TestResult(**test_definition["input"]) - logging.info(f"TestResult is {result.dict()}") - # pylint: disable=W0703 - except Exception as e: - logging.error(f"Error loading data:\n{str(e)}") - assert False - - @pytest.mark.parametrize("test_definition", TEST_RESULT_UNIT, ids=generate_test_ids_dict) - def test_anta_result_init_invalid(self, test_definition: dict[str, Any]) -> None: - """Test Result model. - - Test structure: - --------------- - { - 'name': 'valid_full', - 'input': {"host": '1.1.1.1', 'test': 'pytest_unit_test', 'result': 'success', 'messages': ['test']}, - 'expected_result': 'valid', - } - - """ - if test_definition["expected_result"] == "valid": - pytest.skip("Not concerned by the test") - try: - TestResult(**test_definition["input"]) - except ValueError as e: - logging.warning(f"Error loading data:\n{str(e)}") - else: - logging.error("An exception is expected here") - assert False - - @pytest.mark.parametrize("test_definition", TEST_RESULT_UNIT, ids=generate_test_ids_dict) - def test_anta_result_set_status_success(self, test_definition: dict[str, Any]) -> None: - """Test Result model. - - Test structure: - --------------- - { - 'name': 'valid_full', - 'input': {"host": '1.1.1.1', 'test': 'pytest_unit_test', 'result': 'success', 'messages': ['test']}, - 'expected_result': 'valid', - } - - """ - if test_definition["expected_result"] == "invalid": - pytest.skip("Not concerned by the test") - - result = TestResult(**test_definition["input"]) - - result.is_success() - assert result.result == "success" - result_message_len = len(result.messages) - - if "messages" in test_definition["input"]: - assert result_message_len == len(test_definition["input"]["messages"]) - else: - assert result_message_len == 0 - - # Adding one message - result.is_success("it is a great success") - assert result.result == "success" - assert len(result.messages) == result_message_len + 1 - - @pytest.mark.parametrize("test_definition", TEST_RESULT_UNIT, ids=generate_test_ids_dict) - def test_anta_result_set_status_failure(self, test_definition: dict[str, Any]) -> None: - """Test Result model. - - Test structure: - --------------- - { - 'name': 'valid_full', - 'input': {"host": '1.1.1.1', 'test': 'pytest_unit_test', 'result': 'success', 'messages': ['test']}, - 'expected_result': 'valid', - } - - """ - if test_definition["expected_result"] == "invalid": - pytest.skip("Not concerned by the test") - - result = TestResult(**test_definition["input"]) - - result.is_failure() - assert result.result == "failure" - result_message_len = len(result.messages) - - if "messages" in test_definition["input"]: - assert result_message_len == len(test_definition["input"]["messages"]) - else: - assert result_message_len == 0 - - # Adding one message - result.is_failure("it is a great failure") - assert result.result == "failure" - assert len(result.messages) == result_message_len + 1 - - @pytest.mark.parametrize("test_definition", TEST_RESULT_UNIT, ids=generate_test_ids_dict) - def test_anta_result_set_status_error(self, test_definition: dict[str, Any]) -> None: - """Test Result model. - - Test structure: - --------------- - { - 'name': 'valid_full', - 'input': {"host": '1.1.1.1', 'test': 'pytest_unit_test', 'result': 'success', 'messages': ['test']}, - 'expected_result': 'valid', - } - - """ - if test_definition["expected_result"] == "invalid": - pytest.skip("Not concerned by the test") - - result = TestResult(**test_definition["input"]) - - result.is_error() - assert result.result == "error" - result_message_len = len(result.messages) - - if "messages" in test_definition["input"]: - assert result_message_len == len(test_definition["input"]["messages"]) - else: - assert result_message_len == 0 - - # Adding one message - result.is_error(message="it is a great error") - assert result.result == "error" - assert len(result.messages) == result_message_len + 1 - - # Adding one exception object - e = Exception() - result.is_error(exception=e) - assert result.result == "error" - assert result.error == e - - @pytest.mark.parametrize("test_definition", TEST_RESULT_UNIT, ids=generate_test_ids_dict) - def test_anta_result_set_status_skipped(self, test_definition: dict[str, Any]) -> None: - """Test Result model. - - Test structure: - --------------- - { - 'name': 'valid_full', - 'input': {"host": '1.1.1.1', 'test': 'pytest_unit_test', 'result': 'success', 'messages': ['test']}, - 'expected_result': 'valid', - } - - """ - if test_definition["expected_result"] == "invalid": - pytest.skip("Not concerned by the test") - - result = TestResult(**test_definition["input"]) - - result.is_skipped() - assert result.result == "skipped" - result_message_len = len(result.messages) - - if "messages" in test_definition["input"]: - assert result_message_len == len(test_definition["input"]["messages"]) - else: - assert result_message_len == 0 - - # Adding one message - result.is_skipped("it is a great skipped") - assert result.result == "skipped" - assert len(result.messages) == result_message_len + 1 + @pytest.mark.parametrize("data", TEST_RESULT_SET_STATUS, ids=generate_test_ids_dict) + def test__is_status_foo(self, test_result_factory: Callable[[int], TestResult], data: dict[str, Any]) -> None: + """Test TestResult.is_foo methods.""" + testresult = test_result_factory(1) + assert testresult.result == "unset" + assert len(testresult.messages) == 0 + if data["target"] == "success": + testresult.is_success(data["message"]) + assert testresult.result == data["target"] + assert data["message"] in testresult.messages + if data["target"] == "failure": + testresult.is_failure(data["message"]) + assert testresult.result == data["target"] + assert data["message"] in testresult.messages + if data["target"] == "error": + testresult.is_error(data["message"]) + assert testresult.result == data["target"] + assert data["message"] in testresult.messages + if data["target"] == "skipped": + testresult.is_skipped(data["message"]) + assert testresult.result == data["target"] + assert data["message"] in testresult.messages + # no helper for unset, testing _set_status + if data["target"] == "unset": + testresult._set_status("unset", data["message"]) # pylint: disable=W0212 + assert testresult.result == data["target"] + assert data["message"] in testresult.messages + + @pytest.mark.parametrize("data", TEST_RESULT_SET_STATUS, ids=generate_test_ids_dict) + def test____str__(self, test_result_factory: Callable[[int], TestResult], data: dict[str, Any]) -> None: + """Test TestResult.__str__.""" + testresult = test_result_factory(1) + assert testresult.result == "unset" + assert len(testresult.messages) == 0 + testresult._set_status(data["target"], data["message"]) # pylint: disable=W0212 + assert testresult.result == data["target"] + assert str(testresult) == f"Test VerifyTest1 on device testdevice has result {data['target']}" diff --git a/tests/units/test_catalog.py b/tests/units/test_catalog.py new file mode 100644 index 000000000..ba486a317 --- /dev/null +++ b/tests/units/test_catalog.py @@ -0,0 +1,288 @@ +# Copyright (c) 2023 Arista Networks, Inc. +# Use of this source code is governed by the Apache License 2.0 +# that can be found in the LICENSE file. +""" +test anta.device.py +""" +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest +from pydantic import ValidationError +from yaml import safe_load + +from anta.catalog import AntaCatalog, AntaTestDefinition +from anta.models import AntaTest +from anta.tests.configuration import VerifyZeroTouch +from anta.tests.hardware import VerifyTemperature +from anta.tests.interfaces import VerifyL3MTU +from anta.tests.mlag import VerifyMlagStatus +from anta.tests.software import VerifyEOSVersion +from anta.tests.system import ( + VerifyAgentLogs, + VerifyCoredump, + VerifyCPUUtilization, + VerifyFileSystemUtilization, + VerifyMemoryUtilization, + VerifyNTP, + VerifyReloadCause, + VerifyUptime, +) +from tests.lib.utils import generate_test_ids_list +from tests.units.test_models import FakeTestWithInput + +# Test classes used as expected values + +DATA_DIR: Path = Path(__file__).parent.parent.resolve() / "data" + +INIT_CATALOG_DATA: list[dict[str, Any]] = [ + { + "name": "test_catalog", + "filename": "test_catalog.yml", + "tests": [ + (VerifyZeroTouch, None), + (VerifyTemperature, None), + (VerifyEOSVersion, VerifyEOSVersion.Input(versions=["4.25.4M", "4.26.1F"])), + (VerifyUptime, {"minimum": 86400}), + ], + }, + { + "name": "test_catalog_with_tags", + "filename": "test_catalog_with_tags.yml", + "tests": [ + ( + VerifyUptime, + VerifyUptime.Input( + minimum=10, + filters=VerifyUptime.Input.Filters(tags=["fabric"]), + ), + ), + (VerifyReloadCause, {"filters": {"tags": ["leaf", "spine"]}}), + (VerifyCoredump, VerifyCoredump.Input()), + (VerifyAgentLogs, AntaTest.Input()), + (VerifyCPUUtilization, VerifyCPUUtilization.Input(filters=VerifyCPUUtilization.Input.Filters(tags=["leaf"]))), + (VerifyMemoryUtilization, VerifyMemoryUtilization.Input(filters=VerifyMemoryUtilization.Input.Filters(tags=["testdevice"]))), + (VerifyFileSystemUtilization, None), + (VerifyNTP, {}), + (VerifyMlagStatus, None), + (VerifyL3MTU, {"mtu": 1500, "filters": {"tags": ["demo"]}}), + ], + }, +] +CATALOG_PARSE_FAIL_DATA: list[dict[str, Any]] = [ + { + "name": "undefined_tests", + "filename": "test_catalog_with_undefined_tests.yml", + "error": "FakeTest is not defined in Python module is not valid", + }, +] + +TESTS_SETTER_FAIL_DATA: list[dict[str, Any]] = [ + { + "name": "not_a_list", + "tests": "not_a_list", + "error": "The catalog must contain a list of tests", + }, + { + "name": "not_a_list_of_test_definitions", + "tests": [42, 43], + "error": "A test in the catalog must be an AntaTestDefinition instance", + }, +] + + +class Test_AntaCatalog: + """ + Test for anta.catalog.AntaCatalog + """ + + @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) + def test_parse(self, catalog_data: dict[str, Any]) -> None: + """ + Instantiate AntaCatalog from a file + """ + catalog: AntaCatalog = AntaCatalog.parse(str(DATA_DIR / catalog_data["filename"])) + + assert len(catalog.tests) == len(catalog_data["tests"]) + for test_id, (test, inputs) in enumerate(catalog_data["tests"]): + assert catalog.tests[test_id].test == test + if inputs is not None: + if isinstance(inputs, dict): + inputs = test.Input(**inputs) + assert inputs == catalog.tests[test_id].inputs + + @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) + def test_from_list(self, catalog_data: dict[str, Any]) -> None: + """ + Instantiate AntaCatalog from a list + """ + catalog: AntaCatalog = AntaCatalog.from_list(catalog_data["tests"]) + + assert len(catalog.tests) == len(catalog_data["tests"]) + for test_id, (test, inputs) in enumerate(catalog_data["tests"]): + assert catalog.tests[test_id].test == test + if inputs is not None: + if isinstance(inputs, dict): + inputs = test.Input(**inputs) + assert inputs == catalog.tests[test_id].inputs + + @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) + def test_from_dict(self, catalog_data: dict[str, Any]) -> None: + """ + Instantiate AntaCatalog from a dict + """ + with open(file=str(DATA_DIR / catalog_data["filename"]), mode="r", encoding="UTF-8") as file: + data = safe_load(file) + catalog: AntaCatalog = AntaCatalog.from_dict(data) + + assert len(catalog.tests) == len(catalog_data["tests"]) + for test_id, (test, inputs) in enumerate(catalog_data["tests"]): + assert catalog.tests[test_id].test == test + if inputs is not None: + if isinstance(inputs, dict): + inputs = test.Input(**inputs) + assert inputs == catalog.tests[test_id].inputs + + @pytest.mark.parametrize("catalog_data", CATALOG_PARSE_FAIL_DATA, ids=generate_test_ids_list(CATALOG_PARSE_FAIL_DATA)) + def test_parse_fail(self, catalog_data: dict[str, Any]) -> None: + """ + Errors when instantiating AntaCatalog from a file + """ + with pytest.raises(ValidationError) as exec_info: + AntaCatalog.parse(str(DATA_DIR / catalog_data["filename"])) + assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] + + def test_parse_fail_parsing(self, caplog: pytest.LogCaptureFixture) -> None: + """ + Errors when instantiating AntaCatalog from a file + """ + with pytest.raises(Exception) as exec_info: + AntaCatalog.parse(str(DATA_DIR / "catalog_does_not_exist.yml")) + assert "No such file or directory" in str(exec_info) + assert len(caplog.record_tuples) == 1 + _, _, message = caplog.record_tuples[0] + assert "Unable to parse ANTA Test Catalog file" in message + assert "FileNotFoundError ([Errno 2] No such file or directory" in message + + @pytest.mark.parametrize("catalog_data", CATALOG_FROM_LIST_FAIL_DATA, ids=generate_test_ids_list(CATALOG_FROM_LIST_FAIL_DATA)) + def test_from_list_fail(self, catalog_data: dict[str, Any]) -> None: + """ + Errors when instantiating AntaCatalog from a list of tuples + """ + with pytest.raises(ValidationError) as exec_info: + AntaCatalog.from_list(catalog_data["tests"]) + assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] + + @pytest.mark.parametrize("catalog_data", CATALOG_FROM_DICT_FAIL_DATA, ids=generate_test_ids_list(CATALOG_FROM_DICT_FAIL_DATA)) + def test_from_dict_fail(self, catalog_data: dict[str, Any]) -> None: + """ + Errors when instantiating AntaCatalog from a list of tuples + """ + with open(file=str(DATA_DIR / catalog_data["filename"]), mode="r", encoding="UTF-8") as file: + data = safe_load(file) + with pytest.raises(ValidationError) as exec_info: + AntaCatalog.from_dict(data) + assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] + + def test_filename(self) -> None: + """ + Test filename + """ + catalog = AntaCatalog(filename="test") + assert catalog.filename == Path("test") + catalog = AntaCatalog(filename=Path("test")) + assert catalog.filename == Path("test") + + @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) + def test__tests_setter_success(self, catalog_data: dict[str, Any]) -> None: + """ + Success when setting AntaCatalog.tests from a list of tuples + """ + catalog = AntaCatalog() + catalog.tests = [AntaTestDefinition(test=test, inputs=inputs) for test, inputs in catalog_data["tests"]] + assert len(catalog.tests) == len(catalog_data["tests"]) + for test_id, (test, inputs) in enumerate(catalog_data["tests"]): + assert catalog.tests[test_id].test == test + if inputs is not None: + if isinstance(inputs, dict): + inputs = test.Input(**inputs) + assert inputs == catalog.tests[test_id].inputs + + @pytest.mark.parametrize("catalog_data", TESTS_SETTER_FAIL_DATA, ids=generate_test_ids_list(TESTS_SETTER_FAIL_DATA)) + def test__tests_setter_fail(self, catalog_data: dict[str, Any]) -> None: + """ + Errors when setting AntaCatalog.tests from a list of tuples + """ + catalog = AntaCatalog() + with pytest.raises(ValueError) as exec_info: + catalog.tests = catalog_data["tests"] + assert catalog_data["error"] in str(exec_info) + + def test_get_tests_by_tags(self) -> None: + """ + Test AntaCatalog.test_get_tests_by_tags() + """ + catalog: AntaCatalog = AntaCatalog.parse(str(DATA_DIR / "test_catalog_with_tags.yml")) + tests: list[AntaTestDefinition] = catalog.get_tests_by_tags(tags=["leaf"]) + assert len(tests) == 2 diff --git a/tests/units/test_runner.py b/tests/units/test_runner.py index d1b2f9fd3..26caac43d 100644 --- a/tests/units/test_runner.py +++ b/tests/units/test_runner.py @@ -11,14 +11,18 @@ import pytest +from anta.catalog import AntaCatalog from anta.inventory import AntaInventory -from anta.models import AntaTest from anta.result_manager import ResultManager from anta.runner import main +from .test_models import FakeTest + if TYPE_CHECKING: from pytest import LogCaptureFixture +FAKE_CATALOG: AntaCatalog = AntaCatalog.from_list([(FakeTest, None)]) + @pytest.mark.asyncio async def test_runner_empty_tests(caplog: LogCaptureFixture, test_inventory: AntaInventory) -> None: @@ -29,7 +33,7 @@ async def test_runner_empty_tests(caplog: LogCaptureFixture, test_inventory: Ant test_inventory is a fixture that gives a default inventory for tests """ manager = ResultManager() - await main(manager, test_inventory, [], tags=[]) + await main(manager, test_inventory, AntaCatalog()) assert len(caplog.record_tuples) == 1 assert "The list of tests is empty, exiting" in caplog.records[0].message @@ -44,10 +48,7 @@ async def test_runner_empty_inventory(caplog: LogCaptureFixture) -> None: """ manager = ResultManager() inventory = AntaInventory() - # This is not vaidated in this test - tests: list[tuple[type[AntaTest], AntaTest.Input]] = [(AntaTest, {})] # type: ignore[type-abstract] - await main(manager, inventory, tests, tags=[]) - + await main(manager, inventory, FAKE_CATALOG) assert len(caplog.record_tuples) == 1 assert "The inventory is empty, exiting" in caplog.records[0].message @@ -61,16 +62,13 @@ async def test_runner_no_selected_device(caplog: LogCaptureFixture, test_invento test_inventory is a fixture that gives a default inventory for tests """ manager = ResultManager() - # This is not vaidated in this test - tests: list[tuple[type[AntaTest], AntaTest.Input]] = [(AntaTest, {})] # type: ignore[type-abstract] - - await main(manager, test_inventory, tests, tags=[]) + await main(manager, test_inventory, FAKE_CATALOG) assert "No device in the established state 'True' was found. There is no device to run tests against, exiting" in [record.message for record in caplog.records] # Reset logs and run with tags caplog.clear() - await main(manager, test_inventory, tests, tags=["toto"]) + await main(manager, test_inventory, FAKE_CATALOG, tags=["toto"]) assert "No device in the established state 'True' matching the tags ['toto'] was found. There is no device to run tests against, exiting" in [ record.message for record in caplog.records