diff --git a/spinn_utilities/config_holder.py b/spinn_utilities/config_holder.py index 336094fa..092993d8 100644 --- a/spinn_utilities/config_holder.py +++ b/spinn_utilities/config_holder.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections import defaultdict import logging import os -from typing import Any, Callable, Collection, List, Optional, Union +from typing import Any, Callable, Collection, Dict, List, Optional, Set, Union import spinn_utilities.conf_loader as conf_loader from spinn_utilities.configs import CamelCaseConfigParser from spinn_utilities.exceptions import ConfigException @@ -303,69 +304,87 @@ def config_options(section: str) -> List[str]: def _check_lines(py_path: str, line: str, lines: List[str], index: int, - method: Callable[[str, str], Any]): + method: Callable[[str, str], Any], + used_cfgs: Dict[str, Set[str]], start): """ Support for `_check_python_file`. Gets section and option name. :param str line: Line with get_config call :param list(str) lines: All lines in the file :param int index: index of line with `get_config` call + :param method: Method to call to check cfg + :param dict(str), set(str) used_cfgs: + Dict of used cfg options to be added to :raises ConfigException: If an unexpected or uncovered `get_config` found """ while ")" not in line: index += 1 line += lines[index] - parts = line[line.find("(", line.find("get_config")) + 1: + parts = line[line.find("(", line.find(start)) + 1: line.find(")")].split(",") section = parts[0].strip().replace("'", "").replace('"', '') - option = parts[1].strip() - if option[0] == "'": - option = option.replace("'", "") - elif option[0] == '"': - option = option.replace('"', '') - else: - print(line) - return - try: - method(section, option) - except Exception as original: - raise ConfigException( - f"failed in line:{index} of file: {py_path} with " - f"section:{section} option:{option}") from original + for i in range(1, len(parts)): + try: + option = parts[i].strip() + except IndexError as original: + raise ConfigException( + f"failed in line:{index} of file: {py_path} with {line}") \ + from original + if option[0] == "'": + option = option.replace("'", "") + elif option[0] == '"': + option = option.replace('"', '') + else: + print(line) + return + try: + method(section, option) + except Exception as original: + raise ConfigException( + f"failed in line:{index} of file: {py_path} with " + f"section:{section} option:{option}") from original + used_cfgs[section].add(option) -def _check_python_file(py_path: str): +def _check_python_file(py_path: str, used_cfgs: Dict[str, Set[str]]): """ A testing function to check that all the `get_config` calls work. :param str py_path: path to file to be checked + :param used_cfgs: dict of cfg options found :raises ConfigException: If an unexpected or uncovered `get_config` found """ with open(py_path, 'r', encoding="utf-8") as py_file: - lines = py_file.readlines() + lines = list(py_file) for index, line in enumerate(lines): - if "get_config_bool(" in line: - _check_lines( - py_path, line, lines, index, get_config_bool_or_none) - if "get_config_float(" in line: - _check_lines( - py_path, line, lines, index, get_config_float_or_none) - if "get_config_int(" in line: - _check_lines( - py_path, line, lines, index, get_config_int_or_none) - if "get_config_str(" in line: - _check_lines( - py_path, line, lines, index, get_config_str_or_none) + if ("skip_if_cfg" in line): + _check_lines(py_path, line, lines, index, + get_config_bool_or_none, used_cfgs, "skip_if_cfg") + if ("configuration.get" in line): + _check_lines(py_path, line, lines, index, + get_config_bool_or_none, used_cfgs, + "configuration.get") + if "get_config" not in line: + continue + if (("get_config_bool(" in line) or + ("get_config_bool_or_none(" in line)): + _check_lines(py_path, line, lines, index, + get_config_bool_or_none, used_cfgs, "get_config") + if (("get_config_float(" in line) or + ("get_config_float_or_none(" in line)): + _check_lines(py_path, line, lines, index, + get_config_float_or_none, used_cfgs, "get_config") + if (("get_config_int(" in line) or + ("get_config_int_or_none(" in line)): + _check_lines(py_path, line, lines, index, + get_config_int_or_none, used_cfgs, "get_config") + if (("get_config_str(" in line) or + ("get_config_str_or_none(" in line)): + _check_lines(py_path, line, lines, index, + get_config_str_or_none, used_cfgs, "get_config") if "get_config_str_list(" in line: - _check_lines(py_path, line, lines, index, get_config_str_list) - - -def _check_python_files(directory: str): - for root, _, files in os.walk(directory): - for file_name in files: - if file_name.endswith(".py"): - py_path = os.path.join(root, file_name) - _check_python_file(py_path) + _check_lines(py_path, line, lines, index, + get_config_str_list, used_cfgs, "get_config") def _find_double_defaults(repeaters: Optional[Collection[str]] = ()): @@ -444,13 +463,21 @@ def _check_cfgs(path: str): def run_config_checks(directories: Union[str, Collection[str]], *, exceptions: Union[str, Collection[str]] = (), - repeaters: Optional[Collection[str]] = ()): + repeaters: Optional[Collection[str]] = (), + check_all_used: bool = True): """ Master test. + Checks that all cfg options read have a default value in one of the + default files. + + Checks that all default options declared in the current repository + are used in that repository. + :param module: :param exceptions: :param repeaters: + :param bool check_all_used: Toggle for the used test. :raises ConfigException: If an incorrect directory passed in """ if isinstance(directories, str): @@ -466,6 +493,7 @@ def run_config_checks(directories: Union[str, Collection[str]], *, config1 = CamelCaseConfigParser() config1.read(__default_config_files) + used_cfgs: Dict[str, Set[str]] = defaultdict(set) for directory in directories: if not os.path.isdir(directory): raise ConfigException(f"Unable find {directory}") @@ -481,4 +509,21 @@ def run_config_checks(directories: Union[str, Collection[str]], *, _check_cfg_file(config1, cfg_path) elif file_name.endswith(".py"): py_path = os.path.join(root, file_name) - _check_python_file(py_path) + _check_python_file(py_path, used_cfgs) + + if not check_all_used: + return + + config2 = CamelCaseConfigParser() + config2.read(__default_config_files[-1]) + for section in config2: + if section not in used_cfgs: + if section == config1.default_section: + continue + raise ConfigException(f"cfg {section=} was never used") + found_options = used_cfgs[section] + found_options = set(map(config2.optionxform, found_options)) + for option in config2.options(section): + if option not in found_options: + raise ConfigException( + f"cfg {section=} {option=} was never used") diff --git a/spinn_utilities/spinn_utilities.cfg b/spinn_utilities/spinn_utilities.cfg index b7a7b7b6..fa892ea3 100644 --- a/spinn_utilities/spinn_utilities.cfg +++ b/spinn_utilities/spinn_utilities.cfg @@ -1,5 +1,10 @@ # DO NOT EDIT! -# Make any changes to cfg files in your home directory +# The are the default values +# Edit the cfg in your home directory to change your preferences +# Add / Edit a cfg in the run directory for script specific changes + +[Machine] +machine_spec_file = None [Mode] I_have_a_sense_of_humour = True diff --git a/unittests/test_cfg_checker.py b/unittests/test_cfg_checker.py new file mode 100644 index 00000000..78ed0f14 --- /dev/null +++ b/unittests/test_cfg_checker.py @@ -0,0 +1,29 @@ +# Copyright (c) 2017 The University of Manchester +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import spinn_utilities +from spinn_utilities.config_holder import run_config_checks +from spinn_utilities.config_setup import unittest_setup + + +class TestCfgChecker(unittest.TestCase): + + def setUp(self): + unittest_setup() + + def test_config_checks(self): + spinn_utilities_dir = spinn_utilities.__path__[0] + run_config_checks(directories=[spinn_utilities_dir], + exceptions=["config_holder.py"])