From 1d71454f4adfdf3ee38461badf4c13fdc1ad541e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Volker=20Wei=C3=9Fmann?= Date: Wed, 20 Sep 2023 18:28:53 +0200 Subject: [PATCH] rewriter: Major rewrite of the rewriter and the static introspection tool The rewriter and the static introspection tool used to be very broken, now it is *less* broken. Fixes #11763 --- mesonbuild/ast/__init__.py | 3 +- mesonbuild/ast/interpreter.py | 763 ++++++++++++---- mesonbuild/ast/introspection.py | 185 ++-- mesonbuild/ast/printer.py | 65 +- mesonbuild/build.py | 6 +- mesonbuild/interpreterbase/__init__.py | 4 + mesonbuild/interpreterbase/baseobjects.py | 6 + mesonbuild/interpreterbase/interpreterbase.py | 7 +- mesonbuild/mintro.py | 78 +- mesonbuild/mparser.py | 9 +- mesonbuild/rewriter.py | 811 ++++++++++-------- run_mypy.py | 7 +- test cases/rewrite/1 basic/addSrc.json | 38 + test cases/rewrite/1 basic/addTgt.json | 2 +- test cases/rewrite/1 basic/info.json | 20 + test cases/rewrite/1 basic/meson.build | 16 +- test cases/rewrite/1 basic/rmSrc.json | 40 +- test cases/rewrite/1 basic/rmTgt.json | 5 + .../rewrite/7 tricky dataflow/addSrc.json | 72 ++ .../rewrite/7 tricky dataflow/info.json | 32 + .../rewrite/7 tricky dataflow/meson.build | 35 + test cases/unit/118 rewrite/meson.build | 5 + unittests/allplatformstests.py | 4 +- unittests/rewritetests.py | 141 ++- 24 files changed, 1604 insertions(+), 750 deletions(-) create mode 100644 test cases/rewrite/7 tricky dataflow/addSrc.json create mode 100644 test cases/rewrite/7 tricky dataflow/info.json create mode 100644 test cases/rewrite/7 tricky dataflow/meson.build diff --git a/mesonbuild/ast/__init__.py b/mesonbuild/ast/__init__.py index d14620f71228..6eccdcd57a16 100644 --- a/mesonbuild/ast/__init__.py +++ b/mesonbuild/ast/__init__.py @@ -24,11 +24,10 @@ 'AstVisitor', 'AstPrinter', 'IntrospectionInterpreter', - 'BUILD_TARGET_FUNCTIONS', ] from .interpreter import AstInterpreter -from .introspection import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS +from .introspection import IntrospectionInterpreter from .visitor import AstVisitor from .postprocess import AstConditionLevel, AstIDGenerator, AstIndentationGenerator from .printer import AstPrinter, AstJSONPrinter diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index 382fa41970e0..2fc1656cd243 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -19,26 +19,38 @@ import os import sys import typing as T +from collections import defaultdict +from dataclasses import dataclass +import itertools from .. import mparser, mesonlib from .. import environment +from ..utils.core import HoldableObject + from ..interpreterbase import ( MesonInterpreterObject, + InterpreterObject, InterpreterBase, InvalidArguments, BreakRequest, ContinueRequest, Disabler, default_resolve_key, + is_disabled, + UnknownValue, + ObjectHolder, ) +from ..interpreterbase._unholder import _unholder + from ..interpreter import ( StringHolder, BooleanHolder, IntegerHolder, ArrayHolder, DictHolder, + primitives as P_OBJ ) from ..mparser import ( @@ -47,20 +59,24 @@ ArrayNode, AssignmentNode, BaseNode, - ElementaryNode, EmptyNode, + FunctionNode, IdNode, MethodNode, NotNode, PlusAssignmentNode, TernaryNode, TestCaseClauseNode, + SymbolNode, + Token, ) if T.TYPE_CHECKING: + from .baseobjects import SubProject + from pathlib import Path from .visitor import AstVisitor from ..interpreter import Interpreter - from ..interpreterbase import TYPE_nkwargs, TYPE_nvar + from ..interpreterbase import TYPE_nkwargs, TYPE_nvar, TYPE_var, TYPE_kwargs from ..mparser import ( AndNode, ComparisonNode, @@ -71,38 +87,138 @@ UMinusNode, ) -class DontCareObject(MesonInterpreterObject): - pass - -class MockExecutable(MesonInterpreterObject): - pass - -class MockStaticLibrary(MesonInterpreterObject): - pass - -class MockSharedLibrary(MesonInterpreterObject): - pass +# `IntrospectionFile` is to the `IntrospectionInterpreter` what `File` is to the normal `Interpreter`. +@dataclass +class IntrospectionFile(HoldableObject): + subdir: str + rel: str + def to_abs_path(self, root_dir: Path) -> Path: + return (root_dir / self.subdir / self.rel).resolve() -class MockCustomTarget(MesonInterpreterObject): - pass + def __hash__(self) -> int: + return hash((self.__class__.__name__, self.subdir, self.rel)) -class MockRunTarget(MesonInterpreterObject): +# `IntrospectionFileHolder` is to the `IntrospectionInterpreter` what `FileHolder` is to the normal `Interpreter`. +class IntrospectionFileHolder(ObjectHolder[IntrospectionFile]): pass -ADD_SOURCE = 0 -REMOVE_SOURCE = 1 - -_T = T.TypeVar('_T') -_V = T.TypeVar('_V') +# `IntrospectionDependency` is to the `IntrospectionInterpreter` what `Dependency` is to the normal `Interpreter`. +@dataclass +class IntrospectionDependency(MesonInterpreterObject): + name: str + required: T.Union[bool, UnknownValue] + version: T.List[str] + has_fallback: bool + conditional: bool + node: FunctionNode + +# `IntrospectionBuildTarget` is to the `IntrospectionInterpreter` what `BuildTarget` is to the normal `Interpreter`. +@dataclass +class IntrospectionBuildTarget(MesonInterpreterObject): + name: str + id: str + typename: str + defined_in: str + subdir: str + build_by_default: bool + installed: bool + outputs: T.List[str] + source_nodes: T.List[BaseNode] + extra_files: BaseNode + kwargs: T.Dict[str, T.Union[T.Union[str, int, bool, T.List[T.Any], T.Dict[str, T.Any]], HoldableObject, MesonInterpreterObject]] + node: FunctionNode + +def flatten_nested_lists(ar: T.Any) -> T.List[T.Any]: + flat_list = [] + for x in ar: + if isinstance(x, list): + flat_list += flatten_nested_lists(x) + else: + flat_list.append(x) + return flat_list + +def create_symbol(val: str) -> SymbolNode: + return SymbolNode(Token('', '', 0, 0, 0, (0, 0), val)) + +class DataflowDAG: + src_to_tgts: T.DefaultDict[T.Union[BaseNode, UnknownValue], T.Set[T.Union[BaseNode, UnknownValue]]] + tgt_to_srcs: T.DefaultDict[T.Union[BaseNode, UnknownValue], T.Set[T.Union[BaseNode, UnknownValue]]] + + def __init__(self) -> None: + self.src_to_tgts = defaultdict(set) + self.tgt_to_srcs = defaultdict(set) + + def add_edge(self, source: T.Union[BaseNode, UnknownValue], target: T.Union[BaseNode, UnknownValue]) -> None: + self.src_to_tgts[source].add(target) + self.tgt_to_srcs[target].add(source) + + # Returns all nodes in the DAG that are reachable from a node in `srcs`. + # In other words, A node `a` is part of the returned set exactly if data + # from `srcs` flows into `a`, directly or indirectly. + # Certain edges are ignored. + def reachable(self, srcs: T.Set[T.Union[BaseNode, UnknownValue]], reverse: bool) -> T.Set[T.Union[BaseNode, UnknownValue]]: + reachable = set(srcs) + active = set(srcs) + while True: + oldlen = len(reachable) + if reverse: + new: T.Set[T.Union[BaseNode, UnknownValue]] = set() + for tgt in active: + new.update(src for src in self.tgt_to_srcs[tgt] if not ((isinstance(src, FunctionNode) and src.func_name.value not in ['files', 'get_variable']) or isinstance(src, MethodNode))) + else: + new = set() + for src in active: + if (isinstance(src, FunctionNode) and src.func_name.value not in ['files', 'get_variable']) or isinstance(src, MethodNode): + continue + new.update(tgt for tgt in self.src_to_tgts[src]) + if len(new) == 0: + return reachable + reachable.update(new) + active = new + + # Returns all paths from src to target. + # Certain edges are ignored. + def find_all_paths(self, src: T.Union[BaseNode, UnknownValue], target: T.Union[BaseNode, UnknownValue]) -> T.List[T.List[T.Union[BaseNode, UnknownValue]]]: + queue = [] + queue.append((src, [src])) + paths = [] + while len(queue) != 0: + cur, path = queue.pop() + if cur == target: + paths.append(path) + if (isinstance(cur, FunctionNode) and cur.func_name.value not in ['files', 'get_variable']) or isinstance(cur, MethodNode): + continue + next = [(tgt, path + [tgt]) for tgt in self.src_to_tgts[cur]] + queue += next + return paths class AstInterpreter(InterpreterBase): - def __init__(self, source_root: str, subdir: str, subproject: str, visitors: T.Optional[T.List[AstVisitor]] = None): + def __init__(self, source_root: str, subdir: str, subproject: 'SubProject', visitors: T.Optional[T.List[AstVisitor]] = None): super().__init__(source_root, subdir, subproject) self.visitors = visitors if visitors is not None else [] self.processed_buildfiles: T.Set[str] = set() - self.assignments: T.Dict[str, BaseNode] = {} + self.nesting: T.List[int] = [] + self.cur_assignments: T.DefaultDict[str, T.List[T.Tuple[T.List[int], T.Union[BaseNode, UnknownValue]]]] = defaultdict(list) + self.all_assignment_nodes: T.DefaultDict[str, T.List[AssignmentNode]] = defaultdict(list) + # dataflow_dag is an acyclic directed graph that contains an edge + # from one instance of `BaseNode` to another instance of `BaseNode` if + # data flows directly from one to the other. Example: If meson.build + # contains this: + # var = 'foo' + '123' + # executable(var, 'src.c') + # var = 'bar' + # dataflow_dag will contain an edge from the IdNode corresponding to + # 'var' in line 2 to the ArithmeticNode corresponding to 'foo' + '123'. + # This graph is crucial for e.g. node_to_runtime_value because we have + # to know that 'var' in line2 is 'foo123' and not 'bar'. + self.dataflow_dag = DataflowDAG() + self.funcvals: T.Dict[BaseNode, T.Union[BaseNode, InterpreterObject]] = {} + self.tainted = False self.assign_vals: T.Dict[str, T.Any] = {} - self.reverse_assignment: T.Dict[str, BaseNode] = {} + self.build_func_dict() + self.build_holder_map() + + def build_func_dict(self) -> None: self.funcs.update({'project': self.func_do_nothing, 'test': self.func_do_nothing, 'benchmark': self.func_do_nothing, @@ -135,7 +251,7 @@ def __init__(self, source_root: str, subdir: str, subproject: str, visitors: T.O 'vcs_tag': self.func_do_nothing, 'add_languages': self.func_do_nothing, 'declare_dependency': self.func_do_nothing, - 'files': self.func_do_nothing, + 'files': self.func_files, 'executable': self.func_do_nothing, 'static_library': self.func_do_nothing, 'shared_library': self.func_do_nothing, @@ -144,9 +260,9 @@ def __init__(self, source_root: str, subdir: str, subproject: str, visitors: T.O 'custom_target': self.func_do_nothing, 'run_target': self.func_do_nothing, 'subdir': self.func_subdir, - 'set_variable': self.func_do_nothing, - 'get_variable': self.func_do_nothing, - 'unset_variable': self.func_do_nothing, + 'set_variable': self.func_set_variable, + 'get_variable': self.func_get_variable, + 'unset_variable': self.func_unset_variable, 'is_disabler': self.func_do_nothing, 'is_variable': self.func_do_nothing, 'disabler': self.func_do_nothing, @@ -164,21 +280,28 @@ def __init__(self, source_root: str, subdir: str, subproject: str, visitors: T.O 'debug': self.func_do_nothing, }) - def _unholder_args(self, args: _T, kwargs: _V) -> T.Tuple[_T, _V]: - return args, kwargs + def build_holder_map(self) -> None: + self.holder_map.update({ + # Primitives + list: P_OBJ.ArrayHolder, + dict: P_OBJ.DictHolder, + int: P_OBJ.IntegerHolder, + bool: P_OBJ.BooleanHolder, + str: P_OBJ.StringHolder, - def _holderify(self, res: _T) -> _T: - return res + # Meson types + IntrospectionFile: IntrospectionFileHolder, + }) - def func_do_nothing(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[str, TYPE_nvar]) -> bool: - return True + def func_do_nothing(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> UnknownValue: + return UnknownValue() def load_root_meson_file(self) -> None: super().load_root_meson_file() for i in self.visitors: self.ast.accept(i) - def func_subdir(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[str, TYPE_nvar]) -> None: + def func_subdir(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> None: args = self.flatten_args(args) if len(args) != 1 or not isinstance(args[0], str): sys.stderr.write(f'Unable to evaluate subdir({args}) in AstInterpreter --> Skipping\n') @@ -214,24 +337,52 @@ def func_subdir(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[st self.evaluate_codeblock(codeblock) self.subdir = prev_subdir - def method_call(self, node: BaseNode) -> bool: - return True + def inner_method_call(self, obj: BaseNode, method_name: str, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> InterpreterObject: + for arg in itertools.chain(args, kwargs.values()): + if isinstance(arg, UnknownValue): + return UnknownValue() + + if isinstance(obj, str): + result = StringHolder(obj, T.cast('Interpreter', self)).method_call(method_name, args, kwargs) + elif isinstance(obj, bool): + result = BooleanHolder(obj, T.cast('Interpreter', self)).method_call(method_name, args, kwargs) + elif isinstance(obj, int): + result = IntegerHolder(obj, T.cast('Interpreter', self)).method_call(method_name, args, kwargs) + elif isinstance(obj, list): + result = ArrayHolder(obj, T.cast('Interpreter', self)).method_call(method_name, args, kwargs) + elif isinstance(obj, dict): + result = DictHolder(obj, T.cast('Interpreter', self)).method_call(method_name, args, kwargs) + else: + return self._holderify(UnknownValue()) + return self._holderify(result) + + def method_call(self, node: mparser.MethodNode) -> None: + invocable = node.source_object + self.evaluate_statement(invocable) + obj = self.node_to_runtime_value(invocable) + method_name = node.name.value + (h_args, h_kwargs) = self.reduce_arguments(node.args) + (args, kwargs) = self._unholder_args(h_args, h_kwargs) + res: InterpreterObject + if is_disabled(args, kwargs): + res = Disabler() + else: + res = self.inner_method_call(obj, method_name, args, kwargs) + self.funcvals[node] = res - def evaluate_fstring(self, node: mparser.FormatStringNode) -> str: - assert isinstance(node, mparser.FormatStringNode) - return node.value + def evaluate_fstring(self, node: T.Union[mparser.FormatStringNode, mparser.MultilineFormatStringNode]) -> None: + assert isinstance(node, (mparser.FormatStringNode, mparser.MultilineFormatStringNode)) - def evaluate_arraystatement(self, cur: mparser.ArrayNode) -> TYPE_nvar: - return self.reduce_arguments(cur.args)[0] + def evaluate_arraystatement(self, cur: mparser.ArrayNode) -> None: + for arg in cur.args.arguments: + self.evaluate_statement(arg) - def evaluate_arithmeticstatement(self, cur: ArithmeticNode) -> int: + def evaluate_arithmeticstatement(self, cur: ArithmeticNode) -> None: self.evaluate_statement(cur.left) self.evaluate_statement(cur.right) - return 0 - def evaluate_uminusstatement(self, cur: UMinusNode) -> int: + def evaluate_uminusstatement(self, cur: UMinusNode) -> None: self.evaluate_statement(cur.value) - return 0 def evaluate_ternary(self, node: TernaryNode) -> None: assert isinstance(node, TernaryNode) @@ -239,208 +390,433 @@ def evaluate_ternary(self, node: TernaryNode) -> None: self.evaluate_statement(node.trueblock) self.evaluate_statement(node.falseblock) - def evaluate_dictstatement(self, node: mparser.DictNode) -> TYPE_nkwargs: - def resolve_key(node: mparser.BaseNode) -> str: - if isinstance(node, mparser.BaseStringNode): - return node.value - return '__AST_UNKNOWN__' - arguments, kwargs = self.reduce_arguments(node.args, key_resolver=resolve_key) - assert not arguments - self.argument_depth += 1 - for key, value in kwargs.items(): - if isinstance(key, BaseNode): - self.evaluate_statement(key) - self.argument_depth -= 1 - return {} + def evaluate_dictstatement(self, node: mparser.DictNode) -> None: + for k,v in node.args.kwargs.items(): + self.evaluate_statement(k) + self.evaluate_statement(v) - def evaluate_plusassign(self, node: PlusAssignmentNode) -> None: - assert isinstance(node, PlusAssignmentNode) - # Cheat by doing a reassignment - self.assignments[node.var_name.value] = node.value # Save a reference to the value node - if node.value.ast_id: - self.reverse_assignment[node.value.ast_id] = node - self.assign_vals[node.var_name.value] = self.evaluate_statement(node.value) - - def evaluate_indexing(self, node: IndexNode) -> int: - return 0 - - def unknown_function_called(self, func_name: str) -> None: - pass + def evaluate_indexing(self, node: IndexNode) -> None: + self.evaluate_statement(node.iobject) + self.evaluate_statement(node.index) def reduce_arguments( self, args: mparser.ArgumentNode, key_resolver: T.Callable[[mparser.BaseNode], str] = default_resolve_key, duplicate_key_error: T.Optional[str] = None, - ) -> T.Tuple[T.List[TYPE_nvar], TYPE_nkwargs]: + include_unknown_args: bool = False, + ) -> T.Tuple[ + T.List[InterpreterObject], + T.Dict[str, InterpreterObject] + ]: + for arg in args.arguments: + self.evaluate_statement(arg) + for value in args.kwargs.values(): + self.evaluate_statement(value) if isinstance(args, ArgumentNode): kwargs: T.Dict[str, TYPE_nvar] = {} for key, val in args.kwargs.items(): kwargs[key_resolver(key)] = val if args.incorrect_order(): raise InvalidArguments('All keyword arguments must be after positional arguments.') - return self.flatten_args(args.arguments), kwargs + posargs = args.arguments else: - return self.flatten_args(args), {} - - def evaluate_comparison(self, node: ComparisonNode) -> bool: + posargs = args + kwargs = {} + + return ( + [ + self._holderify(el) + for el in self.flatten_args( + posargs, include_unknown_args=include_unknown_args + ) + ], + {k: self._holderify(self.node_to_runtime_value(v)) for k, v in kwargs.items()}, + ) + + def evaluate_comparison(self, node: ComparisonNode) -> None: self.evaluate_statement(node.left) self.evaluate_statement(node.right) - return False - def evaluate_andstatement(self, cur: AndNode) -> bool: + def evaluate_andstatement(self, cur: AndNode) -> None: self.evaluate_statement(cur.left) self.evaluate_statement(cur.right) - return False - def evaluate_orstatement(self, cur: OrNode) -> bool: + def evaluate_orstatement(self, cur: OrNode) -> None: self.evaluate_statement(cur.left) self.evaluate_statement(cur.right) - return False - def evaluate_notstatement(self, cur: NotNode) -> bool: + def evaluate_notstatement(self, cur: NotNode) -> None: self.evaluate_statement(cur.value) - return False + + def find_potential_writes(self, node: BaseNode) -> T.Set[str]: + if isinstance(node, mparser.ForeachClauseNode): + return {el.value for el in node.varnames} | self.find_potential_writes(node.block) + elif isinstance(node, mparser.CodeBlockNode): + ret = set() + for line in node.lines: + ret.update(self.find_potential_writes(line)) + return ret + elif isinstance(node, (AssignmentNode, PlusAssignmentNode)): + return set([node.var_name.value]) | self.find_potential_writes(node.value) + elif isinstance(node, IdNode): + return set() + elif isinstance(node, ArrayNode): + ret = set() + for arg in node.args.arguments: + ret.update(self.find_potential_writes(arg)) + return ret + elif isinstance(node, mparser.DictNode): + ret = set() + for k, v in node.args.kwargs.items(): + ret.update(self.find_potential_writes(k)) + ret.update(self.find_potential_writes(v)) + return ret + elif isinstance(node, FunctionNode): + ret = set() + for arg in node.args.arguments: + ret.update(self.find_potential_writes(arg)) + for arg in node.args.kwargs.values(): + ret.update(self.find_potential_writes(arg)) + return ret + elif isinstance(node, MethodNode): + ret = self.find_potential_writes(node.source_object) + for arg in node.args.arguments: + ret.update(self.find_potential_writes(arg)) + for arg in node.args.kwargs.values(): + ret.update(self.find_potential_writes(arg)) + return ret + elif isinstance(node, ArithmeticNode): + return self.find_potential_writes(node.left) | self.find_potential_writes(node.right) + elif isinstance(node, (mparser.NumberNode, mparser.BaseStringNode, mparser.BreakNode, mparser.BooleanNode, mparser.FormatStringNode, mparser.ContinueNode)): + return set() + elif isinstance(node, mparser.IfClauseNode): + if isinstance(node.elseblock, EmptyNode): + ret = set() + else: + ret = self.find_potential_writes(node.elseblock.block) + for i in node.ifs: + ret.update(self.find_potential_writes(i)) + return ret + elif isinstance(node, mparser.IndexNode): + return self.find_potential_writes(node.iobject) | self.find_potential_writes(node.index) + elif isinstance(node, mparser.IfNode): + return self.find_potential_writes(node.condition) | self.find_potential_writes(node.block) + elif isinstance(node, (mparser.ComparisonNode, mparser.OrNode, mparser.AndNode)): + return self.find_potential_writes(node.left) | self.find_potential_writes(node.right) + elif isinstance(node, mparser.NotNode): + return self.find_potential_writes(node.value) + elif isinstance(node, mparser.TernaryNode): + return self.find_potential_writes(node.condition) | self.find_potential_writes(node.trueblock) | self.find_potential_writes(node.falseblock) + elif isinstance(node, mparser.UMinusNode): + return self.find_potential_writes(node.value) + elif isinstance(node, mparser.ParenthesizedNode): + return self.find_potential_writes(node.inner) + raise NotImplementedError def evaluate_foreach(self, node: ForeachClauseNode) -> None: + asses = self.find_potential_writes(node) + for ass in asses: + self.cur_assignments[ass].append((self.nesting.copy(), UnknownValue())) try: self.evaluate_codeblock(node.block) except ContinueRequest: pass except BreakRequest: pass + for ass in asses: + self.cur_assignments[ass].append((self.nesting.copy(), UnknownValue())) # In case the foreach loops 0 times. def evaluate_if(self, node: IfClauseNode) -> None: + self.nesting.append(0) for i in node.ifs: self.evaluate_codeblock(i.block) + self.nesting[-1] += 1 if not isinstance(node.elseblock, EmptyNode): self.evaluate_codeblock(node.elseblock.block) - - def get_variable(self, varname: str) -> int: - return 0 - - def assignment(self, node: AssignmentNode) -> None: - assert isinstance(node, AssignmentNode) - self.assignments[node.var_name.value] = node.value # Save a reference to the value node - if node.value.ast_id: - self.reverse_assignment[node.value.ast_id] = node - self.assign_vals[node.var_name.value] = self.evaluate_statement(node.value) # Evaluate the value just in case - - def resolve_node(self, node: BaseNode, include_unknown_args: bool = False, id_loop_detect: T.Optional[T.List[str]] = None) -> T.Optional[T.Any]: - def quick_resolve(n: BaseNode, loop_detect: T.Optional[T.List[str]] = None) -> T.Any: - if loop_detect is None: - loop_detect = [] - if isinstance(n, IdNode): - assert isinstance(n.value, str) - if n.value in loop_detect or n.value not in self.assignments: - return [] - return quick_resolve(self.assignments[n.value], loop_detect = loop_detect + [n.value]) - elif isinstance(n, ElementaryNode): - return n.value + self.nesting.pop() + for var_name in self.cur_assignments: + flag = False + potential_values = [] + oldval = self.get_cur_value(var_name, allow_none=True) + if oldval is not None: + potential_values.append(oldval) + for nesting, value in self.cur_assignments[var_name]: + if len(nesting) > len(self.nesting): + potential_values.append(value) + self.cur_assignments[var_name] = [(nesting, v) for (nesting, v) in self.cur_assignments[var_name] if len(nesting) <= len(self.nesting)] + if len(potential_values) > 1 or (len(potential_values) > 0 and oldval is None): + uv = UnknownValue() + for pv in potential_values: + self.dataflow_dag.add_edge(pv, uv) + self.cur_assignments[var_name].append((self.nesting.copy(), uv)) + + def func_files(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> None: + ret: T.List[T.Union[IntrospectionFile, UnknownValue]] = [] + for arg in args: + if isinstance(arg, str): + ret.append(IntrospectionFile(self.subdir, arg)) + elif isinstance(arg, UnknownValue): + ret.append(UnknownValue()) else: - return n - - if id_loop_detect is None: - id_loop_detect = [] - result = None - - if not isinstance(node, BaseNode): - return None - - assert node.ast_id - if node.ast_id in id_loop_detect: - return None # Loop detected - id_loop_detect += [node.ast_id] - - # Try to evaluate the value of the node - if isinstance(node, IdNode): - result = quick_resolve(node) - - elif isinstance(node, ElementaryNode): - result = node.value - - elif isinstance(node, NotNode): - result = self.resolve_node(node.value, include_unknown_args, id_loop_detect) - if isinstance(result, bool): - result = not result - + raise TypeError + self.funcvals[node] = self._holderify(ret) + + def get_cur_value(self, var_name: str, allow_none: bool = False) -> T.Union[BaseNode, UnknownValue]: + if var_name in set(['meson', 'host_machine', 'build_machine', 'target_machine']): + return UnknownValue() + ret = None + for nesting, value in reversed(self.cur_assignments[var_name]): + if len(self.nesting) >= len(nesting) and self.nesting[:len(nesting)] == nesting: + ret = value + break + if ret is None and allow_none: + return ret + if ret is None and self.tainted: + return UnknownValue() + assert ret is not None + return ret + + # The function `node_to_runtime_value` takes a node of the ast as an + # argument and tries to return the same thing that would be passed to e.g. + # `func_message` if you put `message(node)` in your `meson.build` file and + # run `meson setup`. If this is not possible, `UnknownValue()` is returned. + # There are 3 Reasons why this is sometimes impossible: + # 1. Because the meson rewriter is imperfect and has not implemented everything yet + # 2. Because the value is different on different machines, example: + # ```meson + # node = somedep.found() + # message(node) + # ``` + # will print `true` on some machines and `false` on others, so + # `node_to_runtime_value` does not know whether to return `true` or + # `false` and will return `UnknownValue()`. + # 3. Here: + # ```meson + # foreach x : [1, 2] + # node = x + # message(node) + # endforeach + # ``` + # `node_to_runtime_value` does not know whether to return `1` or `2` and + # will return `UnknownValue()`. + # + # If you have something like + # ``` + # node = [123, somedep.found()] + # ``` + # `node_to_runtime_value` will return `[123, UnknownValue()]`. + def node_to_runtime_value(self, node: T.Union[UnknownValue, BaseNode, TYPE_var]) -> T.Any: + if isinstance(node, (mparser.BaseStringNode, mparser.BooleanNode, mparser.NumberNode)): + return node.value + elif isinstance(node, list): + return [self.node_to_runtime_value(x) for x in node] elif isinstance(node, ArrayNode): - result = node.args.arguments.copy() + return [self.node_to_runtime_value(x) for x in node.args.arguments] + elif isinstance(node, mparser.DictNode): + return {self.node_to_runtime_value(k): self.node_to_runtime_value(v) for k,v in node.args.kwargs.items()} + elif isinstance(node, IdNode): + assert len(self.dataflow_dag.tgt_to_srcs[node]) == 1 + val = next(iter(self.dataflow_dag.tgt_to_srcs[node])) + return self.node_to_runtime_value(val) + elif isinstance(node, (MethodNode, FunctionNode)): + funcval = self.funcvals[node] + if isinstance(funcval, (dict, str)): + return funcval + else: + if isinstance(funcval, BaseNode): + return self.node_to_runtime_value(funcval) + else: + return self.node_to_runtime_value(_unholder(funcval)) + elif isinstance(node, ArithmeticNode): + left = self.node_to_runtime_value(node.left) + right = self.node_to_runtime_value(node.right) + if isinstance(left, list) and isinstance(right, UnknownValue): + return left + [right] + if isinstance(right, list) and isinstance(left, UnknownValue): + return [left] + right + if isinstance(left, UnknownValue) or isinstance(right, UnknownValue): + return UnknownValue() + if node.operation == 'add': + if isinstance(left, dict) and isinstance(right, dict): + ret = left.copy() + for k,v in right.items(): + ret[k] = v + return ret + if isinstance(left, list): + if not isinstance(right, list): + right = [right] + return left + right + return left + right + elif node.operation == 'sub': + return left - right + elif node.operation == 'mul': + return left * right + elif node.operation == 'div': + if isinstance(left, int) and isinstance(right, int): + return left // right + elif isinstance(left, str) and isinstance(right, str): + return os.path.join(left, right).replace('\\', '/') + elif node.operation == 'mod': + if isinstance(left, int) and isinstance(right, int): + return left % right + elif isinstance(node, (UnknownValue, IntrospectionBuildTarget, IntrospectionFile, IntrospectionDependency, str, bool)): + return node + elif isinstance(node, mparser.IndexNode): + iobject = self.node_to_runtime_value(node.iobject) + index = self.node_to_runtime_value(node.index) + if isinstance(iobject, UnknownValue) or isinstance(index, UnknownValue): + return UnknownValue() + return iobject[index] + elif isinstance(node, mparser.ComparisonNode): + left = self.node_to_runtime_value(node.left) + right = self.node_to_runtime_value(node.right) + if isinstance(left, UnknownValue) or isinstance(right, UnknownValue): + return UnknownValue() + if node.ctype == '==': + return left == right + elif node.ctype == '!=': + return left != right + elif node.ctype == 'in': + return left in right + elif node.ctype == 'notin': + return left not in right + elif isinstance(node, mparser.FormatStringNode): + return UnknownValue() + elif isinstance(node, mparser.TernaryNode): + cond = self.node_to_runtime_value(node.condition) + if isinstance(cond, UnknownValue): + return UnknownValue() + if cond is True: + return self.node_to_runtime_value(node.trueblock) + if cond is False: + return self.node_to_runtime_value(node.falseblock) + elif isinstance(node, mparser.OrNode): + left = self.node_to_runtime_value(node.left) + right = self.node_to_runtime_value(node.right) + if isinstance(left, UnknownValue) or isinstance(right, UnknownValue): + return UnknownValue() + return left or right + elif isinstance(node, mparser.AndNode): + left = self.node_to_runtime_value(node.left) + right = self.node_to_runtime_value(node.right) + if isinstance(left, UnknownValue) or isinstance(right, UnknownValue): + return UnknownValue() + return left and right + elif isinstance(node, mparser.UMinusNode): + val = self.node_to_runtime_value(node.value) + if isinstance(val, UnknownValue): + return val + if isinstance(val, (int, float)): + return -val + elif isinstance(node, mparser.NotNode): + val = self.node_to_runtime_value(node.value) + if isinstance(val, UnknownValue): + return val + if isinstance(val, bool): + return not val + elif isinstance(node, mparser.ParenthesizedNode): + return self.node_to_runtime_value(node.inner) + raise NotImplementedError - elif isinstance(node, ArgumentNode): - result = node.arguments.copy() + def assignment(self, node: AssignmentNode) -> None: + assert isinstance(node, AssignmentNode) + self.evaluate_statement(node.value) + self.cur_assignments[node.var_name.value].append((self.nesting.copy(), node.value)) + self.all_assignment_nodes[node.var_name.value].append(node) - elif isinstance(node, ArithmeticNode): - if node.operation != 'add': - return None # Only handle string and array concats - l = self.resolve_node(node.left, include_unknown_args, id_loop_detect) - r = self.resolve_node(node.right, include_unknown_args, id_loop_detect) - if isinstance(l, str) and isinstance(r, str): - result = l + r # String concatenation detected + def evaluate_plusassign(self, node: PlusAssignmentNode) -> None: + assert isinstance(node, PlusAssignmentNode) + self.evaluate_statement(node.value) + lhs = self.get_cur_value(node.var_name.value) + newval: T.Union[UnknownValue, ArithmeticNode] + if isinstance(lhs, UnknownValue): + newval = UnknownValue() + else: + newval = mparser.ArithmeticNode(operation='add', left=lhs, operator=create_symbol('+'), right=node.value) + self.cur_assignments[node.var_name.value].append((self.nesting.copy(), newval)) + self.all_assignment_nodes[node.var_name.value].append(node) + + self.dataflow_dag.add_edge(lhs, newval) + self.dataflow_dag.add_edge(node.value, newval) + + def func_set_variable(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> None: + assert isinstance(node, FunctionNode) + if bool(node.args.kwargs): + raise InvalidArguments('set_variable accepts no keyword arguments') + if len(node.args.arguments) != 2: + raise InvalidArguments('set_variable requires exactly two positional arguments') + var_name = args[0] + value = node.args.arguments[1] + if isinstance(var_name, UnknownValue): + self.evaluate_statement(value) + self.tainted = True + return + assert isinstance(var_name, str) + equiv = AssignmentNode(var_name=IdNode(Token('', '', 0, 0, 0, (0, 0), var_name)), value=value, operator=create_symbol('=')) + equiv.ast_id = str(id(equiv)) + self.evaluate_statement(equiv) + + def func_get_variable(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> None: + assert isinstance(node, FunctionNode) + var_name = args[0] + assert isinstance(var_name, str) + val = self.get_cur_value(var_name) + self.dataflow_dag.add_edge(val, node) + self.funcvals[node] = val + + def func_unset_variable(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> None: + assert isinstance(node, FunctionNode) + if bool(node.args.kwargs): + raise InvalidArguments('unset_variable accepts no keyword arguments') + if len(node.args.arguments) != 1: + raise InvalidArguments('unset_variable requires exactly one positional arguments') + var_name = args[0] + assert isinstance(var_name, str) + self.cur_assignments[var_name].append((self.nesting.copy(), node)) + + def nodes_to_pretty_filelist(self, root_path: Path, subdir: str, nodes: T.List[BaseNode]) -> T.List[T.Union[str, UnknownValue]]: + def src_to_abs(src: T.Union[str, IntrospectionFile, UnknownValue]) -> T.Union[str, UnknownValue]: + if isinstance(src, str): + return os.path.normpath(os.path.join(root_path, subdir, src)) + elif isinstance(src, IntrospectionFile): + return str(src.to_abs_path(root_path)) + elif isinstance(src, UnknownValue): + return src else: - result = self.flatten_args(l, include_unknown_args, id_loop_detect) + self.flatten_args(r, include_unknown_args, id_loop_detect) + raise TypeError - elif isinstance(node, MethodNode): - src = quick_resolve(node.source_object) - margs = self.flatten_args(node.args.arguments, include_unknown_args, id_loop_detect) - mkwargs: T.Dict[str, TYPE_nvar] = {} - method_name = node.name.value - try: - if isinstance(src, str): - result = StringHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs) - elif isinstance(src, bool): - result = BooleanHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs) - elif isinstance(src, int): - result = IntegerHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs) - elif isinstance(src, list): - result = ArrayHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs) - elif isinstance(src, dict): - result = DictHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs) - except mesonlib.MesonException: - return None - - # Ensure that the result is fully resolved (no more nodes) - if isinstance(result, BaseNode): - result = self.resolve_node(result, include_unknown_args, id_loop_detect) - elif isinstance(result, list): - new_res: T.List[TYPE_nvar] = [] - for i in result: - if isinstance(i, BaseNode): - resolved = self.resolve_node(i, include_unknown_args, id_loop_detect) - if resolved is not None: - new_res += self.flatten_args(resolved, include_unknown_args, id_loop_detect) - else: - new_res += [i] - result = new_res - - return result + rtvals = flatten_nested_lists([self.node_to_runtime_value(sn) for sn in nodes]) + return [src_to_abs(x) for x in rtvals] - def flatten_args(self, args_raw: T.Union[TYPE_nvar, T.Sequence[TYPE_nvar]], include_unknown_args: bool = False, id_loop_detect: T.Optional[T.List[str]] = None) -> T.List[TYPE_nvar]: + def flatten_args(self, args_raw: T.Union[TYPE_var, T.Sequence[TYPE_var]], include_unknown_args: bool = False, id_loop_detect: T.Optional[T.List[str]] = None) -> T.List[TYPE_var]: # Make sure we are always dealing with lists if isinstance(args_raw, list): args = args_raw else: args = [args_raw] - flattened_args: T.List[TYPE_nvar] = [] + flattened_args: T.List[TYPE_var] = [] # Resolve the contents of args for i in args: if isinstance(i, BaseNode): - resolved = self.resolve_node(i, include_unknown_args, id_loop_detect) + resolved = self.node_to_runtime_value(i) if resolved is not None: if not isinstance(resolved, list): resolved = [resolved] flattened_args += resolved - elif isinstance(i, (str, bool, int, float)) or include_unknown_args: + elif isinstance(i, (str, bool, int, float, UnknownValue, IntrospectionFile)) or include_unknown_args: flattened_args += [i] + else: + raise NotImplementedError return flattened_args - def flatten_kwargs(self, kwargs: T.Dict[str, TYPE_nvar], include_unknown_args: bool = False) -> T.Dict[str, TYPE_nvar]: + def flatten_kwargs(self, kwargs: T.Dict[str, TYPE_var], include_unknown_args: bool = False) -> T.Dict[str, TYPE_var]: flattened_kwargs = {} for key, val in kwargs.items(): if isinstance(val, BaseNode): - resolved = self.resolve_node(val, include_unknown_args) + resolved = self.node_to_runtime_value(val) if resolved is not None: flattened_kwargs[key] = resolved elif isinstance(val, (str, bool, int, float)) or include_unknown_args: @@ -449,3 +825,24 @@ def flatten_kwargs(self, kwargs: T.Dict[str, TYPE_nvar], include_unknown_args: b def evaluate_testcase(self, node: TestCaseClauseNode) -> Disabler | None: return Disabler(subproject=self.subproject) + + def evaluate_statement(self, cur: mparser.BaseNode) -> None: + if hasattr(cur, 'args'): + for arg in cur.args.arguments: + self.dataflow_dag.add_edge(arg, cur) + for k,v in cur.args.kwargs.items(): + self.dataflow_dag.add_edge(v, cur) + for attr in ['source_object', 'left', 'right', 'items', 'iobject', 'index', 'condition']: + if hasattr(cur, attr): + assert isinstance(getattr(cur, attr), mparser.BaseNode) + self.dataflow_dag.add_edge(getattr(cur, attr), cur) + if isinstance(cur, mparser.IdNode): + self.dataflow_dag.add_edge(self.get_cur_value(cur.value), cur) + else: + super().evaluate_statement(cur) + + def function_call(self, node: mparser.FunctionNode) -> T.Optional[InterpreterObject]: + ret = super().function_call(node) + if ret is not None: + self.funcvals[node] = ret + return ret diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py index 7d91a85d581c..457acd60450a 100644 --- a/mesonbuild/ast/introspection.py +++ b/mesonbuild/ast/introspection.py @@ -25,23 +25,17 @@ from .. import coredata as cdata from ..build import Executable, Jar, SharedLibrary, SharedModule, StaticLibrary from ..compilers import detect_compiler_for -from ..interpreterbase import InvalidArguments +from ..interpreterbase import InvalidArguments, UnknownValue from ..mesonlib import MachineChoice, OptionKey -from ..mparser import BaseNode, ArithmeticNode, ArrayNode, ElementaryNode, IdNode, FunctionNode, BaseStringNode -from .interpreter import AstInterpreter +from ..mparser import BaseNode, ElementaryNode, IdNode, BaseStringNode, FunctionNode +from .interpreter import AstInterpreter, IntrospectionBuildTarget, IntrospectionDependency if T.TYPE_CHECKING: from ..build import BuildTarget - from ..interpreterbase import TYPE_nvar + from ..interpreterbase import TYPE_nvar, TYPE_var from .visitor import AstVisitor -# TODO: it would be nice to not have to duplicate this -BUILD_TARGET_FUNCTIONS = [ - 'executable', 'jar', 'library', 'shared_library', 'shared_module', - 'static_library', 'both_libraries' -] - class IntrospectionHelper(argparse.Namespace): # mimic an argparse namespace def __init__(self, cross_file: str): @@ -54,8 +48,11 @@ def __eq__(self, other: object) -> bool: return NotImplemented class IntrospectionInterpreter(AstInterpreter): - # Interpreter to detect the options without a build directory - # Most of the code is stolen from interpreter.Interpreter + # If you run `meson setup ...` the `Interpreter`-class walks over the AST. + # If you run `meson rewrite ...` and `meson introspect meson.build ...`, + # the `AstInterpreter`-class walks over the AST. + # Works without a build directory. + # Most of the code is stolen from interpreter.Interpreter. def __init__(self, source_root: str, subdir: str, @@ -79,9 +76,9 @@ def __init__(self, self.backend = backend self.default_options = {OptionKey('backend'): self.backend} self.project_data: T.Dict[str, T.Any] = {} - self.targets: T.List[T.Dict[str, T.Any]] = [] - self.dependencies: T.List[T.Dict[str, T.Any]] = [] - self.project_node: BaseNode = None + self.targets: T.List[IntrospectionBuildTarget] = [] + self.dependencies: T.List[IntrospectionDependency] = [] + self.project_node: FunctionNode = None self.funcs.update({ 'add_languages': self.func_add_languages, @@ -96,9 +93,10 @@ def __init__(self, 'both_libraries': self.func_both_lib, }) - def func_project(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[str, TYPE_nvar]) -> None: + def func_project(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> None: if self.project_node: raise InvalidArguments('Second call to project()') + assert isinstance(node, FunctionNode) self.project_node = node if len(args) < 1: raise InvalidArguments('Not enough arguments to project(). Needs at least the project name.') @@ -128,9 +126,8 @@ def func_project(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[s if not self.is_subproject() and 'subproject_dir' in kwargs: spdirname = kwargs['subproject_dir'] - if isinstance(spdirname, BaseStringNode): - assert isinstance(spdirname.value, str) - self.subproject_dir = spdirname.value + assert isinstance(spdirname, str) + self.subproject_dir = spdirname if not self.is_subproject(): self.project_data['subprojects'] = [] subprojects_dir = os.path.join(self.source_root, self.subproject_dir) @@ -157,19 +154,21 @@ def do_subproject(self, dirname: str) -> None: except (mesonlib.MesonException, RuntimeError): return - def func_add_languages(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[str, TYPE_nvar]) -> None: + def func_add_languages(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> UnknownValue: kwargs = self.flatten_kwargs(kwargs) required = kwargs.get('required', True) if isinstance(required, cdata.UserFeatureOption): required = required.is_enabled() + assert isinstance(required, bool) if 'native' in kwargs: native = kwargs.get('native', False) self._add_languages(args, required, MachineChoice.BUILD if native else MachineChoice.HOST) else: for for_machine in [MachineChoice.BUILD, MachineChoice.HOST]: self._add_languages(args, required, for_machine) + return UnknownValue() - def _add_languages(self, raw_langs: T.List[TYPE_nvar], required: bool, for_machine: MachineChoice) -> None: + def _add_languages(self, raw_langs: T.List[TYPE_var], required: bool, for_machine: MachineChoice) -> None: langs: T.List[str] = [] for l in self.flatten_args(raw_langs): if isinstance(l, str): @@ -196,35 +195,39 @@ def _add_languages(self, raw_langs: T.List[TYPE_nvar], required: bool, for_machi options[k] = v self.coredata.add_compiler_options(options, lang, for_machine, self.environment) - def func_dependency(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[str, TYPE_nvar]) -> None: + def func_dependency(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> None: + assert isinstance(node, FunctionNode) args = self.flatten_args(args) - kwargs = self.flatten_kwargs(kwargs) + kwargs = self.flatten_kwargs(kwargs, True) if not args: return name = args[0] + assert isinstance(name, str) has_fallback = 'fallback' in kwargs required = kwargs.get('required', True) version = kwargs.get('version', []) if not isinstance(version, list): version = [version] - if isinstance(required, ElementaryNode): - required = required.value - if not isinstance(required, bool): - required = False - self.dependencies += [{ - 'name': name, - 'required': required, - 'version': version, - 'has_fallback': has_fallback, - 'conditional': node.condition_level > 0, - 'node': node - }] - - def build_target(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs_raw: T.Dict[str, TYPE_nvar], targetclass: T.Type[BuildTarget]) -> T.Optional[T.Dict[str, T.Any]]: + assert isinstance(required, (bool, UnknownValue)) + newdep = IntrospectionDependency( + name=name, + required=required, + version=version, + has_fallback=has_fallback, + conditional=node.condition_level > 0, + node=node) + self.dependencies += [newdep] + self.funcvals[node] = newdep + + def build_target(self, node: BaseNode, args: T.List[TYPE_var], kwargs_raw: T.Dict[str, TYPE_var], targetclass: T.Type[BuildTarget]) -> IntrospectionBuildTarget: + assert isinstance(node, FunctionNode) args = self.flatten_args(args) - if not args or not isinstance(args[0], str): - return None - name = args[0] + if isinstance(args[0], UnknownValue): + name = 'unknown' + else: + assert isinstance(args[0], str) + name = args[0] + srcqueue = [node] extra_queue = [] @@ -237,43 +240,24 @@ def build_target(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs_raw: T.Di kwargs = self.flatten_kwargs(kwargs_raw, True) - def traverse_nodes(inqueue: T.List[BaseNode]) -> T.List[BaseNode]: - res: T.List[BaseNode] = [] - while inqueue: - curr = inqueue.pop(0) - arg_node = None - assert isinstance(curr, BaseNode) - if isinstance(curr, FunctionNode): - arg_node = curr.args - elif isinstance(curr, ArrayNode): - arg_node = curr.args - elif isinstance(curr, IdNode): - # Try to resolve the ID and append the node to the queue - assert isinstance(curr.value, str) - var_name = curr.value - if var_name in self.assignments: - tmp_node = self.assignments[var_name] - if isinstance(tmp_node, (ArrayNode, IdNode, FunctionNode)): - inqueue += [tmp_node] - elif isinstance(curr, ArithmeticNode): - inqueue += [curr.left, curr.right] - if arg_node is None: - continue - arg_nodes = arg_node.arguments.copy() - # Pop the first element if the function is a build target function - if isinstance(curr, FunctionNode) and curr.func_name.value in BUILD_TARGET_FUNCTIONS: - arg_nodes.pop(0) - elementary_nodes = [x for x in arg_nodes if isinstance(x, (str, BaseStringNode))] - inqueue += [x for x in arg_nodes if isinstance(x, (FunctionNode, ArrayNode, IdNode, ArithmeticNode))] - if elementary_nodes: - res += [curr] - return res - - source_nodes = traverse_nodes(srcqueue) - extraf_nodes = traverse_nodes(extra_queue) + oldlen = len(node.args.arguments) + source_nodes = node.args.arguments[1:] + for k, v in node.args.kwargs.items(): + assert isinstance(k, IdNode) + if k.value == 'sources': + source_nodes.append(v) + assert oldlen == len(node.args.arguments) + + extraf_nodes = None + for k, v in node.args.kwargs.items(): + assert isinstance(k, IdNode) + if k.value == 'extra_files': + assert extraf_nodes is None + extraf_nodes = v + # Make sure nothing can crash when creating the build class - kwargs_reduced = {k: v for k, v in kwargs.items() if k in targetclass.known_kwargs and k in {'install', 'build_by_default', 'build_always'}} + kwargs_reduced = {k: v for k, v in kwargs.items() if k in targetclass.known_kwargs and k in {'install', 'build_by_default', 'build_always', 'name_prefix'}} kwargs_reduced = {k: v.value if isinstance(v, ElementaryNode) else v for k, v in kwargs_reduced.items()} kwargs_reduced = {k: v for k, v in kwargs_reduced.items() if not isinstance(v, BaseNode)} for_machine = MachineChoice.HOST @@ -281,29 +265,28 @@ def traverse_nodes(inqueue: T.List[BaseNode]) -> T.List[BaseNode]: empty_sources: T.List[T.Any] = [] # Passing the unresolved sources list causes errors kwargs_reduced['_allow_no_sources'] = True - target = targetclass(name, self.subdir, self.subproject, for_machine, empty_sources, [], objects, + target = targetclass(name, self.subdir, self.subproject, for_machine, empty_sources, None, objects, self.environment, self.coredata.compilers[for_machine], kwargs_reduced) target.process_compilers_late() - new_target = { - 'name': target.get_basename(), - 'id': target.get_id(), - 'type': target.get_typename(), - 'defined_in': os.path.normpath(os.path.join(self.source_root, self.subdir, environment.build_filename)), - 'subdir': self.subdir, - 'build_by_default': target.build_by_default, - 'installed': target.should_install(), - 'outputs': target.get_outputs(), - 'sources': source_nodes, - 'extra_files': extraf_nodes, - 'kwargs': kwargs, - 'node': node, - } + new_target = IntrospectionBuildTarget( + name=target.get_basename(), + id=target.get_id(), + typename=target.get_typename(), + defined_in=os.path.normpath(os.path.join(self.source_root, self.subdir, environment.build_filename)), + subdir=self.subdir, + build_by_default=target.build_by_default, + installed=target.should_install(), + outputs=target.get_outputs(), + source_nodes=source_nodes, + extra_files=extraf_nodes, + kwargs=kwargs, + node=node) self.targets += [new_target] return new_target - def build_library(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[str, TYPE_nvar]) -> T.Optional[T.Dict[str, T.Any]]: + def build_library(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> IntrospectionBuildTarget: default_library = self.coredata.get_option(OptionKey('default_library')) if default_library == 'shared': return self.build_target(node, args, kwargs, SharedLibrary) @@ -313,28 +296,28 @@ def build_library(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[ return self.build_target(node, args, kwargs, SharedLibrary) return None - def func_executable(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[str, TYPE_nvar]) -> T.Optional[T.Dict[str, T.Any]]: + def func_executable(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> IntrospectionBuildTarget: return self.build_target(node, args, kwargs, Executable) - def func_static_lib(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[str, TYPE_nvar]) -> T.Optional[T.Dict[str, T.Any]]: + def func_static_lib(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> IntrospectionBuildTarget: return self.build_target(node, args, kwargs, StaticLibrary) - def func_shared_lib(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[str, TYPE_nvar]) -> T.Optional[T.Dict[str, T.Any]]: + def func_shared_lib(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> IntrospectionBuildTarget: return self.build_target(node, args, kwargs, SharedLibrary) - def func_both_lib(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[str, TYPE_nvar]) -> T.Optional[T.Dict[str, T.Any]]: + def func_both_lib(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> IntrospectionBuildTarget: return self.build_target(node, args, kwargs, SharedLibrary) - def func_shared_module(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[str, TYPE_nvar]) -> T.Optional[T.Dict[str, T.Any]]: + def func_shared_module(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> IntrospectionBuildTarget: return self.build_target(node, args, kwargs, SharedModule) - def func_library(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[str, TYPE_nvar]) -> T.Optional[T.Dict[str, T.Any]]: + def func_library(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> IntrospectionBuildTarget: return self.build_library(node, args, kwargs) - def func_jar(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[str, TYPE_nvar]) -> T.Optional[T.Dict[str, T.Any]]: + def func_jar(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> IntrospectionBuildTarget: return self.build_target(node, args, kwargs, Jar) - def func_build_target(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[str, TYPE_nvar]) -> T.Optional[T.Dict[str, T.Any]]: + def func_build_target(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> IntrospectionBuildTarget: if 'target_type' not in kwargs: return None target_type = kwargs.pop('target_type') @@ -369,11 +352,11 @@ def extract_subproject_dir(self) -> T.Optional[str]: and also calls parse_project() on every subproject. ''' if not self.ast.lines: - return + return None project = self.ast.lines[0] # first line is always project() if not isinstance(project, FunctionNode): - return + return None for kw, val in project.args.kwargs.items(): assert isinstance(kw, IdNode), 'for mypy' if kw.value == 'subproject_dir': diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index 155b5fc5e75d..0b7fcf09322d 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -31,7 +31,40 @@ 'div': '/' } +# Also known as "order of operations" or "binding_power" +# This is the counterpart to Parser.e1, Parser.e2, Parser.e3, Parser.e4, Parser.e5, Parser.e6, Parser.e7, Parser.e8, Parser.e9 +def precedence_level(node: mparser.BaseNode) -> float: + if isinstance(node, (mparser.PlusAssignmentNode, mparser.AssignmentNode, mparser.TernaryNode)): + return 1.0 + elif isinstance(node, mparser.OrNode): + return 2.0 + elif isinstance(node, mparser.AndNode): + return 3.0 + elif isinstance(node, mparser.ComparisonNode): + return 4.0 + elif isinstance(node, mparser.ArithmeticNode): + if node.operation in set(['add', 'sub']): + return 5.1 + elif node.operation in set(['mod', 'mul', 'div']): + return 5.2 + elif isinstance(node, (mparser.NotNode, mparser.UMinusNode)): + return 6.0 + elif isinstance(node, mparser.FunctionNode): + return 7.0 + elif isinstance(node, (mparser.ArrayNode, mparser.DictNode)): + return 8.0 + elif isinstance(node, (mparser.BooleanNode, mparser.IdNode, mparser.NumberNode, mparser.BaseStringNode, mparser.FormatStringNode, mparser.MultilineFormatStringNode, mparser.EmptyNode)): + return 9.0 + elif isinstance(node, mparser.ParenthesizedNode): + # Parenthesize have the highest binding power, but since the AstPrinter + # ignores ParanthesizedNode, the binding power of the inner node is + # relevant. + return precedence_level(node.inner) + raise TypeError + class AstPrinter(AstVisitor): + escape_trans: T.Dict[int, str] = str.maketrans({'\'': '\\\'', '\\': '\\\\'}) + def __init__(self, indent: int = 2, arg_newline_cutoff: int = 5, update_ast_line_nos: bool = False): self.result = '' self.indent = indent @@ -76,12 +109,14 @@ def visit_NumberNode(self, node: mparser.NumberNode) -> None: node.lineno = self.curr_line or node.lineno def escape(self, val: str) -> str: - return val.translate(str.maketrans({'\'': '\\\'', - '\\': '\\\\'})) + return val.translate(self.escape_trans) def visit_StringNode(self, node: mparser.StringNode) -> None: assert isinstance(node.value, str) - self.append("'" + self.escape(node.value) + "'", node) + if '\n' in node.value: + self.append("'''" + self.escape(node.value) + "'''", node) + else: + self.append("'" + self.escape(node.value) + "'", node) node.lineno = self.curr_line or node.lineno def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None: @@ -137,11 +172,23 @@ def visit_ComparisonNode(self, node: mparser.ComparisonNode) -> None: node.lineno = self.curr_line or node.lineno node.right.accept(self) + def maybe_parentheses(self, outer: mparser.BaseNode, inner: mparser.BaseNode, parens: bool) -> None: + prec_outer = precedence_level(outer) + prec_inner = precedence_level(inner) + if parens: + self.append('(', inner) + inner.accept(self) + if parens: + self.append(')', inner) + def visit_ArithmeticNode(self, node: mparser.ArithmeticNode) -> None: - node.left.accept(self) + prec = precedence_level(node) + prec_left = precedence_level(node.left) + prec_right = precedence_level(node.right) + self.maybe_parentheses(node, node.left, prec > prec_left) self.append_padded(arithmic_map[node.operation], node) node.lineno = self.curr_line or node.lineno - node.right.accept(self) + self.maybe_parentheses(node, node.right, prec > prec_right or (prec == prec_right and node.operation in set(['sub', 'div', 'mod']))) def visit_NotNode(self, node: mparser.NotNode) -> None: node.lineno = self.curr_line or node.lineno @@ -252,21 +299,21 @@ def visit_ArgumentNode(self, node: mparser.ArgumentNode) -> None: class RawPrinter(AstVisitor): - def __init__(self): + def __init__(self) -> None: self.result = '' - def visit_default_func(self, node: mparser.BaseNode): + def visit_default_func(self, node: mparser.BaseNode) -> None: self.result += node.value if node.whitespaces: node.whitespaces.accept(self) - def visit_unary_operator(self, node: mparser.UnaryOperatorNode): + def visit_unary_operator(self, node: mparser.UnaryOperatorNode) -> None: node.operator.accept(self) node.value.accept(self) if node.whitespaces: node.whitespaces.accept(self) - def visit_binary_operator(self, node: mparser.BinaryOperatorNode): + def visit_binary_operator(self, node: mparser.BinaryOperatorNode) -> None: node.left.accept(self) node.operator.accept(self) node.right.accept(self) diff --git a/mesonbuild/build.py b/mesonbuild/build.py index c58447b55db8..4c7bbe34e1ba 100644 --- a/mesonbuild/build.py +++ b/mesonbuild/build.py @@ -41,7 +41,7 @@ is_header, is_object, is_source, clink_langs, sort_clink, all_languages, is_known_suffix, detect_static_linker ) -from .interpreterbase import FeatureNew, FeatureDeprecated +from .interpreterbase import FeatureNew, FeatureDeprecated, UnknownValue if T.TYPE_CHECKING: from typing_extensions import Literal @@ -640,7 +640,7 @@ def get_id(self) -> str: def process_kwargs_base(self, kwargs: T.Dict[str, T.Any]) -> None: if 'build_by_default' in kwargs: self.build_by_default = kwargs['build_by_default'] - if not isinstance(self.build_by_default, bool): + if not isinstance(self.build_by_default, (bool, UnknownValue)): raise InvalidArguments('build_by_default must be a boolean value.') elif kwargs.get('install', False): # For backward compatibility, if build_by_default is not explicitly @@ -870,7 +870,7 @@ def can_compile_remove_sources(compiler: 'Compiler', sources: T.List['FileOrStri removed = True return removed - def process_compilers_late(self): + def process_compilers_late(self) -> None: """Processes additional compilers after kwargs have been evaluated. This can add extra compilers that might be required by keyword diff --git a/mesonbuild/interpreterbase/__init__.py b/mesonbuild/interpreterbase/__init__.py index 3cb95303fbcd..a5c3b555e10a 100644 --- a/mesonbuild/interpreterbase/__init__.py +++ b/mesonbuild/interpreterbase/__init__.py @@ -75,6 +75,8 @@ 'TYPE_HoldableTypes', 'HoldableTypes', + + 'UnknownValue', ] from .baseobjects import ( @@ -100,6 +102,8 @@ SubProject, HoldableTypes, + + UnknownValue, ) from .decorators import ( diff --git a/mesonbuild/interpreterbase/baseobjects.py b/mesonbuild/interpreterbase/baseobjects.py index 4966978a0c2b..0e17c266b825 100644 --- a/mesonbuild/interpreterbase/baseobjects.py +++ b/mesonbuild/interpreterbase/baseobjects.py @@ -134,6 +134,12 @@ class MesonInterpreterObject(InterpreterObject): class MutableInterpreterObject: ''' Dummy class to mark the object type as mutable ''' +class UnknownValue(MesonInterpreterObject): + '''This class is only used for the rewriter/static introspection tool and + indicates that a value cannot be determined statically, either because of + limitations in our code or because the value differs from machine to + machine.''' + HoldableTypes = (HoldableObject, int, bool, str, list, dict) TYPE_HoldableTypes = T.Union[TYPE_elementary, HoldableObject] InterpreterObjectTypeVar = T.TypeVar('InterpreterObjectTypeVar', bound=TYPE_HoldableTypes) diff --git a/mesonbuild/interpreterbase/interpreterbase.py b/mesonbuild/interpreterbase/interpreterbase.py index ea6e37ca67ae..d6f1bc2f964c 100644 --- a/mesonbuild/interpreterbase/interpreterbase.py +++ b/mesonbuild/interpreterbase/interpreterbase.py @@ -243,6 +243,8 @@ def evaluate_statement(self, cur: mparser.BaseNode) -> T.Optional[InterpreterObj return self.evaluate_statement(cur.inner) elif isinstance(cur, mparser.TestCaseClauseNode): return self.evaluate_testcase(cur) + elif isinstance(cur, mparser.EmptyNode): + return None else: raise InvalidCode("Unknown statement.") return None @@ -505,7 +507,7 @@ def evaluate_indexing(self, node: mparser.IndexNode) -> InterpreterObject: def function_call(self, node: mparser.FunctionNode) -> T.Optional[InterpreterObject]: func_name = node.func_name.value - (h_posargs, h_kwargs) = self.reduce_arguments(node.args) + (h_posargs, h_kwargs) = self.reduce_arguments(node.args, include_unknown_args = True) (posargs, kwargs) = self._unholder_args(h_posargs, h_kwargs) if is_disabled(posargs, kwargs) and func_name not in {'get_variable', 'set_variable', 'unset_variable', 'is_disabler'}: return Disabler() @@ -533,7 +535,7 @@ def method_call(self, node: mparser.MethodNode) -> T.Optional[InterpreterObject] object_display_name = invocable.__class__.__name__ obj = self.evaluate_statement(invocable) method_name = node.name.value - (h_args, h_kwargs) = self.reduce_arguments(node.args) + (h_args, h_kwargs) = self.reduce_arguments(node.args, include_unknown_args = True) (args, kwargs) = self._unholder_args(h_args, h_kwargs) if is_disabled(args, kwargs): return Disabler() @@ -581,6 +583,7 @@ def reduce_arguments( args: mparser.ArgumentNode, key_resolver: T.Callable[[mparser.BaseNode], str] = default_resolve_key, duplicate_key_error: T.Optional[str] = None, + include_unknown_args: bool = False, ) -> T.Tuple[ T.List[InterpreterObject], T.Dict[str, InterpreterObject] diff --git a/mesonbuild/mintro.py b/mesonbuild/mintro.py index 7bd420510334..cd9f898a0239 100644 --- a/mesonbuild/mintro.py +++ b/mesonbuild/mintro.py @@ -30,13 +30,12 @@ import typing as T from . import build, mesonlib, coredata as cdata -from .ast import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS, AstConditionLevel, AstIDGenerator, AstIndentationGenerator, AstJSONPrinter +from .ast import IntrospectionInterpreter, AstConditionLevel, AstIDGenerator, AstIndentationGenerator, AstJSONPrinter from .backend import backends from .dependencies import Dependency from . import environment -from .interpreterbase import ObjectHolder +from .interpreterbase import ObjectHolder, UnknownValue from .mesonlib import OptionKey -from .mparser import FunctionNode, ArrayNode, ArgumentNode, BaseStringNode if T.TYPE_CHECKING: import argparse @@ -44,6 +43,12 @@ from .interpreter import Interpreter from .mparser import BaseNode +class IntrospectionEncoder(json.JSONEncoder): + def default(self, obj: T.Any) -> T.Any: + if isinstance(obj, UnknownValue): + return 'unknown' + return json.JSONEncoder.default(self, obj) + def get_meson_info_file(info_dir: str) -> str: return os.path.join(info_dir, 'meson-info.json') @@ -64,8 +69,7 @@ def __init__(self, def get_meson_introspection_types(coredata: T.Optional[cdata.CoreData] = None, builddata: T.Optional[build.Build] = None, - backend: T.Optional[backends.Backend] = None, - sourcedir: T.Optional[str] = None) -> 'T.Mapping[str, IntroCommand]': + backend: T.Optional[backends.Backend] = None) -> 'T.Mapping[str, IntroCommand]': if backend and builddata: benchmarkdata = backend.create_test_serialisation(builddata.get_benchmarks()) testdata = backend.create_test_serialisation(builddata.get_tests()) @@ -177,55 +181,34 @@ def get_target_dir(coredata: cdata.CoreData, subdir: str) -> str: else: return subdir -def list_targets_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[str, T.Union[bool, str, T.List[T.Union[str, T.Dict[str, T.Union[str, T.List[str], bool]]]]]]]: - tlist: T.List[T.Dict[str, T.Union[bool, str, T.List[T.Union[str, T.Dict[str, T.Union[str, T.List[str], bool]]]]]]] = [] - root_dir = Path(intr.source_root) - - def nodes_to_paths(node_list: T.List[BaseNode]) -> T.List[Path]: - res: T.List[Path] = [] - for n in node_list: - args: T.List[BaseNode] = [] - if isinstance(n, FunctionNode): - args = list(n.args.arguments) - if n.func_name.value in BUILD_TARGET_FUNCTIONS: - args.pop(0) - elif isinstance(n, ArrayNode): - args = n.args.arguments - elif isinstance(n, ArgumentNode): - args = n.arguments - for j in args: - if isinstance(j, BaseStringNode): - assert isinstance(j.value, str) - res += [Path(j.value)] - elif isinstance(j, str): - res += [Path(j)] - res = [root_dir / i['subdir'] / x for x in res] - res = [x.resolve() for x in res] - return res +def list_targets_from_source(intr: IntrospectionInterpreter) -> T.Any: + tlist = [] + root_dir = Path(intr.source_root).resolve() for i in intr.targets: - sources = nodes_to_paths(i['sources']) - extra_f = nodes_to_paths(i['extra_files']) - outdir = get_target_dir(intr.coredata, i['subdir']) + sources = intr.nodes_to_pretty_filelist(root_dir, i.subdir, i.source_nodes) + extra_files = intr.nodes_to_pretty_filelist(root_dir, i.subdir, [i.extra_files] if i.extra_files else []) + + outdir = get_target_dir(intr.coredata, i.subdir) tlist += [{ - 'name': i['name'], - 'id': i['id'], - 'type': i['type'], - 'defined_in': i['defined_in'], - 'filename': [os.path.join(outdir, x) for x in i['outputs']], - 'build_by_default': i['build_by_default'], + 'name': i.name, + 'id': i.id, + 'type': i.typename, + 'defined_in': i.defined_in, + 'filename': [os.path.join(outdir, x) for x in i.outputs], + 'build_by_default': i.build_by_default, 'target_sources': [{ 'language': 'unknown', 'compiler': [], 'parameters': [], - 'sources': [str(x) for x in sources], + 'sources': sources, 'generated_sources': [] }], 'depends': [], - 'extra_files': [str(x) for x in extra_f], + 'extra_files': extra_files, 'subproject': None, # Subprojects are not supported - 'installed': i['installed'] + 'installed': i.installed }] return tlist @@ -388,7 +371,7 @@ def list_deps_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[str, 'has_fallback', 'conditional', ] - result += [{k: v for k, v in i.items() if k in keys}] + result += [{k: v for k, v in i.__dict__.items() if k in keys}] return result def list_deps(coredata: cdata.CoreData, backend: backends.Backend) -> T.List[T.Dict[str, T.Union[str, T.List[str]]]]: @@ -513,7 +496,7 @@ def print_results(options: argparse.Namespace, results: T.Sequence[T.Tuple[str, return 1 elif len(results) == 1 and not options.force_dict: # Make to keep the existing output format for a single option - print(json.dumps(results[0][1], indent=indent)) + print(json.dumps(results[0][1], indent=indent, cls=IntrospectionEncoder)) else: out = {} for i in results: @@ -542,10 +525,11 @@ def run(options: argparse.Namespace) -> int: datadir = os.path.join(options.builddir, datadir) indent = 4 if options.indent else None results: T.List[T.Tuple[str, T.Union[dict, T.List[T.Any]]]] = [] - sourcedir = '.' if options.builddir == 'meson.build' else options.builddir[:-11] - intro_types = get_meson_introspection_types(sourcedir=sourcedir) + intro_types = get_meson_introspection_types() - if 'meson.build' in [os.path.basename(options.builddir), options.builddir]: + # TODO: This if clause is undocumented. + if os.path.basename(options.builddir) == 'meson.build': + sourcedir = '.' if options.builddir == 'meson.build' else options.builddir[:-11] # Make sure that log entries in other parts of meson don't interfere with the JSON output with redirect_stdout(sys.stderr): backend = backends.get_backend_from_name(options.backend) diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 0f63c9e5a47e..e093ed1fe854 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -204,7 +204,7 @@ def lex(self, filename: str) -> T.Generator[Token, None, None]: lines = value.split('\n') if len(lines) > 1: lineno += len(lines) - 1 - line_start = mo.end() - len(lines[-1]) + line_start = mo.end() - len(lines[-1]) - 3 elif tid == 'eol_cont': lineno += 1 line_start = loc @@ -373,6 +373,13 @@ def set_kwarg(self, name: IdNode, value: BaseNode) -> None: mlog.warning('This will be an error in future Meson releases.') self.kwargs[name] = value + def get_kwarg_or_default(self, name: str, default: BaseNode) -> BaseNode: + for k, v in self.kwargs.items(): + assert isinstance(k, IdNode) + if k.value == name: + return v + return default + def set_kwarg_no_check(self, name: BaseNode, value: BaseNode) -> None: self.kwargs[name] = value diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index a86b97eccabb..d5184573eea9 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -24,21 +24,31 @@ # - reindent? from __future__ import annotations -from .ast import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS, AstConditionLevel, AstIDGenerator, AstIndentationGenerator, AstPrinter +from .ast import IntrospectionInterpreter, AstConditionLevel, AstIDGenerator, AstIndentationGenerator, AstPrinter +from .ast.interpreter import IntrospectionBuildTarget, IntrospectionDependency, flatten_nested_lists, create_symbol +from .interpreterbase import UnknownValue, TV_func from mesonbuild.mesonlib import MesonException, setup_vsenv from . import mlog, environment from functools import wraps -from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BaseStringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, StringNode, SymbolNode +from .mparser import Token, ArrayNode, ArgumentNode, ArithmeticNode, AssignmentNode, BaseNode, BaseStringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, PlusAssignmentNode, StringNode +from .mintro import IntrospectionEncoder import json, os, re, sys import typing as T +from pathlib import Path if T.TYPE_CHECKING: - from .mparser import BaseNode + import argparse + from .mlog import AnsiDecorator + +BUILD_TARGET_FUNCTIONS = [ + 'executable', 'jar', 'library', 'shared_library', 'shared_module', + 'static_library', 'both_libraries', 'custom_target' +] class RewriterException(MesonException): pass -def add_arguments(parser, formatter=None): +def add_arguments(parser: argparse.ArgumentParser, formatter: None = None) -> None: parser.add_argument('-s', '--sourcedir', type=str, default='.', metavar='SRCDIR', help='Path to source directory.') parser.add_argument('-V', '--verbose', action='store_true', default=False, help='Enable verbose output') parser.add_argument('-S', '--skip-errors', dest='skip', action='store_true', default=False, help='Skip errors instead of aborting') @@ -74,12 +84,14 @@ def add_arguments(parser, formatter=None): cmd_parser.add_argument('json', help='JSON string or file to execute') class RequiredKeys: - def __init__(self, keys): + keys: T.Dict[str, T.Any] + + def __init__(self, keys: T.Dict[str, T.Any]): self.keys = keys - def __call__(self, f): + def __call__(self, f: TV_func) -> TV_func: @wraps(f) - def wrapped(*wrapped_args, **wrapped_kwargs): + def wrapped(*wrapped_args: T.Any, **wrapped_kwargs: T.Any) -> T.Any: assert len(wrapped_args) >= 2 cmd = wrapped_args[1] for key, val in self.keys.items(): @@ -102,12 +114,11 @@ def wrapped(*wrapped_args, **wrapped_kwargs): .format(key, choices, cmd[key])) return f(*wrapped_args, **wrapped_kwargs) - return wrapped - -def _symbol(val: str) -> SymbolNode: - return SymbolNode(Token('', '', 0, 0, 0, (0, 0), val)) + return T.cast('TV_func', wrapped) class MTypeBase: + node: BaseNode + def __init__(self, node: T.Optional[BaseNode] = None): if node is None: self.node = self._new_node() # lgtm [py/init-calls-subclass] (node creation does not depend on base class state) @@ -118,33 +129,33 @@ def __init__(self, node: T.Optional[BaseNode] = None): if isinstance(self.node, i): self.node_type = i - def _new_node(self): + def _new_node(self) -> BaseNode: # Overwrite in derived class raise RewriterException('Internal error: _new_node of MTypeBase was called') - def can_modify(self): + def can_modify(self) -> bool: return self.node_type is not None - def get_node(self): + def get_node(self) -> BaseNode: return self.node - def supported_nodes(self): + def supported_nodes(self) -> T.List[type]: # Overwrite in derived class return [] - def set_value(self, value): + def set_value(self, value: T.Any) -> None: # Overwrite in derived class mlog.warning('Cannot set the value of type', mlog.bold(type(self).__name__), '--> skipping') - def add_value(self, value): + def add_value(self, value: T.Any) -> None: # Overwrite in derived class mlog.warning('Cannot add a value of type', mlog.bold(type(self).__name__), '--> skipping') - def remove_value(self, value): + def remove_value(self, value: T.Any) -> None: # Overwrite in derived class mlog.warning('Cannot remove a value of type', mlog.bold(type(self).__name__), '--> skipping') - def remove_regex(self, value): + def remove_regex(self, value: T.Any) -> None: # Overwrite in derived class mlog.warning('Cannot remove a regex in type', mlog.bold(type(self).__name__), '--> skipping') @@ -152,96 +163,107 @@ class MTypeStr(MTypeBase): def __init__(self, node: T.Optional[BaseNode] = None): super().__init__(node) - def _new_node(self): + def _new_node(self) -> BaseNode: return StringNode(Token('', '', 0, 0, 0, None, '')) - def supported_nodes(self): + def supported_nodes(self) -> T.List[type]: return [StringNode] - def set_value(self, value): + def set_value(self, value: T.Any) -> None: + assert hasattr(self.node, 'value') # For mypy self.node.value = str(value) class MTypeBool(MTypeBase): def __init__(self, node: T.Optional[BaseNode] = None): super().__init__(node) - def _new_node(self): + def _new_node(self) -> BaseNode: return BooleanNode(Token('', '', 0, 0, 0, None, False)) - def supported_nodes(self): + def supported_nodes(self) -> T.List[type]: return [BooleanNode] - def set_value(self, value): + def set_value(self, value: T.Any) -> None: + assert hasattr(self.node, 'value') # For mypy self.node.value = bool(value) class MTypeID(MTypeBase): def __init__(self, node: T.Optional[BaseNode] = None): super().__init__(node) - def _new_node(self): + def _new_node(self) -> BaseNode: return IdNode(Token('', '', 0, 0, 0, None, '')) - def supported_nodes(self): + def supported_nodes(self) -> T.List[type]: return [IdNode] - def set_value(self, value): + def set_value(self, value: T.Any) -> None: + assert hasattr(self.node, 'value') # For mypy self.node.value = str(value) class MTypeList(MTypeBase): def __init__(self, node: T.Optional[BaseNode] = None): super().__init__(node) - def _new_node(self): - return ArrayNode(_symbol('['), ArgumentNode(Token('', '', 0, 0, 0, None, '')), _symbol(']')) + def _new_node(self) -> BaseNode: + return ArrayNode(create_symbol('['), ArgumentNode(Token('', '', 0, 0, 0, None, '')), create_symbol(']')) - def _new_element_node(self, value): + def _new_element_node(self, value: T.Any) -> BaseNode: # Overwrite in derived class raise RewriterException('Internal error: _new_element_node of MTypeList was called') - def _ensure_array_node(self): + def _ensure_array_node(self) -> None: if not isinstance(self.node, ArrayNode): tmp = self.node self.node = self._new_node() + assert hasattr(self.node, 'args') # For mypy + assert isinstance(self.node.args, ArgumentNode) # For mypy self.node.args.arguments += [tmp] - def _check_is_equal(self, node, value) -> bool: + def _check_is_equal(self, node: BaseNode, value: T.Any) -> bool: # Overwrite in derived class return False - def _check_regex_matches(self, node, regex: str) -> bool: + def _check_regex_matches(self, node: BaseNode, regex: str) -> bool: # Overwrite in derived class return False - def get_node(self): + def get_node(self) -> BaseNode: if isinstance(self.node, ArrayNode): + assert hasattr(self.node, 'args') # For mypy + assert isinstance(self.node.args, ArgumentNode) # For mypy if len(self.node.args.arguments) == 1: return self.node.args.arguments[0] return self.node - def supported_element_nodes(self): + def supported_element_nodes(self) -> T.List[type]: # Overwrite in derived class return [] - def supported_nodes(self): + def supported_nodes(self) -> T.List[type]: return [ArrayNode] + self.supported_element_nodes() - def set_value(self, value): + def set_value(self, value: T.Any) -> None: if not isinstance(value, list): value = [value] self._ensure_array_node() + assert hasattr(self.node, 'args') # For mypy + assert isinstance(self.node.args, ArgumentNode) # For mypy self.node.args.arguments = [] # Remove all current nodes for i in value: self.node.args.arguments += [self._new_element_node(i)] - def add_value(self, value): + def add_value(self, value: T.Any) -> None: if not isinstance(value, list): value = [value] self._ensure_array_node() for i in value: + assert hasattr(self.node, 'args') # For mypy + assert isinstance(self.node.args, ArgumentNode) # For mypy self.node.args.arguments += [self._new_element_node(i)] - def _remove_helper(self, value, equal_func): - def check_remove_node(node): + def _remove_helper(self, value: T.Any, equal_func: T.Callable[[T.Any, T.Any], bool]) -> None: + def check_remove_node(node: BaseNode) -> bool: for j in value: if equal_func(i, j): return True @@ -250,59 +272,61 @@ def check_remove_node(node): if not isinstance(value, list): value = [value] self._ensure_array_node() + assert hasattr(self.node, 'args') # For mypy + assert isinstance(self.node.args, ArgumentNode) # For mypy removed_list = [] for i in self.node.args.arguments: if not check_remove_node(i): removed_list += [i] self.node.args.arguments = removed_list - def remove_value(self, value): + def remove_value(self, value: T.Any) -> None: self._remove_helper(value, self._check_is_equal) - def remove_regex(self, regex: str): + def remove_regex(self, regex: str) -> None: self._remove_helper(regex, self._check_regex_matches) class MTypeStrList(MTypeList): def __init__(self, node: T.Optional[BaseNode] = None): super().__init__(node) - def _new_element_node(self, value): + def _new_element_node(self, value: T.Any) -> BaseNode: return StringNode(Token('', '', 0, 0, 0, None, str(value))) - def _check_is_equal(self, node, value) -> bool: + def _check_is_equal(self, node: BaseNode, value: T.Any) -> bool: if isinstance(node, BaseStringNode): - return node.value == value + return bool(node.value == value) return False - def _check_regex_matches(self, node, regex: str) -> bool: + def _check_regex_matches(self, node: BaseNode, regex: str) -> bool: if isinstance(node, BaseStringNode): return re.match(regex, node.value) is not None return False - def supported_element_nodes(self): + def supported_element_nodes(self) -> T.List[type]: return [StringNode] class MTypeIDList(MTypeList): def __init__(self, node: T.Optional[BaseNode] = None): super().__init__(node) - def _new_element_node(self, value): + def _new_element_node(self, value: T.Any) -> BaseNode: return IdNode(Token('', '', 0, 0, 0, None, str(value))) - def _check_is_equal(self, node, value) -> bool: + def _check_is_equal(self, node: BaseNode, value: T.Any) -> bool: if isinstance(node, IdNode): - return node.value == value + return bool(node.value == value) return False - def _check_regex_matches(self, node, regex: str) -> bool: + def _check_regex_matches(self, node: BaseNode, regex: str) -> bool: if isinstance(node, BaseStringNode): return re.match(regex, node.value) is not None return False - def supported_element_nodes(self): + def supported_element_nodes(self) -> T.List[type]: return [IdNode] -rewriter_keys = { +rewriter_keys: T.Dict[str, T.Dict[str, T.Any]] = { 'default_options': { 'operation': (str, None, ['set', 'delete']), 'options': (dict, {}, None) @@ -356,13 +380,15 @@ def supported_element_nodes(self): } class Rewriter: + info_dump: T.Optional[T.Dict[str, T.Dict[str, T.Any]]] + def __init__(self, sourcedir: str, generator: str = 'ninja', skip_errors: bool = False): self.sourcedir = sourcedir self.interpreter = IntrospectionInterpreter(sourcedir, '', generator, visitors = [AstIDGenerator(), AstIndentationGenerator(), AstConditionLevel()]) self.skip_errors = skip_errors - self.modified_nodes = [] - self.to_remove_nodes = [] - self.to_add_nodes = [] + self.modified_nodes: T.List[BaseNode] = [] + self.to_remove_nodes: T.List[BaseNode] = [] + self.to_add_nodes: T.List[BaseNode] = [] self.functions = { 'default_options': self.process_default_options, 'kwargs': self.process_kwargs, @@ -370,89 +396,100 @@ def __init__(self, sourcedir: str, generator: str = 'ninja', skip_errors: bool = } self.info_dump = None - def analyze_meson(self): + def analyze_meson(self) -> None: mlog.log('Analyzing meson file:', mlog.bold(os.path.join(self.sourcedir, environment.build_filename))) self.interpreter.analyze() mlog.log(' -- Project:', mlog.bold(self.interpreter.project_data['descriptive_name'])) mlog.log(' -- Version:', mlog.cyan(self.interpreter.project_data['version'])) - def add_info(self, cmd_type: str, cmd_id: str, data: dict): + def add_info(self, cmd_type: str, cmd_id: str, data: dict) -> None: if self.info_dump is None: self.info_dump = {} if cmd_type not in self.info_dump: self.info_dump[cmd_type] = {} self.info_dump[cmd_type][cmd_id] = data - def print_info(self): + def print_info(self) -> None: if self.info_dump is None: return - sys.stderr.write(json.dumps(self.info_dump, indent=2)) + sys.stderr.write(json.dumps(self.info_dump, indent=2, cls=IntrospectionEncoder)) - def on_error(self): + def on_error(self) -> T.Tuple[AnsiDecorator, AnsiDecorator]: if self.skip_errors: return mlog.cyan('-->'), mlog.yellow('skipping') return mlog.cyan('-->'), mlog.red('aborting') - def handle_error(self): + def handle_error(self) -> None: if self.skip_errors: return None raise MesonException('Rewriting the meson.build failed') - def find_target(self, target: str): - def check_list(name: str) -> T.List[BaseNode]: - result = [] - for i in self.interpreter.targets: - if name in {i['name'], i['id']}: - result += [i] - return result - - targets = check_list(target) - if targets: - if len(targets) == 1: - return targets[0] - else: - mlog.error('There are multiple targets matching', mlog.bold(target)) - for i in targets: - mlog.error(' -- Target name', mlog.bold(i['name']), 'with ID', mlog.bold(i['id'])) - mlog.error('Please try again with the unique ID of the target', *self.on_error()) - self.handle_error() - return None - - # Check the assignments - tgt = None - if target in self.interpreter.assignments: - node = self.interpreter.assignments[target] - if isinstance(node, FunctionNode): - if node.func_name.value in {'executable', 'jar', 'library', 'shared_library', 'shared_module', 'static_library', 'both_libraries'}: - tgt = self.interpreter.assign_vals[target] - - return tgt - - def find_dependency(self, dependency: str): - def check_list(name: str): - for i in self.interpreter.dependencies: - if name == i['name']: - return i + def all_assignments(self, varname: str) -> T.List[BaseNode]: + assigned_values = [] + for ass in self.interpreter.all_assignment_nodes[varname]: + if isinstance(ass, PlusAssignmentNode): + continue + assert isinstance(ass, AssignmentNode) + assigned_values.append(ass.value) + return assigned_values + + + def find_target(self, target: str) -> T.Optional[IntrospectionBuildTarget]: + for i in self.interpreter.targets: + if target == i.id: + return i + + potential_tgts = [] + for i in self.interpreter.targets: + if target == i.name: + potential_tgts.append(i) + + if len(potential_tgts) == 0: + potenial_tgts_1 = self.all_assignments(target) + potenial_tgts_1 = [self.interpreter.node_to_runtime_value(el) for el in potenial_tgts_1] + potential_tgts = [el for el in potenial_tgts_1 if isinstance(el, IntrospectionBuildTarget)] + + if len(potential_tgts) == 0: return None + elif len(potential_tgts) == 1: + return potential_tgts[0] + else: + mlog.error('There are multiple targets matching', mlog.bold(target)) + for i in potential_tgts: + mlog.error(' -- Target name', mlog.bold(i.name), 'with ID', mlog.bold(i.id)) + mlog.error('Please try again with the unique ID of the target', *self.on_error()) + self.handle_error() + return None + + def find_dependency(self, dependency: str) -> T.Optional[IntrospectionDependency]: + potential_deps = [] + for i in self.interpreter.dependencies: + if i.name == dependency: + potential_deps.append(i) - dep = check_list(dependency) - if dep is not None: - return dep + checking_varnames = len(potential_deps) == 0 - # Check the assignments - if dependency in self.interpreter.assignments: - node = self.interpreter.assignments[dependency] - if isinstance(node, FunctionNode): - if node.func_name.value == 'dependency': - name = self.interpreter.flatten_args(node.args)[0] - dep = check_list(name) + if checking_varnames: + potential_deps1 = self.all_assignments(dependency) + potential_deps = [self.interpreter.node_to_runtime_value(el) for el in potential_deps1 if isinstance(el, FunctionNode) and el.func_name.value == 'dependency'] - return dep + if len(potential_deps) == 0: + return None + elif len(potential_deps) == 1: + return potential_deps[0] + else: + mlog.error('There are multiple dependencies matching', mlog.bold(dependency)) + for i in potential_deps: + mlog.error(' -- Dependency name', i) + if checking_varnames: + mlog.error('Please try again with the name of the dependency', *self.on_error()) + self.handle_error() + return None @RequiredKeys(rewriter_keys['default_options']) - def process_default_options(self, cmd): + def process_default_options(self, cmd: T.Dict[str, T.Any]) -> None: # First, remove the old values - kwargs_cmd = { + kwargs_cmd: T.Dict[str, T.Any] = { 'function': 'project', 'id': "/", 'operation': 'remove_regex', @@ -496,7 +533,7 @@ def process_default_options(self, cmd): self.process_kwargs(kwargs_cmd) @RequiredKeys(rewriter_keys['kwargs']) - def process_kwargs(self, cmd): + def process_kwargs(self, cmd: T.Dict[str, T.Any]) -> None: mlog.log('Processing function type', mlog.bold(cmd['function']), 'with id', mlog.cyan("'" + cmd['id'] + "'")) if cmd['function'] not in rewriter_func_kwargs: mlog.error('Unknown function type', cmd['function'], *self.on_error()) @@ -517,26 +554,26 @@ def process_kwargs(self, cmd): node = self.interpreter.project_node arg_node = node.args elif cmd['function'] == 'target': - tmp = self.find_target(cmd['id']) - if tmp: - node = tmp['node'] + tmp_tgt = self.find_target(cmd['id']) + if tmp_tgt: + node = tmp_tgt.node arg_node = node.args elif cmd['function'] == 'dependency': - tmp = self.find_dependency(cmd['id']) - if tmp: - node = tmp['node'] + tmp_dep = self.find_dependency(cmd['id']) + if tmp_dep: + node = tmp_dep.node arg_node = node.args if not node: mlog.error('Unable to find the function node') assert isinstance(node, FunctionNode) assert isinstance(arg_node, ArgumentNode) # Transform the key nodes to plain strings - arg_node.kwargs = {k.value: v for k, v in arg_node.kwargs.items()} + kwargs = {T.cast(IdNode, k).value: v for k, v in arg_node.kwargs.items()} # Print kwargs info if cmd['operation'] == 'info': - info_data = {} - for key, val in sorted(arg_node.kwargs.items()): + info_data: T.Dict[str, T.Any] = {} + for key, val in sorted(kwargs.items()): info_data[key] = None if isinstance(val, ElementaryNode): info_data[key] = val.value @@ -562,17 +599,17 @@ def process_kwargs(self, cmd): # Remove the key from the kwargs if cmd['operation'] == 'delete': - if key in arg_node.kwargs: + if key in kwargs: mlog.log(' -- Deleting', mlog.bold(key), 'from the kwargs') - del arg_node.kwargs[key] + del kwargs[key] num_changed += 1 else: mlog.log(' -- Key', mlog.bold(key), 'is already deleted') continue - if key not in arg_node.kwargs: - arg_node.kwargs[key] = None - modifier = kwargs_def[key](arg_node.kwargs[key]) + if key not in kwargs: + kwargs[key] = None + modifier = kwargs_def[key](kwargs[key]) if not modifier.can_modify(): mlog.log(' -- Skipping', mlog.bold(key), 'because it is to complex to modify') @@ -592,44 +629,194 @@ def process_kwargs(self, cmd): modifier.remove_regex(val) # Write back the result - arg_node.kwargs[key] = modifier.get_node() + kwargs[key] = modifier.get_node() num_changed += 1 # Convert the keys back to IdNode's - arg_node.kwargs = {IdNode(Token('', '', 0, 0, 0, None, k)): v for k, v in arg_node.kwargs.items()} + arg_node.kwargs = {IdNode(Token('', '', 0, 0, 0, None, k)): v for k, v in kwargs.items()} for k, v in arg_node.kwargs.items(): k.level = v.level if num_changed > 0 and node not in self.modified_nodes: self.modified_nodes += [node] - def find_assignment_node(self, node: BaseNode) -> AssignmentNode: - if node.ast_id and node.ast_id in self.interpreter.reverse_assignment: - return self.interpreter.reverse_assignment[node.ast_id] + def find_assignment_node(self, node: BaseNode) -> T.Optional[AssignmentNode]: + for k,v in self.interpreter.all_assignment_nodes.items(): + for ass in v: + if ass.value == node: + return ass return None - @RequiredKeys(rewriter_keys['target']) - def process_target(self, cmd): - mlog.log('Processing target', mlog.bold(cmd['target']), 'operation', mlog.cyan(cmd['operation'])) - target = self.find_target(cmd['target']) - if target is None and cmd['operation'] != 'target_add': - mlog.error('Unknown target', mlog.bold(cmd['target']), *self.on_error()) - return self.handle_error() + def affects_no_other_targets(self, candidate: BaseNode) -> bool: + affected = self.interpreter.dataflow_dag.reachable(set([candidate]), False) + affected_targets = [x for x in affected if isinstance(x, FunctionNode) and x.func_name.value in BUILD_TARGET_FUNCTIONS] + return len(affected_targets) == 1 + + def get_relto(self, target_node: BaseNode, node: BaseNode) -> Path: + cwd = Path(os.getcwd()) + all_paths = self.interpreter.dataflow_dag.find_all_paths(node, target_node) + # len(all_paths) == 0 would imply that data does not flow from node to + # target_node. This would imply that adding sources to node would not + # add the source to the target. + assert len(all_paths) > 0 + if len(all_paths) > 1: + return None + return (cwd / next(x for x in all_paths[0] if isinstance(x, FunctionNode)).filename).parent - # Make source paths relative to the current subdir - def rel_source(src: str) -> str: - subdir = os.path.abspath(os.path.join(self.sourcedir, target['subdir'])) - if os.path.isabs(src): - return os.path.relpath(src, subdir) - elif not os.path.exists(src): - return src # Trust the user when the source doesn't exist - # Make sure that the path is relative to the subdir - return os.path.relpath(os.path.abspath(src), subdir) + def add_src_or_extra(self, op: str, target: IntrospectionBuildTarget, newfiles: T.List[str], to_sort_nodes: T.List[T.Union[FunctionNode, ArrayNode]]) -> None: + assert op in set(['src_add', 'extra_files_add']) + + if op == 'src_add': + old: T.Set[T.Union[BaseNode, UnknownValue]] = set(target.source_nodes) + elif op == 'extra_files_add': + if target.extra_files is None: + old = set() + else: + old = set([target.extra_files]) + tgt_function: FunctionNode = target.node + + cwd = Path(os.getcwd()) + target_dir_abs = cwd / os.path.dirname(target.node.filename) + source_root_abs = cwd / self.interpreter.source_root + + candidates1 = self.interpreter.dataflow_dag.reachable(old, True) + # A node is a member of the set `candidates1` exactly if data from this node + # flow into one of the `dest` nodes. We assume that this implies that if we + # add `foo.c` to this node, then 'foo.c' will be added to one of these + # nodes. This assumption is not always true: + # ar = ['a.c', 'b.c'] + # srcs = ar[1] + # executable('name', srcs) + # Data flows from `ar` to `srcs`, but if we add 'foo.c': + # ar = ['a.c', 'b.c', 'foo.c'] + # srcs = ar[1] + # executable('name', srcs) + # this does not add 'foo.c' to `srcs`. This is a known bug/limitation of + # the meson rewriter that could be fixed by replacing `reachable` with a + # more advanced analysis. But this is a lot of work and I think e.g. + # `srcs = ar[1]` is rare in real-world projects, so I will just leave + # this for now. + + candidates2 = {x for x in candidates1 if isinstance(x, (FunctionNode, ArrayNode))} + + # If we have this meson.build file: + # shared = ['shared.c'] + # executable('foo', shared + ['foo.c']) + # executable('bar', shared + ['bar.c']) + # and we are tasked with adding 'new.c' to 'foo', we should do e.g this: + # shared = ['shared.c'] + # executable('foo', shared + ['foo.c', 'new.c']) + # executable('bar', shared + ['bar.c']) + # but never this: + # shared = ['shared.c', 'new.c'] + # executable('foo', shared + ['foo.c']) + # executable('bar', shared + ['bar.c']) + # We do this by removing the `['shared.c']`-node from `candidates2`. + candidates2 = {x for x in candidates2 if self.affects_no_other_targets(x)} + + def path_contains_unknowns(candidate: BaseNode) -> bool: + all_paths = self.interpreter.dataflow_dag.find_all_paths(candidate, target.node) + for path in all_paths: + for el in path: + if isinstance(el, UnknownValue): + return True + return False - if target is not None: - cmd['sources'] = [rel_source(x) for x in cmd['sources']] + candidates2 = {x for x in candidates2 if not path_contains_unknowns(x)} + + candidates2 = {x for x in candidates2 if self.get_relto(target.node, x) is not None} + + flag_update_srcnodes = None + chosen: T.Union[FunctionNode, ArrayNode] = None + new_kwarg_flag = False + if len(candidates2) > 0: + # So that files(['a', 'b']) gets modified to files(['a', 'b', 'c']) instead of files(['a', 'b'], 'c') + if len({x for x in candidates2 if isinstance(x, ArrayNode)}) > 0: + candidates2 = {x for x in candidates2 if isinstance(x, ArrayNode)} + + # We choose one more or less arbitrary candidate + chosen = min(candidates2, key=lambda x: (x.lineno, x.colno)) + flag_update_srcnodes = False + elif op == 'src_add': + chosen = target.node + flag_update_srcnodes = True + elif op == 'extra_files_add': + chosen = ArrayNode(create_symbol('['), ArgumentNode(Token('', tgt_function.filename, 0, 0, 0, None, '[]')), create_symbol(']')) + + # this is fundamentally error prone + self.interpreter.dataflow_dag.add_edge(chosen, target.node) + + extra_files_idnode = IdNode(Token('string', tgt_function.filename, 0, 0, 0, None, 'extra_files')) + if tgt_function not in self.modified_nodes: + self.modified_nodes += [tgt_function] + new_extra_files_node: BaseNode + if target.node.args.get_kwarg_or_default('extra_files', None) is None: + # Target has no extra_files kwarg, create one + new_kwarg_flag = True + new_extra_files_node = chosen + else: + new_kwarg_flag = True + old_extra_files = target.node.args.get_kwarg_or_default('extra_files', None) + target.node.args.kwargs = {k: v for k, v in target.node.args.kwargs.items() if not (isinstance(k, IdNode) and k.value == 'extra_files')} + new_extra_files_node = ArithmeticNode('add', old_extra_files, create_symbol('+'), chosen) + + tgt_function.args.kwargs[extra_files_idnode] = new_extra_files_node + + if op != 'src_add': + flag_update_srcnodes = False + + newfiles_relto = self.get_relto(target.node, chosen) + old_src_list = flatten_nested_lists([self.interpreter.node_to_runtime_value(sn) for sn in old]) + + if op == 'src_add': + name = 'Source' + elif op == 'extra_files_add': + name = 'Extra file' + # Generate the new String nodes + to_append = [] + added = [] + + old_src_list = [(target_dir_abs / x).resolve() if isinstance(x, str) else x.to_abs_path(source_root_abs) for x in old_src_list if not isinstance(x, UnknownValue)] + for _newf in sorted(set(newfiles)): + newf = Path(_newf) + if os.path.isabs(newf): + newf = Path(newf) + else: + newf = source_root_abs / newf + if newf in old_src_list: + mlog.log(' -- ', name, mlog.green(str(newf)), 'is already defined for the target --> skipping') + continue + + mlog.log(' -- Adding ', name.lower(), mlog.green(str(newf)), 'at', + mlog.yellow(f'{chosen.filename}:{chosen.lineno}')) + added.append(newf) + mocktarget = self.interpreter.funcvals[target.node] + assert isinstance(mocktarget, IntrospectionBuildTarget) + print("adding ", str(newf), 'to', mocktarget.name) + + token = Token('string', chosen.filename, 0, 0, 0, None, str(os.path.relpath(newf, newfiles_relto))) + to_append += [StringNode(token)] + + assert isinstance(chosen, (FunctionNode, ArrayNode)) + arg_node = chosen.args + # Append to the AST at the right place + arg_node.arguments += to_append + + # Mark the node as modified + if chosen not in to_sort_nodes: + to_sort_nodes += [chosen] + # If the extra_files array is newly created, i.e. if new_kwarg_flag is + # True, don't mark it as its parent function node already is, otherwise + # this would cause double modification. + if chosen not in self.modified_nodes and not new_kwarg_flag: + self.modified_nodes += [chosen] + + def rm_src_or_extra(self, op: str, target: IntrospectionBuildTarget, to_be_removed: T.List[str], to_sort_nodes: T.List[T.Union[FunctionNode, ArrayNode]]) -> None: + assert op in set(['src_rm', 'extra_files_rm']) + cwd = Path(os.getcwd()) + source_root_abs = cwd / self.interpreter.source_root # Utility function to get a list of the sources from a node - def arg_list_from_node(n): + def arg_list_from_node(n: BaseNode) -> T.List[BaseNode]: args = [] if isinstance(n, FunctionNode): args = list(n.args.arguments) @@ -641,167 +828,79 @@ def arg_list_from_node(n): args = n.arguments return args - to_sort_nodes = [] - - if cmd['operation'] == 'src_add': - node = None - if target['sources']: - node = target['sources'][0] - else: - node = target['node'] - assert node is not None - - # Generate the current source list - src_list = [] - for i in target['sources']: - for j in arg_list_from_node(i): - if isinstance(j, BaseStringNode): - src_list += [j.value] - - # Generate the new String nodes - to_append = [] - for i in sorted(set(cmd['sources'])): - if i in src_list: - mlog.log(' -- Source', mlog.green(i), 'is already defined for the target --> skipping') + # Helper to find the exact string node and its parent + def find_node(src: str) -> T.Tuple[T.Optional[BaseNode], T.Optional[BaseStringNode]]: + if op == 'src_rm': + nodes = self.interpreter.dataflow_dag.reachable(set(target.source_nodes), True).union(set([target.node])) + elif op == 'extra_files_rm': + nodes = self.interpreter.dataflow_dag.reachable(set([target.extra_files]), True) + for i in nodes: + if isinstance(i, UnknownValue): continue - mlog.log(' -- Adding source', mlog.green(i), 'at', - mlog.yellow(f'{node.filename}:{node.lineno}')) - token = Token('string', node.filename, 0, 0, 0, None, i) - to_append += [StringNode(token)] - - # Append to the AST at the right place - arg_node = None - if isinstance(node, (FunctionNode, ArrayNode)): - arg_node = node.args - elif isinstance(node, ArgumentNode): - arg_node = node - assert arg_node is not None - arg_node.arguments += to_append - - # Mark the node as modified - if arg_node not in to_sort_nodes and not isinstance(node, FunctionNode): - to_sort_nodes += [arg_node] - if node not in self.modified_nodes: - self.modified_nodes += [node] - - elif cmd['operation'] == 'src_rm': - # Helper to find the exact string node and its parent - def find_node(src): - for i in target['sources']: + relto = self.get_relto(target.node, i) + if relto is not None: for j in arg_list_from_node(i): if isinstance(j, BaseStringNode): - if j.value == src: + if os.path.normpath(relto / j.value) == os.path.normpath(source_root_abs / src): return i, j - return None, None + return None, None + + if op == 'src_rm': + name = 'source' + elif op == 'extra_files_rm': + name = 'extra file' + + for i in to_be_removed: + # Try to find the node with the source string + root, string_node = find_node(i) + if root is None: + mlog.warning(' -- Unable to find', name, mlog.green(i), 'in the target') + continue + if not self.affects_no_other_targets(string_node): + mlog.warning(' -- Removing the', name, mlog.green(i), 'is too compilicated') + continue - for i in cmd['sources']: - # Try to find the node with the source string - root, string_node = find_node(i) - if root is None: - mlog.warning(' -- Unable to find source', mlog.green(i), 'in the target') - continue + if not isinstance(root, (FunctionNode, ArrayNode)): + raise NotImplementedError # I'm lazy - # Remove the found string node from the argument list - arg_node = None - if isinstance(root, (FunctionNode, ArrayNode)): - arg_node = root.args - elif isinstance(root, ArgumentNode): - arg_node = root - assert arg_node is not None - mlog.log(' -- Removing source', mlog.green(i), 'from', - mlog.yellow(f'{string_node.filename}:{string_node.lineno}')) - arg_node.arguments.remove(string_node) - - # Mark the node as modified - if arg_node not in to_sort_nodes and not isinstance(root, FunctionNode): - to_sort_nodes += [arg_node] - if root not in self.modified_nodes: - self.modified_nodes += [root] - - elif cmd['operation'] == 'extra_files_add': - tgt_function: FunctionNode = target['node'] - mark_array = True - try: - node = target['extra_files'][0] - except IndexError: - # Specifying `extra_files` with a list that flattens to empty gives an empty - # target['extra_files'] list, account for that. - try: - extra_files_key = next(k for k in tgt_function.args.kwargs.keys() if isinstance(k, IdNode) and k.value == 'extra_files') - node = tgt_function.args.kwargs[extra_files_key] - except StopIteration: - # Target has no extra_files kwarg, create one - node = ArrayNode(_symbol('['), ArgumentNode(Token('', tgt_function.filename, 0, 0, 0, None, '[]')), _symbol(']')) - tgt_function.args.kwargs[IdNode(Token('string', tgt_function.filename, 0, 0, 0, None, 'extra_files'))] = node - mark_array = False - if tgt_function not in self.modified_nodes: - self.modified_nodes += [tgt_function] - target['extra_files'] = [node] - if isinstance(node, IdNode): - node = self.interpreter.assignments[node.value] - target['extra_files'] = [node] - if not isinstance(node, ArrayNode): - mlog.error('Target', mlog.bold(cmd['target']), 'extra_files argument must be a list', *self.on_error()) - return self.handle_error() + # Remove the found string node from the argument list + arg_node = root.args + mlog.log(' -- Removing', name, mlog.green(i), 'from', + mlog.yellow(f'{string_node.filename}:{string_node.lineno}')) + arg_node.arguments.remove(string_node) - # Generate the current extra files list - extra_files_list = [] - for i in target['extra_files']: - for j in arg_list_from_node(i): - if isinstance(j, BaseStringNode): - extra_files_list += [j.value] - - # Generate the new String nodes - to_append = [] - for i in sorted(set(cmd['sources'])): - if i in extra_files_list: - mlog.log(' -- Extra file', mlog.green(i), 'is already defined for the target --> skipping') - continue - mlog.log(' -- Adding extra file', mlog.green(i), 'at', - mlog.yellow(f'{node.filename}:{node.lineno}')) - token = Token('string', node.filename, 0, 0, 0, None, i) - to_append += [StringNode(token)] + # Mark the node as modified + if root not in to_sort_nodes: + to_sort_nodes += [root] + if root not in self.modified_nodes: + self.modified_nodes += [root] - # Append to the AST at the right place - arg_node = node.args - arg_node.arguments += to_append - # Mark the node as modified - if arg_node not in to_sort_nodes: - to_sort_nodes += [arg_node] - # If the extra_files array is newly created, don't mark it as its parent function node already is, - # otherwise this would cause double modification. - if mark_array and node not in self.modified_nodes: - self.modified_nodes += [node] - - elif cmd['operation'] == 'extra_files_rm': - # Helper to find the exact string node and its parent - def find_node(src): - for i in target['extra_files']: - for j in arg_list_from_node(i): - if isinstance(j, BaseStringNode): - if j.value == src: - return i, j - return None, None + @RequiredKeys(rewriter_keys['target']) + def process_target(self, cmd: T.Dict[str, T.Any]) -> None: + mlog.log('Processing target', mlog.bold(cmd['target']), 'operation', mlog.cyan(cmd['operation'])) + target = self.find_target(cmd['target']) + if target is None and cmd['operation'] != 'target_add': + mlog.error('Unknown target', mlog.bold(cmd['target']), *self.on_error()) + return self.handle_error() - for i in cmd['sources']: - # Try to find the node with the source string - root, string_node = find_node(i) - if root is None: - mlog.warning(' -- Unable to find extra file', mlog.green(i), 'in the target') - continue + # Make source paths relative to the current subdir + def rel_source(src: str) -> str: + subdir = os.path.abspath(os.path.join(self.sourcedir, target.subdir)) + if os.path.isabs(src): + return os.path.relpath(src, subdir) + elif not os.path.exists(src): + return src # Trust the user when the source doesn't exist + # Make sure that the path is relative to the subdir + return os.path.relpath(os.path.abspath(src), subdir) - # Remove the found string node from the argument list - arg_node = root.args - mlog.log(' -- Removing extra file', mlog.green(i), 'from', - mlog.yellow(f'{string_node.filename}:{string_node.lineno}')) - arg_node.arguments.remove(string_node) + to_sort_nodes: T.List[T.Union[FunctionNode, ArrayNode]] = [] - # Mark the node as modified - if arg_node not in to_sort_nodes and not isinstance(root, FunctionNode): - to_sort_nodes += [arg_node] - if root not in self.modified_nodes: - self.modified_nodes += [root] + if cmd['operation'] in {'src_add', 'extra_files_add'}: + self.add_src_or_extra(cmd['operation'], target, cmd['sources'], to_sort_nodes) + + elif cmd['operation'] in {'src_rm', 'extra_files_rm'}: + self.rm_src_or_extra(cmd['operation'], target, cmd['sources'], to_sort_nodes) elif cmd['operation'] == 'target_add': if target is not None: @@ -811,21 +910,21 @@ def find_node(src): id_base = re.sub(r'[- ]', '_', cmd['target']) target_id = id_base + '_exe' if cmd['target_type'] == 'executable' else '_lib' source_id = id_base + '_sources' - filename = os.path.join(cmd['subdir'], environment.build_filename) + filename = os.path.join(os.getcwd(), self.interpreter.source_root, cmd['subdir'], environment.build_filename) # Build src list src_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - src_arr_node = ArrayNode(_symbol('['), src_arg_node, _symbol(']')) + src_arr_node = ArrayNode(create_symbol('['), src_arg_node, create_symbol(']')) src_far_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - src_fun_node = FunctionNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), 'files')), _symbol('('), src_far_node, _symbol(')')) - src_ass_node = AssignmentNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), source_id)), _symbol('='), src_fun_node) + src_fun_node = FunctionNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), 'files')), create_symbol('('), src_far_node, create_symbol(')')) + src_ass_node = AssignmentNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), source_id)), create_symbol('='), src_fun_node) src_arg_node.arguments = [StringNode(Token('string', filename, 0, 0, 0, None, x)) for x in cmd['sources']] src_far_node.arguments = [src_arr_node] # Build target tgt_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - tgt_fun_node = FunctionNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), cmd['target_type'])), _symbol('('), tgt_arg_node, _symbol(')')) - tgt_ass_node = AssignmentNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), target_id)), _symbol('='), tgt_fun_node) + tgt_fun_node = FunctionNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), cmd['target_type'])), create_symbol('('), tgt_arg_node, create_symbol(')')) + tgt_ass_node = AssignmentNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), target_id)), create_symbol('='), tgt_fun_node) tgt_arg_node.arguments = [ StringNode(Token('string', filename, 0, 0, 0, None, cmd['target'])), IdNode(Token('string', filename, 0, 0, 0, None, source_id)) @@ -836,44 +935,53 @@ def find_node(src): self.to_add_nodes += [src_ass_node, tgt_ass_node] elif cmd['operation'] == 'target_rm': - to_remove = self.find_assignment_node(target['node']) + to_remove: T.Optional[T.Union[AssignmentNode, FunctionNode]] = self.find_assignment_node(target.node) if to_remove is None: - to_remove = target['node'] + to_remove = target.node self.to_remove_nodes += [to_remove] mlog.log(' -- Removing target', mlog.green(cmd['target']), 'at', mlog.yellow(f'{to_remove.filename}:{to_remove.lineno}')) elif cmd['operation'] == 'info': # T.List all sources in the target - src_list = [] - for i in target['sources']: - for j in arg_list_from_node(i): - if isinstance(j, BaseStringNode): - src_list += [j.value] - extra_files_list = [] - for i in target['extra_files']: - for j in arg_list_from_node(i): - if isinstance(j, BaseStringNode): - extra_files_list += [j.value] + + cwd = Path(os.getcwd()) + source_root_abs = cwd / self.interpreter.source_root + + src_list = self.interpreter.nodes_to_pretty_filelist(source_root_abs, target.subdir, target.source_nodes) + extra_files_list = self.interpreter.nodes_to_pretty_filelist(source_root_abs, target.subdir, [target.extra_files] if target.extra_files else []) + + src_list = ['unknown' if isinstance(x, UnknownValue) else os.path.relpath(x, source_root_abs) for x in src_list] + extra_files_list = ['unknown' if isinstance(x, UnknownValue) else os.path.relpath(x, source_root_abs) for x in extra_files_list] + test_data = { - 'name': target['name'], + 'name': target.name, 'sources': src_list, 'extra_files': extra_files_list } - self.add_info('target', target['id'], test_data) + self.add_info('target', target.id, test_data) # Sort files for i in to_sort_nodes: - convert = lambda text: int(text) if text.isdigit() else text.lower() - alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)] - path_sorter = lambda key: ([(key.count('/') <= idx, alphanum_key(x)) for idx, x in enumerate(key.split('/'))]) - - unknown = [x for x in i.arguments if not isinstance(x, BaseStringNode)] - sources = [x for x in i.arguments if isinstance(x, BaseStringNode)] + def convert(text: str) -> T.Union[int, str]: + return int(text) if text.isdigit() else text.lower() + def alphanum_key(key: str) -> T.List[T.Union[int, str]]: + return [convert(c) for c in re.split('([0-9]+)', key)] + def path_sorter(key: str) -> T.List[T.Tuple[bool, T.List[T.Union[int, str]]]]: + return [(key.count('/') <= idx, alphanum_key(x)) for idx, x in enumerate(key.split('/'))] + + if isinstance(i, FunctionNode) and i.func_name.value in BUILD_TARGET_FUNCTIONS: + src_args = i.args.arguments[1:] + target_name = [i.args.arguments[0]] + else: + src_args = i.args.arguments + target_name = [] + unknown: T.List[BaseNode] = [x for x in src_args if not isinstance(x, BaseStringNode)] + sources: T.List[BaseStringNode] = [x for x in src_args if isinstance(x, BaseStringNode)] sources = sorted(sources, key=lambda x: path_sorter(x.value)) - i.arguments = unknown + sources + i.args.arguments = target_name + unknown + T.cast(T.List[BaseNode], sources) - def process(self, cmd): + def process(self, cmd: T.Dict[str, T.Any]) -> None: if 'type' not in cmd: raise RewriterException('Command has no key "type"') if cmd['type'] not in self.functions: @@ -881,7 +989,7 @@ def process(self, cmd): .format(cmd['type'], list(self.functions.keys()))) self.functions[cmd['type']](cmd) - def apply_changes(self): + def apply_changes(self) -> None: assert all(hasattr(x, 'lineno') and hasattr(x, 'colno') and hasattr(x, 'filename') for x in self.modified_nodes) assert all(hasattr(x, 'lineno') and hasattr(x, 'colno') and hasattr(x, 'filename') for x in self.to_remove_nodes) assert all(isinstance(x, (ArrayNode, FunctionNode)) for x in self.modified_nodes) @@ -889,7 +997,7 @@ def apply_changes(self): # Sort based on line and column in reversed order work_nodes = [{'node': x, 'action': 'modify'} for x in self.modified_nodes] work_nodes += [{'node': x, 'action': 'rm'} for x in self.to_remove_nodes] - work_nodes = sorted(work_nodes, key=lambda x: (x['node'].lineno, x['node'].colno), reverse=True) + work_nodes = sorted(work_nodes, key=lambda x: (T.cast(BaseNode, x['node']).lineno, T.cast(BaseNode, x['node']).colno), reverse=True) work_nodes += [{'node': x, 'action': 'add'} for x in self.to_add_nodes] # Generating the new replacement string @@ -898,11 +1006,11 @@ def apply_changes(self): new_data = '' if i['action'] == 'modify' or i['action'] == 'add': printer = AstPrinter() - i['node'].accept(printer) + T.cast(BaseNode, i['node']).accept(printer) printer.post_process() new_data = printer.result.strip() data = { - 'file': i['node'].filename, + 'file': T.cast(BaseNode, i['node']).filename, 'str': new_data, 'node': i['node'], 'action': i['action'] @@ -910,11 +1018,11 @@ def apply_changes(self): str_list += [data] # Load build files - files = {} + files: T.Dict[str, T.Any] = {} for i in str_list: if i['file'] in files: continue - fpath = os.path.realpath(os.path.join(self.sourcedir, i['file'])) + fpath = os.path.realpath(T.cast(str, i['file'])) fdata = '' # Create an empty file if it does not exist if not os.path.exists(fpath): @@ -931,14 +1039,14 @@ def apply_changes(self): line_offsets += [offset] offset += len(j) - files[i['file']] = { + files[T.cast(str, i['file'])] = { 'path': fpath, 'raw': fdata, 'offsets': line_offsets } # Replace in source code - def remove_node(i): + def remove_node(i: T.Dict[str, T.Any]) -> None: offsets = files[i['file']]['offsets'] raw = files[i['file']]['raw'] node = i['node'] @@ -966,7 +1074,7 @@ def remove_node(i): if i['action'] in {'modify', 'rm'}: remove_node(i) elif i['action'] == 'add': - files[i['file']]['raw'] += i['str'] + '\n' + files[T.cast(str, i['file'])]['raw'] += T.cast(str, i['str']) + '\n' # Write the files back for key, val in files.items(): @@ -997,7 +1105,7 @@ def list_to_dict(in_list: T.List[str]) -> T.Dict[str, str]: raise TypeError('in_list parameter of list_to_dict must have an even length.') return result -def generate_target(options) -> T.List[dict]: +def generate_target(options: argparse.Namespace) -> T.List[T.Dict[str, T.Any]]: return [{ 'type': 'target', 'target': options.target, @@ -1007,7 +1115,7 @@ def generate_target(options) -> T.List[dict]: 'target_type': options.tgt_type, }] -def generate_kwargs(options) -> T.List[dict]: +def generate_kwargs(options: argparse.Namespace) -> T.List[T.Dict[str, T.Any]]: return [{ 'type': 'kwargs', 'function': options.function, @@ -1016,19 +1124,19 @@ def generate_kwargs(options) -> T.List[dict]: 'kwargs': list_to_dict(options.kwargs), }] -def generate_def_opts(options) -> T.List[dict]: +def generate_def_opts(options: argparse.Namespace) -> T.List[T.Dict[str, T.Any]]: return [{ 'type': 'default_options', 'operation': options.operation, 'options': list_to_dict(options.options), }] -def generate_cmd(options) -> T.List[dict]: +def generate_cmd(options: argparse.Namespace) -> T.List[T.Dict[str, T.Any]]: if os.path.exists(options.json): with open(options.json, encoding='utf-8') as fp: - return json.load(fp) + return T.cast(T.List[T.Dict[str, T.Any]], json.load(fp)) else: - return json.loads(options.json) + return T.cast(T.List[T.Dict[str, T.Any]], json.loads(options.json)) # Map options.type to the actual type name cli_type_map = { @@ -1041,7 +1149,7 @@ def generate_cmd(options) -> T.List[dict]: 'cmd': generate_cmd, } -def run(options): +def run(options: argparse.Namespace) -> int: if not options.verbose: mlog.set_quiet() @@ -1059,10 +1167,21 @@ def run(options): if not isinstance(commands, list): raise TypeError('Command is not a list') - for i in commands: - if not isinstance(i, object): + for i, cmd in enumerate(commands): + if not isinstance(cmd, object): raise TypeError('Command is not an object') - rewriter.process(i) + rewriter.process(cmd) + + if i == len(commands) - 1: # Improves the performance, is not necessary for correctness. + continue + + rewriter.apply_changes() + rewriter.modified_nodes = [] + rewriter.to_remove_nodes = [] + rewriter.to_add_nodes = [] + # The AST changed, so we need to update every information that was derived from the AST + rewriter.interpreter = IntrospectionInterpreter(rewriter.sourcedir, '', rewriter.interpreter.backend, visitors = [AstIDGenerator(), AstIndentationGenerator(), AstConditionLevel()]) + rewriter.analyze_meson() rewriter.apply_changes() rewriter.print_info() diff --git a/run_mypy.py b/run_mypy.py index cf0e0e3de253..0d75635859fd 100755 --- a/run_mypy.py +++ b/run_mypy.py @@ -11,7 +11,6 @@ modules = [ # fully typed submodules - # 'mesonbuild/ast/', 'mesonbuild/cargo/', 'mesonbuild/cmake/', 'mesonbuild/compilers/', @@ -25,6 +24,11 @@ # specific files 'mesonbuild/arglist.py', + 'mesonbuild/ast/__init__.py', + 'mesonbuild/ast/interpreter.py', + 'mesonbuild/ast/introspection.py', + 'mesonbuild/ast/postprocess.py', + 'mesonbuild/ast/visitor.py', 'mesonbuild/backend/backends.py', # 'mesonbuild/coredata.py', 'mesonbuild/depfile.py', @@ -68,6 +72,7 @@ 'mesonbuild/mtest.py', 'mesonbuild/optinterpreter.py', 'mesonbuild/programs.py', + 'mesonbuild/rewriter.py', 'run_mypy.py', 'run_project_tests.py', diff --git a/test cases/rewrite/1 basic/addSrc.json b/test cases/rewrite/1 basic/addSrc.json index b8bc43916c27..52603f60b2d0 100644 --- a/test cases/rewrite/1 basic/addSrc.json +++ b/test cases/rewrite/1 basic/addSrc.json @@ -41,6 +41,24 @@ "operation": "src_add", "sources": ["a6.cpp", "a1.cpp"] }, + { + "type": "target", + "target": "trivialprog10", + "operation": "src_add", + "sources": ["fileA.cpp", "fileB.cpp", "a1.cpp"] + }, + { + "type": "target", + "target": "trivialprog11", + "operation": "src_add", + "sources": ["fileA.cpp", "a1.cpp"] + }, + { + "type": "target", + "target": "trivialprog12", + "operation": "src_add", + "sources": ["fileA.cpp", "fileB.cpp", "a1.cpp"] + }, { "type": "target", "target": "trivialprog0", @@ -90,5 +108,25 @@ "type": "target", "target": "trivialprog9", "operation": "info" + }, + { + "type": "target", + "target": "trivialprog10", + "operation": "info" + }, + { + "type": "target", + "target": "trivialprog11", + "operation": "info" + }, + { + "type": "target", + "target": "trivialprog12", + "operation": "info" + }, + { + "type": "target", + "target": "rightName", + "operation": "info" } ] diff --git a/test cases/rewrite/1 basic/addTgt.json b/test cases/rewrite/1 basic/addTgt.json index 2f4e7e256c52..02d600a8251a 100644 --- a/test cases/rewrite/1 basic/addTgt.json +++ b/test cases/rewrite/1 basic/addTgt.json @@ -1,7 +1,7 @@ [ { "type": "target", - "target": "trivialprog10", + "target": "trivialprog13", "operation": "target_add", "sources": ["new1.cpp", "new2.cpp"], "target_type": "shared_library" diff --git a/test cases/rewrite/1 basic/info.json b/test cases/rewrite/1 basic/info.json index 0f1a3bd8cae5..9977f5a879e8 100644 --- a/test cases/rewrite/1 basic/info.json +++ b/test cases/rewrite/1 basic/info.json @@ -53,5 +53,25 @@ "type": "target", "target": "trivialprog10", "operation": "info" + }, + { + "type": "target", + "target": "trivialprog11", + "operation": "info" + }, + { + "type": "target", + "target": "trivialprog12", + "operation": "info" + }, + { + "type": "target", + "target": "trivialprog13", + "operation": "info" + }, + { + "type": "target", + "target": "rightName", + "operation": "info" } ] diff --git a/test cases/rewrite/1 basic/meson.build b/test cases/rewrite/1 basic/meson.build index 0f87c452019b..5fe952769235 100644 --- a/test cases/rewrite/1 basic/meson.build +++ b/test cases/rewrite/1 basic/meson.build @@ -4,6 +4,16 @@ src1 = ['main.cpp', 'fileA.cpp'] src2 = files(['fileB.cpp', 'fileC.cpp']) src3 = src1 src4 = [src3] +src5 = ['main.cpp', 'fileA.cpp'] +src5 += ['fileB.cpp'] +src6 = ['main.cpp', 'fileA.cpp'] +src6 = files(src6) +src7 = ['main.cpp', 'fileA.cpp'] +src8 = src7 +src7 = ['fileB.cpp', 'fileC.cpp'] +name = 'rightName' +trickyName = get_variable('name') +name = 'wrongName' # Magic comment @@ -14,6 +24,10 @@ exe3 = executable('trivialprog3', ['main.cpp', 'fileA.cpp']) exe4 = executable('trivialprog4', ['main.cpp', ['fileA.cpp']]) exe5 = executable('trivialprog5', [src2, 'main.cpp']) exe6 = executable('trivialprog6', 'main.cpp', 'fileA.cpp') -exe7 = executable('trivialprog7', 'fileB.cpp', src1, 'fileC.cpp') +exe7 = executable('trivialprog7', 'fileB.cpp', get_variable('src1'), 'fileC.cpp') exe8 = executable('trivialprog8', src3) executable('trivialprog9', src4) +executable('trivialprog10', src5) +executable('trivialprog11', src6) +executable('trivialprog12', src8) +executable(trickyName, 'main.cpp') diff --git a/test cases/rewrite/1 basic/rmSrc.json b/test cases/rewrite/1 basic/rmSrc.json index 2e7447c681d0..de56bbe1bf5f 100644 --- a/test cases/rewrite/1 basic/rmSrc.json +++ b/test cases/rewrite/1 basic/rmSrc.json @@ -1,10 +1,4 @@ [ - { - "type": "target", - "target": "trivialprog1", - "operation": "src_rm", - "sources": ["fileA.cpp"] - }, { "type": "target", "target": "trivialprog3", @@ -21,7 +15,7 @@ "type": "target", "target": "trivialprog5", "operation": "src_rm", - "sources": ["fileB.cpp"] + "sources": ["main.cpp"] }, { "type": "target", @@ -35,6 +29,18 @@ "operation": "src_rm", "sources": ["fileB.cpp"] }, + { + "type": "target", + "target": "trivialprog10", + "operation": "src_rm", + "sources": ["fileA.cpp", "fileB.cpp"] + }, + { + "type": "target", + "target": "trivialprog11", + "operation": "src_rm", + "sources": ["fileA.cpp"] + }, { "type": "target", "target": "trivialprog0", @@ -84,5 +90,25 @@ "type": "target", "target": "trivialprog9", "operation": "info" + }, + { + "type": "target", + "target": "trivialprog10", + "operation": "info" + }, + { + "type": "target", + "target": "trivialprog11", + "operation": "info" + }, + { + "type": "target", + "target": "trivialprog12", + "operation": "info" + }, + { + "type": "target", + "target": "rightName", + "operation": "info" } ] diff --git a/test cases/rewrite/1 basic/rmTgt.json b/test cases/rewrite/1 basic/rmTgt.json index dbaf02535795..bc6dc302e9d0 100644 --- a/test cases/rewrite/1 basic/rmTgt.json +++ b/test cases/rewrite/1 basic/rmTgt.json @@ -13,5 +13,10 @@ "type": "target", "target": "trivialprog9", "operation": "target_rm" + }, + { + "type": "target", + "target": "rightName", + "operation": "target_rm" } ] diff --git a/test cases/rewrite/7 tricky dataflow/addSrc.json b/test cases/rewrite/7 tricky dataflow/addSrc.json new file mode 100644 index 000000000000..1b6aa485b6ac --- /dev/null +++ b/test cases/rewrite/7 tricky dataflow/addSrc.json @@ -0,0 +1,72 @@ +[ + { + "type": "target", + "target": "tgt1", + "operation": "src_add", + "sources": [ + "new.c" + ] + }, + { + "type": "target", + "target": "tgt2", + "operation": "src_add", + "sources": [ + "new.c" + ] + }, + { + "type": "target", + "target": "tgt3", + "operation": "src_add", + "sources": [ + "new.c" + ] + }, + { + "type": "target", + "target": "tgt5", + "operation": "src_add", + "sources": [ + "new.c" + ] + }, + { + "type": "target", + "target": "tgt6", + "operation": "src_add", + "sources": [ + "new.c" + ] + }, + { + "type": "target", + "target": "tgt1", + "operation": "info" + }, + { + "type": "target", + "target": "tgt2", + "operation": "info" + }, + { + "type": "target", + "target": "tgt3", + "operation": "info" + }, + { + "type": "target", + "target": "tgt4", + "operation": "info" + }, + { + "type": "target", + "target": "tgt5", + "operation": "info" + }, + { + "type": "target", + "target": "tgt6", + "operation": "info" + } +] diff --git a/test cases/rewrite/7 tricky dataflow/info.json b/test cases/rewrite/7 tricky dataflow/info.json new file mode 100644 index 000000000000..5fc65dd8cd3d --- /dev/null +++ b/test cases/rewrite/7 tricky dataflow/info.json @@ -0,0 +1,32 @@ +[ + { + "type": "target", + "target": "tgt1", + "operation": "info" + }, + { + "type": "target", + "target": "tgt2", + "operation": "info" + }, + { + "type": "target", + "target": "tgt3", + "operation": "info" + }, + { + "type": "target", + "target": "tgt4", + "operation": "info" + }, + { + "type": "target", + "target": "tgt5", + "operation": "info" + }, + { + "type": "target", + "target": "tgt6", + "operation": "info" + } +] diff --git a/test cases/rewrite/7 tricky dataflow/meson.build b/test cases/rewrite/7 tricky dataflow/meson.build new file mode 100644 index 000000000000..4c183ed97ecb --- /dev/null +++ b/test cases/rewrite/7 tricky dataflow/meson.build @@ -0,0 +1,35 @@ +project('rewrite tricky dataflow', 'c') + +# Adding a file to `begin` will add this file to the sources of `tgt1`, but +# not to any other target. But a buggy rewriter might think that adding a file +# to `begin` will also add this file to `end` and will refuse to add a file to +# `begin`. +begin = ['foo.c'] +tgt1 = library('tgt1', begin) +distraction = executable('distraction', link_with: tgt1) + + +tgt2_srcs = ['foo.c'] +if meson.host_machine().system() == 'windows' # Some condition that cannot be known statically + tgt2_srcs += ['bar.c'] +endif +executable('tgt2', tgt2_srcs) + + +tgt34_srcs = ['foo.c'] +executable('tgt3', tgt34_srcs) +if meson.host_machine().system() == 'windows' + tgt34_srcs += ['bar.c'] +endif +executable('tgt4', tgt34_srcs) + + +dont_add_here_5 = ['foo.c'] +ct = custom_target('ct', output: 'out.c', input: dont_add_here_5, command: ['placeholder', '@INPUT@', '@OUTPUT@']) +executable('tgt5', ct) + + +dont_add_here_6 = ['foo.c'] +gen = generator(find_program('cp'), output: '@BASENAME@_copy.c', arguments: ['@INPUT@', '@OUTPUT@']) +generated = gen.process(dont_add_here_6) +executable('tgt6', generated) diff --git a/test cases/unit/118 rewrite/meson.build b/test cases/unit/118 rewrite/meson.build index 7d0330b9e2e3..545bb0fdeb37 100644 --- a/test cases/unit/118 rewrite/meson.build +++ b/test cases/unit/118 rewrite/meson.build @@ -62,6 +62,7 @@ cppcoro = declare_dependency( ) +cpp_compiler = meson.get_compiler('cpp') if get_option('unicode') #if comment #if comment 2 mfc=cpp_compiler.find_library(get_option('debug')?'mfc140ud':'mfc140u') @@ -80,6 +81,10 @@ assert(not (3 in [1, 2]), '''3 shouldn't be in [1, 2]''') assert('b' in ['a', 'b'], ''''b' should be in ['a', 'b']''') assert('c' not in ['a', 'b'], ''''c' shouldn't be in ['a', 'b']''') +exe1 = 'exe1' +exe2 = 'exe2' +exe3 = 'exe3' + assert(exe1 in [exe1, exe2], ''''exe1 should be in [exe1, exe2]''') assert(exe3 not in [exe1, exe2], ''''exe3 shouldn't be in [exe1, exe2]''') diff --git a/unittests/allplatformstests.py b/unittests/allplatformstests.py index acab026e954c..a87859ac5a1a 100644 --- a/unittests/allplatformstests.py +++ b/unittests/allplatformstests.py @@ -3386,6 +3386,8 @@ def test_introspect_targets_from_source(self): # Account for differences in output res_wb = [i for i in res_wb if i['type'] != 'custom'] for i in res_wb: + if i['id'] == 'test1@exe': + i['build_by_default'] = 'unknown' i['filename'] = [os.path.relpath(x, self.builddir) for x in i['filename']] for k in ('install_filename', 'dependencies', 'win_subsystem'): if k in i: @@ -3504,7 +3506,7 @@ def test_introspect_dependencies_from_source(self): }, { 'name': 'bugDep1', - 'required': True, + 'required': 'unknown', 'version': [], 'has_fallback': False, 'conditional': False diff --git a/unittests/rewritetests.py b/unittests/rewritetests.py index c33884442675..ddf62b9fb55f 100644 --- a/unittests/rewritetests.py +++ b/unittests/rewritetests.py @@ -57,6 +57,18 @@ def rewrite(self, directory, args): args = [args] return self.rewrite_raw(directory, ['command'] + args) + # The rewriter sorts the sources alphabetically, but this is very unstable + # and buggy, so we do not test it. + def assertEqualIgnoreOrder(self, a, b): + def deepsort(x): + if isinstance(x, list): + return sorted([deepsort(el) for el in x]) + elif isinstance(x, dict): + return {k: deepsort(v) for k,v in x.items()} + else: + return x + self.assertDictEqual(deepsort(a), deepsort(b)) + def test_target_source_list(self): self.prime('1 basic') out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) @@ -72,32 +84,40 @@ def test_target_source_list(self): 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileB.cpp', 'fileC.cpp', 'main.cpp', 'fileA.cpp'], 'extra_files': []}, 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog10@exe': {'name': 'trivialprog10', 'sources': ['main.cpp', 'fileA.cpp', 'fileB.cpp'], 'extra_files': []}, + 'trivialprog11@exe': {'name': 'trivialprog11', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog12@exe': {'name': 'trivialprog12', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'rightName@exe': {'name': 'rightName', 'sources': ['main.cpp'], 'extra_files': []}, } } - self.assertDictEqual(out, expected) + self.assertEqualIgnoreOrder(out, expected) def test_target_add_sources(self): self.prime('1 basic') out = self.rewrite(self.builddir, os.path.join(self.builddir, 'addSrc.json')) expected = { 'target': { - 'trivialprog0@exe': {'name': 'trivialprog0', 'sources': ['a1.cpp', 'a2.cpp', 'a6.cpp', 'fileA.cpp', 'main.cpp', 'a7.cpp', 'fileB.cpp', 'fileC.cpp'], 'extra_files': []}, - 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['a1.cpp', 'a2.cpp', 'a6.cpp', 'fileA.cpp', 'main.cpp'], 'extra_files': []}, - 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['a7.cpp', 'fileB.cpp', 'fileC.cpp'], 'extra_files': []}, + 'trivialprog0@exe': {'name': 'trivialprog0', 'sources': ['main.cpp', 'fileA.cpp', 'fileB.cpp', 'fileC.cpp'], 'extra_files': []}, + 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp', 'fileA.cpp', 'a1.cpp', 'a2.cpp'], 'extra_files': []}, + 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['fileB.cpp', 'fileC.cpp', 'a7.cpp'], 'extra_files': []}, 'trivialprog3@exe': {'name': 'trivialprog3', 'sources': ['a5.cpp', 'fileA.cpp', 'main.cpp'], 'extra_files': []}, - 'trivialprog4@exe': {'name': 'trivialprog4', 'sources': ['a5.cpp', 'main.cpp', 'fileA.cpp'], 'extra_files': []}, - 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['a3.cpp', 'main.cpp', 'a7.cpp', 'fileB.cpp', 'fileC.cpp'], 'extra_files': []}, + 'trivialprog4@exe': {'name': 'trivialprog4', 'sources': ['main.cpp', 'a5.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['fileB.cpp', 'fileC.cpp', 'a3.cpp', 'main.cpp'], 'extra_files': []}, 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp', 'fileA.cpp', 'a4.cpp'], 'extra_files': []}, - 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileB.cpp', 'fileC.cpp', 'a1.cpp', 'a2.cpp', 'a6.cpp', 'fileA.cpp', 'main.cpp'], 'extra_files': []}, - 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['a1.cpp', 'a2.cpp', 'a6.cpp', 'fileA.cpp', 'main.cpp'], 'extra_files': []}, - 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['a1.cpp', 'a2.cpp', 'a6.cpp', 'fileA.cpp', 'main.cpp'], 'extra_files': []}, + 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileB.cpp', 'fileC.cpp', 'fileA.cpp', 'main.cpp'], 'extra_files': []}, + 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp', 'fileA.cpp', 'a1.cpp', 'a6.cpp' ], 'extra_files': []}, + 'trivialprog10@exe': {'name': 'trivialprog10', 'sources': ['main.cpp', 'fileA.cpp', 'fileB.cpp', 'a1.cpp'], 'extra_files': []}, + 'trivialprog11@exe': {'name': 'trivialprog11', 'sources': ['a1.cpp', 'fileA.cpp', 'main.cpp'], 'extra_files': []}, + 'trivialprog12@exe': {'name': 'trivialprog12', 'sources': ['a1.cpp', 'fileA.cpp', 'fileB.cpp', 'main.cpp'], 'extra_files': []}, + 'rightName@exe': {'name': 'rightName', 'sources': ['main.cpp'], 'extra_files': []}, } } - self.assertDictEqual(out, expected) + self.assertEqualIgnoreOrder(out, expected) # Check the written file out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) - self.assertDictEqual(out, expected) + self.assertEqualIgnoreOrder(out, expected) def test_target_add_sources_abs(self): self.prime('1 basic') @@ -106,7 +126,7 @@ def test_target_add_sources_abs(self): inf = json.dumps([{"type": "target", "target": "trivialprog1", "operation": "info"}]) self.rewrite(self.builddir, add) out = self.rewrite(self.builddir, inf) - expected = {'target': {'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['a1.cpp', 'a2.cpp', 'a6.cpp', 'fileA.cpp', 'main.cpp'], 'extra_files': []}}} + expected = {'target': {'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp', 'fileA.cpp', 'a1.cpp', 'a2.cpp', 'a6.cpp'], 'extra_files': []}}} self.assertDictEqual(out, expected) def test_target_remove_sources(self): @@ -114,28 +134,32 @@ def test_target_remove_sources(self): out = self.rewrite(self.builddir, os.path.join(self.builddir, 'rmSrc.json')) expected = { 'target': { - 'trivialprog0@exe': {'name': 'trivialprog0', 'sources': ['main.cpp', 'fileC.cpp'], 'extra_files': []}, - 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp'], 'extra_files': []}, - 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['fileC.cpp'], 'extra_files': []}, + 'trivialprog0@exe': {'name': 'trivialprog0', 'sources': ['main.cpp', 'fileA.cpp', 'fileB.cpp', 'fileC.cpp'], 'extra_files': []}, + 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['fileB.cpp', 'fileC.cpp'], 'extra_files': []}, 'trivialprog3@exe': {'name': 'trivialprog3', 'sources': ['main.cpp'], 'extra_files': []}, 'trivialprog4@exe': {'name': 'trivialprog4', 'sources': ['main.cpp'], 'extra_files': []}, - 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp', 'fileC.cpp'], 'extra_files': []}, + 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['fileB.cpp', 'fileC.cpp'], 'extra_files': []}, 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp'], 'extra_files': []}, - 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileC.cpp', 'main.cpp'], 'extra_files': []}, - 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp'], 'extra_files': []}, - 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp'], 'extra_files': []}, + 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileC.cpp', 'main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog10@exe': {'name': 'trivialprog10', 'sources': ['main.cpp'], 'extra_files': []}, + 'trivialprog11@exe': {'name': 'trivialprog11', 'sources': ['main.cpp'], 'extra_files': []}, + 'trivialprog12@exe': {'name': 'trivialprog12', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'rightName@exe': {'name': 'rightName', 'sources': ['main.cpp'], 'extra_files': []}, } } - self.assertDictEqual(out, expected) + self.assertEqualIgnoreOrder(out, expected) # Check the written file out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) - self.assertDictEqual(out, expected) + self.assertEqualIgnoreOrder(out, expected) def test_target_subdir(self): self.prime('2 subdirs') out = self.rewrite(self.builddir, os.path.join(self.builddir, 'addSrc.json')) - expected = {'name': 'something', 'sources': ['first.c', 'second.c', 'third.c'], 'extra_files': []} + expected = {'name': 'something', 'sources': ['third.c', f'sub2{os.path.sep}first.c', f'sub2{os.path.sep}second.c'], 'extra_files': []} self.assertDictEqual(list(out['target'].values())[0], expected) # Check the written file @@ -156,9 +180,12 @@ def test_target_remove(self): 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileB.cpp', 'fileC.cpp', 'main.cpp', 'fileA.cpp'], 'extra_files': []}, 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog10@exe': {'name': 'trivialprog10', 'sources': ['main.cpp', 'fileA.cpp', 'fileB.cpp'], 'extra_files': []}, + 'trivialprog11@exe': {'name': 'trivialprog11', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog12@exe': {'name': 'trivialprog12', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, } } - self.assertDictEqual(out, expected) + self.assertEqualIgnoreOrder(out, expected) def test_target_add(self): self.prime('1 basic') @@ -177,10 +204,14 @@ def test_target_add(self): 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileB.cpp', 'fileC.cpp', 'main.cpp', 'fileA.cpp'], 'extra_files': []}, 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, - 'trivialprog10@sha': {'name': 'trivialprog10', 'sources': ['new1.cpp', 'new2.cpp'], 'extra_files': []}, + 'trivialprog10@exe': {'name': 'trivialprog10', 'sources': ['main.cpp', 'fileA.cpp', 'fileB.cpp'], 'extra_files': []}, + 'trivialprog11@exe': {'name': 'trivialprog11', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog12@exe': {'name': 'trivialprog12', 'sources': ['main.cpp', 'fileA.cpp'], 'extra_files': []}, + 'trivialprog13@sha': {'name': 'trivialprog13', 'sources': ['new1.cpp', 'new2.cpp'], 'extra_files': []}, + 'rightName@exe': {'name': 'rightName', 'sources': ['main.cpp'], 'extra_files': []}, } } - self.assertDictEqual(out, expected) + self.assertEqualIgnoreOrder(out, expected) def test_target_remove_subdir(self): self.prime('2 subdirs') @@ -192,7 +223,7 @@ def test_target_add_subdir(self): self.prime('2 subdirs') self.rewrite(self.builddir, os.path.join(self.builddir, 'addTgt.json')) out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) - expected = {'name': 'something', 'sources': ['first.c', 'second.c'], 'extra_files': []} + expected = {'name': 'something', 'sources': [f'sub2{os.path.sep}first.c', f'sub2{os.path.sep}second.c'], 'extra_files': []} self.assertDictEqual(out['target']['94b671c@@something@exe'], expected) def test_target_source_sorting(self): @@ -243,12 +274,13 @@ def test_target_source_sorting(self): def test_target_same_name_skip(self): self.prime('4 same name targets') - out = self.rewrite(self.builddir, os.path.join(self.builddir, 'addSrc.json')) + #out = self.rewrite(self.builddir, os.path.join(self.builddir, 'addSrc.json')) out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) - expected = {'name': 'myExe', 'sources': ['main.cpp'], 'extra_files': []} + expected1 = {'name': 'myExe', 'sources': ['main.cpp'], 'extra_files': []} + expected2 = {'name': 'myExe', 'sources': [f'sub1{os.path.sep}main.cpp'], 'extra_files': []} self.assertEqual(len(out['target']), 2) - for val in out['target'].values(): - self.assertDictEqual(expected, val) + self.assertDictEqual(expected1, out['target']['myExe@exe']) + self.assertDictEqual(expected2, out['target']['9a11041@@myExe@exe']) def test_kwargs_info(self): self.prime('3 kwargs') @@ -358,48 +390,67 @@ def test_target_add_extra_files(self): out = self.rewrite(self.builddir, os.path.join(self.builddir, 'addExtraFiles.json')) expected = { 'target': { - 'trivialprog0@exe': {'name': 'trivialprog0', 'sources': ['main.cpp'], 'extra_files': ['a1.hpp', 'a2.hpp', 'a6.hpp', 'fileA.hpp', 'main.hpp', 'a7.hpp', 'fileB.hpp', 'fileC.hpp']}, - 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp'], 'extra_files': ['a1.hpp', 'a2.hpp', 'a6.hpp', 'fileA.hpp', 'main.hpp']}, + 'trivialprog0@exe': {'name': 'trivialprog0', 'sources': ['main.cpp'], 'extra_files': ['fileA.hpp', 'main.hpp', 'fileB.hpp', 'fileC.hpp']}, + 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp'], 'extra_files': ['a1.hpp', 'a2.hpp', 'fileA.hpp', 'main.hpp']}, 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['main.cpp'], 'extra_files': ['a7.hpp', 'fileB.hpp', 'fileC.hpp']}, 'trivialprog3@exe': {'name': 'trivialprog3', 'sources': ['main.cpp'], 'extra_files': ['a5.hpp', 'fileA.hpp', 'main.hpp']}, 'trivialprog4@exe': {'name': 'trivialprog4', 'sources': ['main.cpp'], 'extra_files': ['a5.hpp', 'main.hpp', 'fileA.hpp']}, - 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp'], 'extra_files': ['a3.hpp', 'main.hpp', 'a7.hpp', 'fileB.hpp', 'fileC.hpp']}, - 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp'], 'extra_files': ['a1.hpp', 'a2.hpp', 'a6.hpp', 'fileA.hpp', 'main.hpp']}, - 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['main.cpp'], 'extra_files': ['a1.hpp', 'a2.hpp', 'a6.hpp', 'fileA.hpp', 'main.hpp']}, + 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp'], 'extra_files': ['a3.hpp', 'main.hpp', 'fileB.hpp', 'fileC.hpp']}, + 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp'], 'extra_files': ['fileA.hpp', 'main.hpp']}, + 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['main.cpp'], 'extra_files': ['a1.hpp', 'a6.hpp', 'fileA.hpp', 'main.hpp']}, 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp'], 'extra_files': ['a2.hpp', 'a7.hpp']}, 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp'], 'extra_files': ['a8.hpp', 'a9.hpp']}, 'trivialprog10@exe': {'name': 'trivialprog10', 'sources': ['main.cpp'], 'extra_files': ['a1.hpp', 'a4.hpp']}, } } - self.assertDictEqual(out, expected) + self.assertEqualIgnoreOrder(out, expected) # Check the written file out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) - self.assertDictEqual(out, expected) + self.assertEqualIgnoreOrder(out, expected) def test_target_remove_extra_files(self): self.prime('6 extra_files') out = self.rewrite(self.builddir, os.path.join(self.builddir, 'rmExtraFiles.json')) expected = { 'target': { - 'trivialprog0@exe': {'name': 'trivialprog0', 'sources': ['main.cpp'], 'extra_files': ['main.hpp', 'fileC.hpp']}, - 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp'], 'extra_files': ['main.hpp']}, - 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['main.cpp'], 'extra_files': ['fileC.hpp']}, + 'trivialprog0@exe': {'name': 'trivialprog0', 'sources': ['main.cpp'], 'extra_files': ['main.hpp', 'fileA.hpp', 'fileB.hpp', 'fileC.hpp']}, + 'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp'], 'extra_files': ['main.hpp', 'fileA.hpp']}, + 'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['main.cpp'], 'extra_files': ['fileB.hpp', 'fileC.hpp']}, 'trivialprog3@exe': {'name': 'trivialprog3', 'sources': ['main.cpp'], 'extra_files': ['main.hpp']}, 'trivialprog4@exe': {'name': 'trivialprog4', 'sources': ['main.cpp'], 'extra_files': ['main.hpp']}, - 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp'], 'extra_files': ['main.hpp', 'fileC.hpp']}, - 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp'], 'extra_files': ['main.hpp']}, - 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['main.cpp'], 'extra_files': ['main.hpp']}, + 'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp'], 'extra_files': ['fileB.hpp', 'fileC.hpp', 'main.hpp']}, + 'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp'], 'extra_files': ['main.hpp', 'fileA.hpp']}, + 'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['main.cpp'], 'extra_files': ['main.hpp', 'fileA.hpp']}, 'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp'], 'extra_files': []}, 'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp'], 'extra_files': []}, 'trivialprog10@exe': {'name': 'trivialprog10', 'sources': ['main.cpp'], 'extra_files': []}, } } - self.assertDictEqual(out, expected) + self.assertEqualIgnoreOrder(out, expected) # Check the written file out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) - self.assertDictEqual(out, expected) + self.assertEqualIgnoreOrder(out, expected) + + def test_tricky_dataflow(self): + self.prime('7 tricky dataflow') + out = self.rewrite(self.builddir, os.path.join(self.builddir, 'addSrc.json')) + expected = { + 'target': { + 'tgt1@sha': {'name': 'tgt1', 'sources': ['foo.c', 'new.c'], 'extra_files': []}, + 'tgt2@exe': {'name': 'tgt2', 'sources': ['new.c', 'unknown'], 'extra_files': []}, + 'tgt3@exe': {'name': 'tgt3', 'sources': ['foo.c', 'new.c'], 'extra_files': []}, + 'tgt4@exe': {'name': 'tgt4', 'sources': ['unknown'], 'extra_files': []}, + 'tgt5@exe': {'name': 'tgt5', 'sources': ['unknown', 'new.c'], 'extra_files': []}, + 'tgt6@exe': {'name': 'tgt6', 'sources': ['unknown', 'new.c'], 'extra_files': []}, + } + } + self.assertEqualIgnoreOrder(out, expected) + + # Check the written file + out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) + self.assertEqualIgnoreOrder(out, expected) def test_raw_printer_is_idempotent(self): test_path = Path(self.unit_test_dir, '118 rewrite')