rewriter: Major rewrite of the rewriter and the static introspection …

…tool

The rewriter and the static introspection tool
used to be very broken, now it is *less* broken.

Fixes mesonbuild#11763

mesonbuild/ast/__init__.py

@@ -24,11 +24,10 @@
     'AstVisitor',
     'AstPrinter',
     'IntrospectionInterpreter',
-    'BUILD_TARGET_FUNCTIONS',
 ]
 
 from .interpreter import AstInterpreter
-from .introspection import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS
+from .introspection import IntrospectionInterpreter
 from .visitor import AstVisitor
 from .postprocess import AstConditionLevel, AstIDGenerator, AstIndentationGenerator
 from .printer import AstPrinter, AstJSONPrinter

mesonbuild/ast/printer.py

@@ -31,7 +31,40 @@
     'div': '/'
 }
 
+# Also known as "order of operations" or "binding_power"
+# This is the counterpart to Parser.e1, Parser.e2, Parser.e3, Parser.e4, Parser.e5, Parser.e6, Parser.e7, Parser.e8, Parser.e9
+def precedence_level(node: mparser.BaseNode) -> float:
+    if isinstance(node, (mparser.PlusAssignmentNode, mparser.AssignmentNode, mparser.TernaryNode)):
+        return 1.0
+    elif isinstance(node, mparser.OrNode):
+        return 2.0
+    elif isinstance(node, mparser.AndNode):
+        return 3.0
+    elif isinstance(node, mparser.ComparisonNode):
+        return 4.0
+    elif isinstance(node, mparser.ArithmeticNode):
+        if node.operation in set(['add', 'sub']):
+            return 5.1
+        elif node.operation in set(['mod', 'mul', 'div']):
+            return 5.2
+    elif isinstance(node, (mparser.NotNode, mparser.UMinusNode)):
+        return 6.0
+    elif isinstance(node, mparser.FunctionNode):
+        return 7.0
+    elif isinstance(node, (mparser.ArrayNode, mparser.DictNode)):
+        return 8.0
+    elif isinstance(node, (mparser.BooleanNode, mparser.IdNode, mparser.NumberNode, mparser.BaseStringNode, mparser.FormatStringNode, mparser.MultilineFormatStringNode, mparser.EmptyNode)):
+        return 9.0
+    elif isinstance(node, mparser.ParenthesizedNode):
+        # Parenthesize have the highest binding power, but since the AstPrinter
+        # ignores ParanthesizedNode, the binding power of the inner node is
+        # relevant.
+        return precedence_level(node.inner)
+    raise TypeError
+
 class AstPrinter(AstVisitor):
+    escape_trans: T.Dict[int, str] = str.maketrans({'\'': '\\\'', '\\': '\\\\'})
+
     def __init__(self, indent: int = 2, arg_newline_cutoff: int = 5, update_ast_line_nos: bool = False):
         self.result = ''
         self.indent = indent
@@ -76,12 +109,14 @@ def visit_NumberNode(self, node: mparser.NumberNode) -> None:
         node.lineno = self.curr_line or node.lineno
 
     def escape(self, val: str) -> str:
-        return val.translate(str.maketrans({'\'': '\\\'',
-                                            '\\': '\\\\'}))
+        return val.translate(self.escape_trans)
 
     def visit_StringNode(self, node: mparser.StringNode) -> None:
         assert isinstance(node.value, str)
-        self.append("'" + self.escape(node.value) + "'", node)
+        if '\n' in node.value:
+            self.append("'''" + self.escape(node.value) + "'''", node)
+        else:
+            self.append("'" + self.escape(node.value) + "'", node)
         node.lineno = self.curr_line or node.lineno
 
     def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None:
@@ -137,11 +172,23 @@ def visit_ComparisonNode(self, node: mparser.ComparisonNode) -> None:
         node.lineno = self.curr_line or node.lineno
         node.right.accept(self)
 
+    def maybe_parentheses(self, outer: mparser.BaseNode, inner: mparser.BaseNode, parens: bool) -> None:
+        prec_outer = precedence_level(outer)
+        prec_inner = precedence_level(inner)
+        if parens:
+            self.append('(', inner)
+        inner.accept(self)
+        if parens:
+            self.append(')', inner)
+
     def visit_ArithmeticNode(self, node: mparser.ArithmeticNode) -> None:
-        node.left.accept(self)
+        prec = precedence_level(node)
+        prec_left = precedence_level(node.left)
+        prec_right = precedence_level(node.right)
+        self.maybe_parentheses(node, node.left, prec > prec_left)
         self.append_padded(arithmic_map[node.operation], node)
         node.lineno = self.curr_line or node.lineno
-        node.right.accept(self)
+        self.maybe_parentheses(node, node.right, prec > prec_right or (prec == prec_right  and node.operation in set(['sub', 'div', 'mod'])))
 
     def visit_NotNode(self, node: mparser.NotNode) -> None:
         node.lineno = self.curr_line or node.lineno
@@ -252,21 +299,21 @@ def visit_ArgumentNode(self, node: mparser.ArgumentNode) -> None:
 
 class RawPrinter(AstVisitor):
 
-    def __init__(self):
+    def __init__(self) -> None:
         self.result = ''
 
-    def visit_default_func(self, node: mparser.BaseNode):
+    def visit_default_func(self, node: mparser.BaseNode) -> None:
         self.result += node.value
         if node.whitespaces:
             node.whitespaces.accept(self)
 
-    def visit_unary_operator(self, node: mparser.UnaryOperatorNode):
+    def visit_unary_operator(self, node: mparser.UnaryOperatorNode) -> None:
         node.operator.accept(self)
         node.value.accept(self)
         if node.whitespaces:
             node.whitespaces.accept(self)
 
-    def visit_binary_operator(self, node: mparser.BinaryOperatorNode):
+    def visit_binary_operator(self, node: mparser.BinaryOperatorNode) -> None:
         node.left.accept(self)
         node.operator.accept(self)
         node.right.accept(self)

mesonbuild/build.py

@@ -41,7 +41,7 @@
     is_header, is_object, is_source, clink_langs, sort_clink, all_languages,
     is_known_suffix, detect_static_linker
 )
-from .interpreterbase import FeatureNew, FeatureDeprecated
+from .interpreterbase import FeatureNew, FeatureDeprecated, UnknownValue
 
 if T.TYPE_CHECKING:
     from typing_extensions import Literal
@@ -640,7 +640,7 @@ def get_id(self) -> str:
     def process_kwargs_base(self, kwargs: T.Dict[str, T.Any]) -> None:
         if 'build_by_default' in kwargs:
             self.build_by_default = kwargs['build_by_default']
-            if not isinstance(self.build_by_default, bool):
+            if not isinstance(self.build_by_default, (bool, UnknownValue)):
                 raise InvalidArguments('build_by_default must be a boolean value.')
         elif kwargs.get('install', False):
             # For backward compatibility, if build_by_default is not explicitly
@@ -870,7 +870,7 @@ def can_compile_remove_sources(compiler: 'Compiler', sources: T.List['FileOrStri
                 removed = True
         return removed
 
-    def process_compilers_late(self):
+    def process_compilers_late(self) -> None:
         """Processes additional compilers after kwargs have been evaluated.
 
         This can add extra compilers that might be required by keyword

mesonbuild/interpreterbase/__init__.py

@@ -75,6 +75,8 @@
     'TYPE_HoldableTypes',
 
     'HoldableTypes',
+
+    'UnknownValue',
 ]
 
 from .baseobjects import (
@@ -100,6 +102,8 @@
     SubProject,
 
     HoldableTypes,
+
+    UnknownValue,
 )
 
 from .decorators import (

mesonbuild/interpreterbase/baseobjects.py

@@ -134,6 +134,12 @@ class MesonInterpreterObject(InterpreterObject):
 class MutableInterpreterObject:
     ''' Dummy class to mark the object type as mutable '''
 
+class UnknownValue(MesonInterpreterObject):
+    '''This class is only used for the rewriter/static introspection tool and
+    indicates that a value cannot be determined statically, either because of
+    limitations in our code or because the value differs from machine to
+    machine.'''
+
 HoldableTypes = (HoldableObject, int, bool, str, list, dict)
 TYPE_HoldableTypes = T.Union[TYPE_elementary, HoldableObject]
 InterpreterObjectTypeVar = T.TypeVar('InterpreterObjectTypeVar', bound=TYPE_HoldableTypes)

mesonbuild/interpreterbase/interpreterbase.py

@@ -243,6 +243,8 @@ def evaluate_statement(self, cur: mparser.BaseNode) -> T.Optional[InterpreterObj
             return self.evaluate_statement(cur.inner)
         elif isinstance(cur, mparser.TestCaseClauseNode):
             return self.evaluate_testcase(cur)
+        elif isinstance(cur, mparser.EmptyNode):
+            return None
         else:
             raise InvalidCode("Unknown statement.")
         return None
@@ -505,7 +507,7 @@ def evaluate_indexing(self, node: mparser.IndexNode) -> InterpreterObject:
 
     def function_call(self, node: mparser.FunctionNode) -> T.Optional[InterpreterObject]:
         func_name = node.func_name.value
-        (h_posargs, h_kwargs) = self.reduce_arguments(node.args)
+        (h_posargs, h_kwargs) = self.reduce_arguments(node.args, include_unknown_args = True)
         (posargs, kwargs) = self._unholder_args(h_posargs, h_kwargs)
         if is_disabled(posargs, kwargs) and func_name not in {'get_variable', 'set_variable', 'unset_variable', 'is_disabler'}:
             return Disabler()
@@ -533,7 +535,7 @@ def method_call(self, node: mparser.MethodNode) -> T.Optional[InterpreterObject]
             object_display_name = invocable.__class__.__name__
             obj = self.evaluate_statement(invocable)
         method_name = node.name.value
-        (h_args, h_kwargs) = self.reduce_arguments(node.args)
+        (h_args, h_kwargs) = self.reduce_arguments(node.args, include_unknown_args = True)
         (args, kwargs) = self._unholder_args(h_args, h_kwargs)
         if is_disabled(args, kwargs):
             return Disabler()
@@ -581,6 +583,7 @@ def reduce_arguments(
                 args: mparser.ArgumentNode,
                 key_resolver: T.Callable[[mparser.BaseNode], str] = default_resolve_key,
                 duplicate_key_error: T.Optional[str] = None,
+                include_unknown_args: bool = False,
             ) -> T.Tuple[
                 T.List[InterpreterObject],
                 T.Dict[str, InterpreterObject]

mesonbuild/mintro.py

@@ -30,20 +30,25 @@
 import typing as T
 
 from . import build, mesonlib, coredata as cdata
-from .ast import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS, AstConditionLevel, AstIDGenerator, AstIndentationGenerator, AstJSONPrinter
+from .ast import IntrospectionInterpreter, AstConditionLevel, AstIDGenerator, AstIndentationGenerator, AstJSONPrinter
 from .backend import backends
 from .dependencies import Dependency
 from . import environment
-from .interpreterbase import ObjectHolder
+from .interpreterbase import ObjectHolder, UnknownValue
 from .mesonlib import OptionKey
-from .mparser import FunctionNode, ArrayNode, ArgumentNode, BaseStringNode
 
 if T.TYPE_CHECKING:
     import argparse
 
     from .interpreter import Interpreter
     from .mparser import BaseNode
 
+class IntrospectionEncoder(json.JSONEncoder):
+    def default(self, obj: T.Any) -> T.Any:
+        if isinstance(obj, UnknownValue):
+            return 'unknown'
+        return json.JSONEncoder.default(self, obj)
+
 def get_meson_info_file(info_dir: str) -> str:
     return os.path.join(info_dir, 'meson-info.json')
 
@@ -64,8 +69,7 @@ def __init__(self,
 
 def get_meson_introspection_types(coredata: T.Optional[cdata.CoreData] = None,
                                   builddata: T.Optional[build.Build] = None,
-                                  backend: T.Optional[backends.Backend] = None,
-                                  sourcedir: T.Optional[str] = None) -> 'T.Mapping[str, IntroCommand]':
+                                  backend: T.Optional[backends.Backend] = None) -> 'T.Mapping[str, IntroCommand]':
     if backend and builddata:
         benchmarkdata = backend.create_test_serialisation(builddata.get_benchmarks())
         testdata = backend.create_test_serialisation(builddata.get_tests())
@@ -177,55 +181,34 @@ def get_target_dir(coredata: cdata.CoreData, subdir: str) -> str:
     else:
         return subdir
 
-def list_targets_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[str, T.Union[bool, str, T.List[T.Union[str, T.Dict[str, T.Union[str, T.List[str], bool]]]]]]]:
-    tlist: T.List[T.Dict[str, T.Union[bool, str, T.List[T.Union[str, T.Dict[str, T.Union[str, T.List[str], bool]]]]]]] = []
-    root_dir = Path(intr.source_root)
-
-    def nodes_to_paths(node_list: T.List[BaseNode]) -> T.List[Path]:
-        res: T.List[Path] = []
-        for n in node_list:
-            args: T.List[BaseNode] = []
-            if isinstance(n, FunctionNode):
-                args = list(n.args.arguments)
-                if n.func_name.value in BUILD_TARGET_FUNCTIONS:
-                    args.pop(0)
-            elif isinstance(n, ArrayNode):
-                args = n.args.arguments
-            elif isinstance(n, ArgumentNode):
-                args = n.arguments
-            for j in args:
-                if isinstance(j, BaseStringNode):
-                    assert isinstance(j.value, str)
-                    res += [Path(j.value)]
-                elif isinstance(j, str):
-                    res += [Path(j)]
-        res = [root_dir / i['subdir'] / x for x in res]
-        res = [x.resolve() for x in res]
-        return res
+def list_targets_from_source(intr: IntrospectionInterpreter) -> T.Any:
+    tlist = []
+    root_dir = Path(intr.source_root).resolve()
 
     for i in intr.targets:
-        sources = nodes_to_paths(i['sources'])
-        extra_f = nodes_to_paths(i['extra_files'])
-        outdir = get_target_dir(intr.coredata, i['subdir'])
+        sources = intr.nodes_to_pretty_filelist(root_dir, i.subdir, i.source_nodes)
+        extra_files = intr.nodes_to_pretty_filelist(root_dir, i.subdir, [i.extra_files] if i.extra_files else [])
+
+        outdir = get_target_dir(intr.coredata, i.subdir)
 
         tlist += [{
-            'name': i['name'],
-            'id': i['id'],
-            'type': i['type'],
-            'defined_in': i['defined_in'],
-            'filename': [os.path.join(outdir, x) for x in i['outputs']],
-            'build_by_default': i['build_by_default'],
+            'name': i.name,
+            'id': i.id,
+            'type': i.typename,
+            'defined_in': i.defined_in,
+            'filename': [os.path.join(outdir, x) for x in i.outputs],
+            'build_by_default': i.build_by_default,
             'target_sources': [{
                 'language': 'unknown',
                 'compiler': [],
                 'parameters': [],
-                'sources': [str(x) for x in sources],
+                'sources': sources,
                 'generated_sources': []
             }],
             'depends': [],
-            'extra_files': [str(x) for x in extra_f],
+            'extra_files': extra_files,
             'subproject': None, # Subprojects are not supported
-            'installed': i['installed']
+            'installed': i.installed
         }]
 
     return tlist
@@ -388,7 +371,7 @@ def list_deps_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[str,
             'has_fallback',
             'conditional',
         ]
-        result += [{k: v for k, v in i.items() if k in keys}]
+        result += [{k: v for k, v in i.__dict__.items() if k in keys}]
     return result
 
 def list_deps(coredata: cdata.CoreData, backend: backends.Backend) -> T.List[T.Dict[str, T.Union[str, T.List[str]]]]:
@@ -513,7 +496,7 @@ def print_results(options: argparse.Namespace, results: T.Sequence[T.Tuple[str,
         return 1
     elif len(results) == 1 and not options.force_dict:
         # Make to keep the existing output format for a single option
-        print(json.dumps(results[0][1], indent=indent))
+        print(json.dumps(results[0][1], indent=indent, cls=IntrospectionEncoder))
     else:
         out = {}
         for i in results:
@@ -542,10 +525,11 @@ def run(options: argparse.Namespace) -> int:
         datadir = os.path.join(options.builddir, datadir)
     indent = 4 if options.indent else None
     results: T.List[T.Tuple[str, T.Union[dict, T.List[T.Any]]]] = []
-    sourcedir = '.' if options.builddir == 'meson.build' else options.builddir[:-11]
-    intro_types = get_meson_introspection_types(sourcedir=sourcedir)
+    intro_types = get_meson_introspection_types()
 
-    if 'meson.build' in [os.path.basename(options.builddir), options.builddir]:
+    # TODO: This if clause is undocumented.
+    if os.path.basename(options.builddir) == 'meson.build':
+        sourcedir = '.' if options.builddir == 'meson.build' else options.builddir[:-11]
         # Make sure that log entries in other parts of meson don't interfere with the JSON output
         with redirect_stdout(sys.stderr):
             backend = backends.get_backend_from_name(options.backend)

mesonbuild/mparser.py

@@ -204,7 +204,7 @@ def lex(self, filename: str) -> T.Generator[Token, None, None]:
                         lines = value.split('\n')
                         if len(lines) > 1:
                             lineno += len(lines) - 1
-                            line_start = mo.end() - len(lines[-1])
+                            line_start = mo.end() - len(lines[-1]) - 3
                     elif tid == 'eol_cont':
                         lineno += 1
                         line_start = loc
@@ -373,6 +373,13 @@ def set_kwarg(self, name: IdNode, value: BaseNode) -> None:
             mlog.warning('This will be an error in future Meson releases.')
         self.kwargs[name] = value
 
+    def get_kwarg_or_default(self, name: str, default: BaseNode) -> BaseNode:
+        for k, v in self.kwargs.items():
+            assert isinstance(k, IdNode)
+            if k.value == name:
+                return v
+        return default
+
     def set_kwarg_no_check(self, name: BaseNode, value: BaseNode) -> None:
         self.kwargs[name] = value