Major rewrite of the rewriter and the static introspection tool

The rewriter and the static introspection tool
used to be very broken, now it is *less* broken.

The most important changes are:

1. We now have class UnknownValue for more explicit handling
	of situations that are too complex/impossible.

2. If you write
	```
	var = 'foo'
	name = var
	var = 'bar'
	executable(name, 'foo.c')
	```
	the tool now knows that the name of the executable is foo and not bar.
	See dataflow_dag and node_to_runtime_value for details on how we do this.

Fixes #11763

mesonbuild/ast/__init__.py

@@ -24,11 +24,10 @@
     'AstVisitor',
     'AstPrinter',
     'IntrospectionInterpreter',
-    'BUILD_TARGET_FUNCTIONS',
 ]
 
 from .interpreter import AstInterpreter
-from .introspection import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS
+from .introspection import IntrospectionInterpreter
 from .visitor import AstVisitor
 from .postprocess import AstConditionLevel, AstIDGenerator, AstIndentationGenerator
 from .printer import AstPrinter, AstJSONPrinter

mesonbuild/ast/printer.py

@@ -17,6 +17,7 @@
 from __future__ import annotations
 
 from .. import mparser
+from ..mesonlib import MesonBugException
 from .visitor import AstVisitor
 
 from itertools import zip_longest
@@ -32,35 +33,35 @@
 }
 
 # Also known as "order of operations" or "binding_power"
-# This is the counterpart to Parser.e1, Parser.e2, Parser.e3, Parser.e4, Parser.e5, Parser.e6, Parser.e7, Parser.e8, Parser.e9
-def precedence_level(node: mparser.BaseNode) -> float:
+# This is the counterpart to Parser.e1, Parser.e2, Parser.e3, Parser.e4, Parser.e5, Parser.e6, Parser.e7, Parser.e8, Parser.e9, Parser.e10
+def precedence_level(node: mparser.BaseNode) -> int:
     if isinstance(node, (mparser.PlusAssignmentNode, mparser.AssignmentNode, mparser.TernaryNode)):
-        return 1.0
+        return 1
     elif isinstance(node, mparser.OrNode):
-        return 2.0
+        return 2
     elif isinstance(node, mparser.AndNode):
-        return 3.0
+        return 3
     elif isinstance(node, mparser.ComparisonNode):
-        return 4.0
+        return 4
     elif isinstance(node, mparser.ArithmeticNode):
         if node.operation in set(['add', 'sub']):
-            return 5.1
+            return 5
         elif node.operation in set(['mod', 'mul', 'div']):
-            return 5.2
+            return 6
     elif isinstance(node, (mparser.NotNode, mparser.UMinusNode)):
-        return 6.0
+        return 7
     elif isinstance(node, mparser.FunctionNode):
-        return 7.0
+        return 8
     elif isinstance(node, (mparser.ArrayNode, mparser.DictNode)):
-        return 8.0
+        return 9
     elif isinstance(node, (mparser.BooleanNode, mparser.IdNode, mparser.NumberNode, mparser.BaseStringNode, mparser.FormatStringNode, mparser.MultilineFormatStringNode, mparser.EmptyNode)):
-        return 9.0
+        return 10
     elif isinstance(node, mparser.ParenthesizedNode):
         # Parenthesize have the highest binding power, but since the AstPrinter
         # ignores ParanthesizedNode, the binding power of the inner node is
         # relevant.
         return precedence_level(node.inner)
-    raise TypeError
+    raise MesonBugException('Unhandled node type')
 
 class AstPrinter(AstVisitor):
     escape_trans: T.Dict[int, str] = str.maketrans({'\'': '\\\'', '\\': '\\\\'})
@@ -186,7 +187,7 @@ def visit_ArithmeticNode(self, node: mparser.ArithmeticNode) -> None:
         self.maybe_parentheses(node, node.left, prec > prec_left)
         self.append_padded(arithmic_map[node.operation], node)
         node.lineno = self.curr_line or node.lineno
-        self.maybe_parentheses(node, node.right, prec > prec_right or (prec == prec_right and node.operation in set(['sub', 'div', 'mod'])))
+        self.maybe_parentheses(node, node.right, prec > prec_right or (prec == prec_right and node.operation in {'sub', 'div', 'mod'}))
 
     def visit_NotNode(self, node: mparser.NotNode) -> None:
         node.lineno = self.curr_line or node.lineno

mesonbuild/build.py

@@ -41,7 +41,7 @@
     is_header, is_object, is_source, clink_langs, sort_clink, all_languages,
     is_known_suffix, detect_static_linker
 )
-from .interpreterbase import FeatureNew, FeatureDeprecated
+from .interpreterbase import FeatureNew, FeatureDeprecated, UnknownValue
 
 if T.TYPE_CHECKING:
     from typing_extensions import Literal, TypedDict
@@ -651,7 +651,7 @@ def get_id(self) -> str:
     def process_kwargs_base(self, kwargs: T.Dict[str, T.Any]) -> None:
         if 'build_by_default' in kwargs:
             self.build_by_default = kwargs['build_by_default']
-            if not isinstance(self.build_by_default, bool):
+            if not isinstance(self.build_by_default, (bool, UnknownValue)):
                 raise InvalidArguments('build_by_default must be a boolean value.')
 
         if not self.build_by_default and kwargs.get('install', False):
@@ -892,7 +892,7 @@ def can_compile_remove_sources(compiler: 'Compiler', sources: T.List['FileOrStri
                 removed = True
         return removed
 
-    def process_compilers_late(self):
+    def process_compilers_late(self) -> None:
         """Processes additional compilers after kwargs have been evaluated.
 
         This can add extra compilers that might be required by keyword

mesonbuild/interpreterbase/__init__.py

@@ -71,6 +71,8 @@
     'TYPE_HoldableTypes',
 
     'HoldableTypes',
+
+    'UnknownValue',
 ]
 
 from .baseobjects import (
@@ -93,6 +95,8 @@
     SubProject,
 
     HoldableTypes,
+
+    UnknownValue,
 )
 
 from .decorators import (

mesonbuild/interpreterbase/baseobjects.py

@@ -131,6 +131,12 @@ class MesonInterpreterObject(InterpreterObject):
 class MutableInterpreterObject:
     ''' Dummy class to mark the object type as mutable '''
 
+class UnknownValue(MesonInterpreterObject):
+    '''This class is only used for the rewriter/static introspection tool and
+    indicates that a value cannot be determined statically, either because of
+    limitations in our code or because the value differs from machine to
+    machine.'''
+
 HoldableTypes = (HoldableObject, int, bool, str, list, dict)
 TYPE_HoldableTypes = T.Union[TYPE_elementary, HoldableObject]
 InterpreterObjectTypeVar = T.TypeVar('InterpreterObjectTypeVar', bound=TYPE_HoldableTypes)

mesonbuild/interpreterbase/interpreterbase.py

@@ -243,6 +243,8 @@ def evaluate_statement(self, cur: mparser.BaseNode) -> T.Optional[InterpreterObj
             return self.evaluate_statement(cur.inner)
         elif isinstance(cur, mparser.TestCaseClauseNode):
             return self.evaluate_testcase(cur)
+        elif isinstance(cur, mparser.EmptyNode):
+            return None
         else:
             raise InvalidCode("Unknown statement.")
         return None
@@ -505,7 +507,7 @@ def evaluate_indexing(self, node: mparser.IndexNode) -> InterpreterObject:
 
     def function_call(self, node: mparser.FunctionNode) -> T.Optional[InterpreterObject]:
         func_name = node.func_name.value
-        (h_posargs, h_kwargs) = self.reduce_arguments(node.args)
+        (h_posargs, h_kwargs) = self.reduce_arguments(node.args, include_unknown_args=True)
         (posargs, kwargs) = self._unholder_args(h_posargs, h_kwargs)
         if is_disabled(posargs, kwargs) and func_name not in {'get_variable', 'set_variable', 'unset_variable', 'is_disabler'}:
             return Disabler()
@@ -533,7 +535,7 @@ def method_call(self, node: mparser.MethodNode) -> T.Optional[InterpreterObject]
             object_display_name = invocable.__class__.__name__
             obj = self.evaluate_statement(invocable)
         method_name = node.name.value
-        (h_args, h_kwargs) = self.reduce_arguments(node.args)
+        (h_args, h_kwargs) = self.reduce_arguments(node.args, include_unknown_args = True)
         (args, kwargs) = self._unholder_args(h_args, h_kwargs)
         if is_disabled(args, kwargs):
             return Disabler()
@@ -581,6 +583,7 @@ def reduce_arguments(
                 args: mparser.ArgumentNode,
                 key_resolver: T.Callable[[mparser.BaseNode], str] = default_resolve_key,
                 duplicate_key_error: T.Optional[str] = None,
+                include_unknown_args: bool = False,
             ) -> T.Tuple[
                 T.List[InterpreterObject],
                 T.Dict[str, InterpreterObject]

mesonbuild/mintro.py

@@ -29,20 +29,23 @@
 import sys
 import typing as T
 
-from . import build, mesonlib, coredata as cdata
-from .ast import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS, AstConditionLevel, AstIDGenerator, AstIndentationGenerator, AstJSONPrinter
+from . import build, environment, mesonlib, coredata as cdata
+from .ast import IntrospectionInterpreter, AstConditionLevel, AstIDGenerator, AstIndentationGenerator, AstJSONPrinter
 from .backend import backends
 from .dependencies import Dependency
-from . import environment
-from .interpreterbase import ObjectHolder
+from .interpreterbase import ObjectHolder, UnknownValue
 from .mesonlib import OptionKey
-from .mparser import FunctionNode, ArrayNode, ArgumentNode, BaseStringNode
 
 if T.TYPE_CHECKING:
     import argparse
 
     from .interpreter import Interpreter
-    from .mparser import BaseNode
+
+class IntrospectionEncoder(json.JSONEncoder):
+    def default(self, obj: T.Any) -> T.Any:
+        if isinstance(obj, UnknownValue):
+            return 'unknown'
+        return json.JSONEncoder.default(self, obj)
 
 def get_meson_info_file(info_dir: str) -> str:
     return os.path.join(info_dir, 'meson-info.json')
@@ -64,7 +67,7 @@ def __init__(self,
 
 def get_meson_introspection_types(coredata: T.Optional[cdata.CoreData] = None,
                                   builddata: T.Optional[build.Build] = None,
-                                  backend: T.Optional[backends.Backend] = None) -> 'T.Mapping[str, IntroCommand]':
+                                  backend: T.Optional[backends.Backend] = None) -> T.Mapping[str, IntroCommand]:
     if backend and builddata:
         benchmarkdata = backend.create_test_serialisation(builddata.get_benchmarks())
         testdata = backend.create_test_serialisation(builddata.get_tests())
@@ -176,55 +179,34 @@ def get_target_dir(coredata: cdata.CoreData, subdir: str) -> str:
     else:
         return subdir
 
-def list_targets_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[str, T.Union[bool, str, T.List[T.Union[str, T.Dict[str, T.Union[str, T.List[str], bool]]]]]]]:
-    tlist: T.List[T.Dict[str, T.Union[bool, str, T.List[T.Union[str, T.Dict[str, T.Union[str, T.List[str], bool]]]]]]] = []
-    root_dir = Path(intr.source_root)
-
-    def nodes_to_paths(node_list: T.List[BaseNode]) -> T.List[Path]:
-        res: T.List[Path] = []
-        for n in node_list:
-            args: T.List[BaseNode] = []
-            if isinstance(n, FunctionNode):
-                args = list(n.args.arguments)
-                if n.func_name.value in BUILD_TARGET_FUNCTIONS:
-                    args.pop(0)
-            elif isinstance(n, ArrayNode):
-                args = n.args.arguments
-            elif isinstance(n, ArgumentNode):
-                args = n.arguments
-            for j in args:
-                if isinstance(j, BaseStringNode):
-                    assert isinstance(j.value, str)
-                    res += [Path(j.value)]
-                elif isinstance(j, str):
-                    res += [Path(j)]
-        res = [root_dir / i['subdir'] / x for x in res]
-        res = [x.resolve() for x in res]
-        return res
+def list_targets_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[str, object]]:
+    tlist = []
+    root_dir = Path(intr.source_root).resolve()
 
     for i in intr.targets:
-        sources = nodes_to_paths(i['sources'])
-        extra_f = nodes_to_paths(i['extra_files'])
-        outdir = get_target_dir(intr.coredata, i['subdir'])
+        sources = intr.nodes_to_pretty_filelist(root_dir, i.subdir, i.source_nodes)
+        extra_files = intr.nodes_to_pretty_filelist(root_dir, i.subdir, [i.extra_files] if i.extra_files else [])
+
+        outdir = get_target_dir(intr.coredata, i.subdir)
 
         tlist += [{
-            'name': i['name'],
-            'id': i['id'],
-            'type': i['type'],
-            'defined_in': i['defined_in'],
-            'filename': [os.path.join(outdir, x) for x in i['outputs']],
-            'build_by_default': i['build_by_default'],
+            'name': i.name,
+            'id': i.id,
+            'type': i.typename,
+            'defined_in': i.defined_in,
+            'filename': [os.path.join(outdir, x) for x in i.outputs],
+            'build_by_default': i.build_by_default,
             'target_sources': [{
                 'language': 'unknown',
                 'compiler': [],
                 'parameters': [],
-                'sources': [str(x) for x in sources],
+                'sources': sources,
                 'generated_sources': []
             }],
             'depends': [],
-            'extra_files': [str(x) for x in extra_f],
+            'extra_files': extra_files,
             'subproject': None, # Subprojects are not supported
-            'installed': i['installed']
+            'installed': i.installed
         }]
 
     return tlist
@@ -377,17 +359,16 @@ def list_compilers(coredata: cdata.CoreData) -> T.Dict[str, T.Dict[str, T.Dict[s
             }
     return compilers
 
-def list_deps_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[str, T.Union[str, bool]]]:
-    result: T.List[T.Dict[str, T.Union[str, bool]]] = []
+def list_deps_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[str, object]]:
+    result: T.List[T.Dict[str, object]] = []
     for i in intr.dependencies:
-        keys = [
-            'name',
-            'required',
-            'version',
-            'has_fallback',
-            'conditional',
-        ]
-        result += [{k: v for k, v in i.items() if k in keys}]
+        result += [{
+            'name': i.name,
+            'required': i.required,
+            'version': i.version,
+            'has_fallback': i.has_fallback,
+            'conditional': i.conditional,
+        }]
     return result
 
 def list_deps(coredata: cdata.CoreData, backend: backends.Backend) -> T.List[T.Dict[str, T.Union[str, T.List[str]]]]:
@@ -512,12 +493,12 @@ def print_results(options: argparse.Namespace, results: T.Sequence[T.Tuple[str,
         return 1
     elif len(results) == 1 and not options.force_dict:
         # Make to keep the existing output format for a single option
-        print(json.dumps(results[0][1], indent=indent))
+        print(json.dumps(results[0][1], indent=indent, cls=IntrospectionEncoder))
     else:
         out = {}
         for i in results:
             out[i[0]] = i[1]
-        print(json.dumps(out, indent=indent))
+        print(json.dumps(out, indent=indent, cls=IntrospectionEncoder))
     return 0
 
 def get_infodir(builddir: T.Optional[str] = None) -> str:
@@ -541,10 +522,11 @@ def run(options: argparse.Namespace) -> int:
         datadir = os.path.join(options.builddir, datadir)
     indent = 4 if options.indent else None
     results: T.List[T.Tuple[str, T.Union[dict, T.List[T.Any]]]] = []
-    sourcedir = '.' if options.builddir == 'meson.build' else options.builddir[:-11]
     intro_types = get_meson_introspection_types()
 
-    if 'meson.build' in [os.path.basename(options.builddir), options.builddir]:
+    # TODO: This if clause is undocumented.
+    if os.path.basename(options.builddir) == environment.build_filename:
+        sourcedir = '.' if options.builddir == environment.build_filename else options.builddir[:-len(environment.build_filename)]
         # Make sure that log entries in other parts of meson don't interfere with the JSON output
         with redirect_stdout(sys.stderr):
             backend = backends.get_backend_from_name(options.backend)