From bbaf51f5912920bb3f21c21f4178f0a52773e213 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Sun, 3 Mar 2024 00:14:13 -0700 Subject: [PATCH 01/61] Define map HOF --- concat/stdlib/pyinterop/__init__.py | 34 +++++++++++++++++++++++++++++ concat/typecheck/types.py | 7 ++++++ 2 files changed, 41 insertions(+) diff --git a/concat/stdlib/pyinterop/__init__.py b/concat/stdlib/pyinterop/__init__.py index 71ebc7c..890e49f 100644 --- a/concat/stdlib/pyinterop/__init__.py +++ b/concat/stdlib/pyinterop/__init__.py @@ -1,8 +1,10 @@ """Concat-Python interoperation helpers.""" +from concat.common_types import ConcatFunction import concat.stdlib.ski from concat.typecheck.types import ( ForAll, IndividualVariable, + ObjectType, SequenceVariable, StackEffect, TypeSequence, @@ -51,6 +53,25 @@ TypeSequence([_stack_type_var, _y]), ), ), + 'map': ObjectType( + _x, + { + '__call__': StackEffect( + TypeSequence( + [ + _rest_var, + StackEffect( + TypeSequence([_rest_var, _y]), + TypeSequence([_rest_var, _z]), + ), + iterable_type[_y,], + ] + ), + TypeSequence([_rest_var, iterable_type[_z,]]), + ) + }, + [_rest_var, _y, _z], + ), 'to_dict': ForAll( [_stack_type_var, _x, _y], StackEffect( @@ -339,6 +360,19 @@ def import_advanced(stack: List[object], stash: List[object]) -> None: ) +def map(stack: List[object], stash: List[object]) -> None: + 'f iterable -- map(f, iterable)' + iterable = cast(Iterable[object], stack.pop()) + f = cast(ConcatFunction, stack.pop()) + + def python_f(x: object) -> object: + stack.append(x) + f(stack, stash) + return stack.pop() + + stack.append(map(python_f, iterable)) + + def open(stack: List[object], stash: List[object]) -> None: 'kwargs -- open(**kwargs)' # open has a lot of arguments stack.append(builtins.open(**cast(Mapping[str, Any], stack.pop()))) diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 4a94900..2270948 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -1935,6 +1935,13 @@ def _mapping_to_str(mapping: Mapping) -> str: int_type, ], 'join': py_function_type[TypeSequence([_x, iterable_type[_x,]]), _x], + '__iter__': py_function_type[TypeSequence([]), iterator_type[_x,]], + 'index': py_function_type[ + TypeSequence( + [_x, optional_type[int_type,], optional_type[int_type,]] + ), + int_type, + ], }, nominal=True, ) From 978db2be01619999f2ca220858168a5d23ed7fbd Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Tue, 5 Mar 2024 22:15:58 -0700 Subject: [PATCH 02/61] Resolve "from-import"-ed names using stub declarations Now, we don't need to import modules and potentially execute arbitrary code in the type checker to get types. --- concat/examples/continuation.cat | 7 +- concat/parse.py | 22 ++- concat/stdlib/continuation.cati | 20 +++ concat/stdlib/continuation.py | 123 ++------------ concat/stdlib/pyinterop/__init__.cati | 5 + concat/tests/test_typecheck.py | 12 +- concat/typecheck/__init__.py | 224 +++++++++++++++++--------- concat/typecheck/types.py | 19 ++- 8 files changed, 236 insertions(+), 196 deletions(-) create mode 100644 concat/stdlib/continuation.cati create mode 100644 concat/stdlib/pyinterop/__init__.cati diff --git a/concat/examples/continuation.cat b/concat/examples/continuation.cat index 0745dae..137593a 100644 --- a/concat/examples/continuation.cat +++ b/concat/examples/continuation.cat @@ -6,6 +6,7 @@ from concat.stdlib.continuation import ContinuationMonad from concat.stdlib.continuation import call_with_current_continuation from concat.stdlib.continuation import eval_cont from concat.stdlib.continuation import cont_pure +from concat.stdlib.continuation import map_cont from concat.stdlib.continuation import bind_cont from concat.stdlib.continuation import cont_from_cps from concat.stdlib.pyinterop import to_dict @@ -21,10 +22,14 @@ def abort(k:forall *s. (*s x:`a -- *s n:none) -- n:none): # arguments. +def ignore_int(i:int -- n:none): + drop None + + def ten_times(k:forall *s. (*s i:int -- *s c:ContinuationMonad[none, int]) -- c:ContinuationMonad[none, none]): # k 0 $(k:forall *s. (*s i:int -- *s c:ContinuationMonad[none, int]) n:int: - 1 + dup pick call eval_cont drop dup 10 < + 1 + dup pick call $:~ignore_int map_cont eval_cont drop dup 10 < ) loop drop drop $:~abort cont_from_cps diff --git a/concat/parse.py b/concat/parse.py index a993ea5..95d47aa 100644 --- a/concat/parse.py +++ b/concat/parse.py @@ -352,6 +352,7 @@ def __init__( decorators: Optional['Words'] = None, bases: Iterable['Words'] = (), keyword_args: Iterable[Tuple[str, WordNode]] = (), + type_parameters: Iterable[Node] = (), ): super().__init__() self.location = location @@ -360,6 +361,7 @@ def __init__( self.decorators = [] if decorators is None else decorators self.bases = bases self.keyword_args = keyword_args + self.type_parameters = type_parameters def token(typ: str) -> concat.parser_combinators.Parser: @@ -654,15 +656,24 @@ def from_import_star_statement_parser() -> Generator: parsers['import-statement'] |= from_import_star_statement_parser - # This parses a class definition statement. - # classdef statement = CLASS, NAME, decorator*, [ bases ], keyword arg*, - # COLON, suite ; - # bases = tuple word ; - # keyword arg = NAME, EQUAL, word ; @concat.parser_combinators.generate('classdef statement') def classdef_statement_parser(): + """This parses a class definition statement. + + classdef statement = CLASS, NAME, + [ LSQB, type variable, (COMMA, type variable)*, [ COMMA ], RSQB ], + decorator*, [ bases ], keyword arg*, + COLON, suite ; + bases = tuple word ; + keyword arg = NAME, EQUAL, word ;""" location = (yield token('CLASS')).start name_token = yield token('NAME') + type_parameters = yield bracketed( + token('LSQB'), + parsers['type-variable'].sep_by(token('COMMA')) + << token('COMMA').optional(), + token('RSQB'), + ).optional() decorators = yield decorator.many() bases_list = yield bases.optional() keyword_args = yield keyword_arg.map(tuple).many() @@ -675,6 +686,7 @@ def classdef_statement_parser(): decorators, bases_list, keyword_args, + type_parameters=type_parameters or [] ) parsers['classdef-statement'] = classdef_statement_parser diff --git a/concat/stdlib/continuation.cati b/concat/stdlib/continuation.cati new file mode 100644 index 0000000..c2835f9 --- /dev/null +++ b/concat/stdlib/continuation.cati @@ -0,0 +1,20 @@ +class ContinuationMonad[`r, `a]: + () + +def call_with_current_continuation(*s f:forall *t. (*t g:forall *u. (*u x:`a -- *u cont1:ContinuationMonad[`r, `b]) -- *t cont2:ContinuationMonad[`r, `a]) -- *s cont:ContinuationMonad[`r, `a]): + () + +def eval_cont(*s cont:ContinuationMonad[`r, `r] -- *s res:`r): + () + +def cont_pure(*s x:`a -- *s cont:ContinuationMonad[`r, `a]): + () + +def map_cont(*s cont:ContinuationMonad[`r, `a] f:forall *t. (*t x:`a -- *t y:`b) -- *s cont2:ContinuationMonad[`r, `b]): + () + +def bind_cont(*s cont:ContinuationMonad[`r, `a] f:forall *t. (*t x:`a -- *t cont:ContinuationMonad[`r, `b]) -- *s cont2:ContinuationMonad[`r, `b]): + () + +def cont_from_cps(*s cps:forall *t. (*t k:forall *u. (*u x:`a -- *u res:`r) -- *t res:`r) -- *s cont:ContinuationMonad[`r, `a]): + () diff --git a/concat/stdlib/continuation.py b/concat/stdlib/continuation.py index 6eed251..12bdc6e 100644 --- a/concat/stdlib/continuation.py +++ b/concat/stdlib/continuation.py @@ -7,7 +7,7 @@ TypeSequence, continuation_monad_type, ) -from typing import Callable, Generic, List, NoReturn, Type, TypeVar, cast +from typing import Any, Callable, Generic, List, NoReturn, Type, TypeVar, cast _A = TypeVar('_A', covariant=True) @@ -93,112 +93,6 @@ def compose( _s, _t, _u = SequenceVariable(), SequenceVariable(), SequenceVariable() _a, _b, _r = IndividualVariable(), IndividualVariable(), IndividualVariable() -globals()['@@types'] = { - 'ContinuationMonad': continuation_monad_type, - 'call_with_current_continuation': ForAll( - [_s, _r, _a, _b], - StackEffect( - TypeSequence( - [ - _s, - ForAll( - [_t], - StackEffect( - TypeSequence( - [ - _t, - ForAll( - [_u], - StackEffect( - TypeSequence([_u, _a]), - TypeSequence( - [ - _u, - continuation_monad_type[ - _r, _b - ], - ] - ), - ), - ), - ] - ), - TypeSequence( - [_t, continuation_monad_type[_r, _a]] - ), - ), - ), - ] - ), - TypeSequence([_s, continuation_monad_type[_r, _a]]), - ), - ), - 'eval_cont': ForAll( - [_s, _r], - StackEffect( - TypeSequence([_s, continuation_monad_type[_r, _r]]), - TypeSequence([_s, _r]), - ), - ), - 'cont_pure': ForAll( - [_s, _a, _r], - StackEffect( - TypeSequence([_s, _a]), - TypeSequence([_s, continuation_monad_type[_r, _a]]), - ), - ), - 'bind_cont': ForAll( - [_s, _r, _a, _b], - StackEffect( - TypeSequence( - [ - _s, - continuation_monad_type[_r, _a], - ForAll( - [_t], - StackEffect( - TypeSequence([_t, _a]), - TypeSequence( - [_t, continuation_monad_type[_r, _b]] - ), - ), - ), - ] - ), - TypeSequence([_s, continuation_monad_type[_r, _b]]), - ), - ), - 'cont_from_cps': ForAll( - [_s, _t, _u, _a, _r], - StackEffect( - TypeSequence( - [ - _s, - ForAll( - [_t], - StackEffect( - TypeSequence( - [ - _t, - ForAll( - [_u], - StackEffect( - TypeSequence([_u, _a]), - TypeSequence([_u, _r]), - ), - ), - ] - ), - TypeSequence([_t, _r]), - ), - ), - ] - ), - TypeSequence([_s, continuation_monad_type[_r, _a]]), - ), - ), -} - def call_with_current_continuation( stack: List[object], stash: List[object] @@ -235,6 +129,21 @@ def cont_pure(stack: List[object], _: List[object]) -> None: stack.append(result) +def map_cont(stack: List[Any], stash: List[Any]) -> None: + f, cont = ( + cast(ConcatFunction, stack.pop()), + cast(ContinuationMonad, stack.pop()), + ) + + def python_function(b: _B) -> _C: + stack.append(b) + f(stack, stash) + return stack.pop() + + result = cont.map(f) + stack.append(result) + + def bind_cont(stack: List[object], stash: List[object]) -> None: f, cont = ( cast(ConcatFunction, stack.pop()), diff --git a/concat/stdlib/pyinterop/__init__.cati b/concat/stdlib/pyinterop/__init__.cati new file mode 100644 index 0000000..e13f030 --- /dev/null +++ b/concat/stdlib/pyinterop/__init__.cati @@ -0,0 +1,5 @@ +def getitem(*stack_type_var obj:subscriptable[`x, `y] index:`x -- *stack_type_var res:`y): + () + +def to_dict(*stack_type_var it:Optional[iterable[tuple[`x, `y]]] -- *stack_type_var d:dict[`x, `y]): + () diff --git a/concat/tests/test_typecheck.py b/concat/tests/test_typecheck.py index 4401dab..d4c7b70 100644 --- a/concat/tests/test_typecheck.py +++ b/concat/tests/test_typecheck.py @@ -62,7 +62,7 @@ def build_parsers() -> concat.parse.ParserDict: class TestTypeChecker(unittest.TestCase): @given(from_type(concat.parse.AttributeWordNode)) def test_attribute_word(self, attr_word) -> None: - _, type = concat.typecheck.infer( + _, type, _ = concat.typecheck.infer( concat.typecheck.Environment(), [attr_word], initial_stack=TypeSequence( @@ -84,7 +84,7 @@ def test_attribute_word(self, attr_word) -> None: def test_add_operator_inference(self, a: int, b: int) -> None: try_prog = '{!r} {!r} +\n'.format(a, b) tree = parse(try_prog) - _, type = concat.typecheck.infer( + _, type, _ = concat.typecheck.infer( concat.typecheck.Environment( {'+': concat.typecheck.preamble_types.types['+']} ), @@ -99,7 +99,7 @@ def test_add_operator_inference(self, a: int, b: int) -> None: def test_if_then_inference(self) -> None: try_prog = 'True $() if_then\n' tree = parse(try_prog) - _, type = concat.typecheck.infer( + _, type, _ = concat.typecheck.infer( concat.typecheck.Environment( concat.typecheck.preamble_types.types ), @@ -111,7 +111,7 @@ def test_if_then_inference(self) -> None: def test_call_inference(self) -> None: try_prog = '$(42) call\n' tree = parse(try_prog) - _, type = concat.typecheck.infer( + _, type, _ = concat.typecheck.infer( concat.typecheck.Environment( concat.typecheck.preamble_types.types ), @@ -124,7 +124,7 @@ def test_call_inference(self) -> None: @given(sampled_from(['None', '...', 'NotImplemented'])) def test_constants(self, constant_name) -> None: - _, effect = concat.typecheck.infer( + _, effect, _ = concat.typecheck.infer( concat.typecheck.Environment( concat.typecheck.preamble_types.types ), @@ -173,7 +173,7 @@ def seek_file(file:file offset:int whence:int --): def test_cast_word(self) -> None: """Test that the type checker properly checks casts.""" tree = parse('"str" cast (int)') - _, type = concat.typecheck.infer( + _, type, _ = concat.typecheck.infer( Environment(concat.typecheck.preamble_types.types), tree.children, is_top_level=True, diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index b9db76f..fd79ebe 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -8,7 +8,11 @@ import builtins import collections.abc from concat.lex import Token +import functools import importlib +import importlib.util +import itertools +import pathlib import sys from typing import ( Generator, @@ -184,13 +188,15 @@ def check( environment: Environment, program: concat.astutils.WordsOrStatements, source_dir: str = '.', -) -> None: + check_bodies: bool = True +) -> Environment: import concat.typecheck.preamble_types environment = Environment( {**concat.typecheck.preamble_types.types, **environment} ) - infer(environment, program, None, True, source_dir) + res = infer(environment, program, None, True, source_dir, check_bodies=check_bodies) + return res[2] # FIXME: I'm really passing around a bunch of state here. I could create an @@ -202,7 +208,8 @@ def infer( is_top_level=False, source_dir='.', initial_stack: Optional[TypeSequence] = None, -) -> Tuple[Substitutions, StackEffect]: + check_bodies: bool = True +) -> Tuple[Substitutions, StackEffect, Environment]: """The infer function described by Kleffner.""" e = list(e) current_subs = Substitutions() @@ -228,6 +235,10 @@ def infer( if isinstance(child, concat.parse.AttributeWordNode): top = o1[-1] attr_type = top.get_type_of_attribute(child.value) + if not isinstance(attr_type, IndividualType): + raise TypeError( + f'{attr_type} must be an individual type' + ) if should_instantiate: attr_type = attr_type.instantiate() rest_types = o1[:-1] @@ -242,6 +253,10 @@ def infer( if child.value not in gamma: raise NameError(child) name_type = gamma[child.value] + if not isinstance(name_type, IndividualType): + raise TypeError( + f'{name_type} must be an individual type' + ) if should_instantiate: name_type = name_type.instantiate() current_effect = StackEffect( @@ -257,12 +272,13 @@ def infer( # The majority of quotations I've written don't comsume # anything on the stack, so make that the default. input_stack = TypeSequence([SequenceVariable()]) - S2, fun_type = infer( + S2, fun_type, _ = infer( S1(gamma), child.children, extensions=extensions, source_dir=source_dir, initial_stack=input_stack, + check_bodies=check_bodies ) current_subs, current_effect = ( S2(S1), @@ -282,12 +298,13 @@ def infer( collected_type = o element_type: IndividualType = object_type for item in node.list_children: - phi1, fun_type = infer( + phi1, fun_type, _ = infer( phi(gamma), item, extensions=extensions, source_dir=source_dir, initial_stack=collected_type, + check_bodies=check_bodies ) collected_type = fun_type.output # FIXME: Infer the type of elements in the list based on @@ -314,12 +331,13 @@ def infer( collected_type = current_effect.output element_types: List[IndividualType] = [] for item in node.tuple_children: - phi1, fun_type = infer( + phi1, fun_type, _ = infer( phi(gamma), item, extensions=extensions, source_dir=source_dir, initial_stack=collected_type, + check_bodies=check_bodies ) collected_type = fun_type.output assert isinstance(collected_type[-1], IndividualType) @@ -343,32 +361,26 @@ def infer( ) elif isinstance(node, concat.parse.FromImportStatementNode): imported_name = node.asname or node.imported_name - # mutate type environment - gamma[imported_name] = object_type - # We will try to find a more specific type. - sys.path, old_path = [source_dir, *sys.path], sys.path - module = importlib.import_module(node.value) - sys.path = old_path + module_parts = node.value.split('.') + module_spec = None + path = None + for module_prefix in itertools.accumulate(module_parts, lambda a, b: f'{a}.{b}'): + for finder in sys.meta_path: + module_spec = finder.find_spec(module_prefix, path) + if module_spec is not None: + path = module_spec.submodule_search_locations + break + module_path = module_spec.origin # For now, assume the module's written in Python. - try: - # TODO: Support star imports - gamma[imported_name] = current_subs( - getattr(module, '@@types')[node.imported_name] - ) - except (KeyError, builtins.AttributeError): - # attempt introspection to get a more specific type - if callable(getattr(module, node.imported_name)): - args_var = SequenceVariable() - gamma[imported_name] = ObjectType( - IndividualVariable(), - { - '__call__': py_function_type[ - TypeSequence([args_var]), object_type - ], - }, - type_parameters=[args_var], - nominal=True, - ) + stub_path = pathlib.Path(module_path).with_suffix('.cati') + stub_env = _check_stub(stub_path) + imported_type = stub_env.get(node.imported_name) + if imported_type is None: + raise TypeError(f'Cannot find {node.imported_name} in module {node.value}') + # TODO: Support star imports + gamma[imported_name] = current_subs( + imported_type + ) elif isinstance(node, concat.parse.ImportStatementNode): # TODO: Support all types of import correctly. if node.asname is not None: @@ -399,32 +411,34 @@ def infer( declared_type = S(declared_type) recursion_env = gamma.copy() recursion_env[name] = declared_type.generalized_wrt(S(gamma)) - phi1, inferred_type = infer( - S(recursion_env), - node.body, - is_top_level=False, - extensions=extensions, - initial_stack=declared_type.input, - ) - # We want to check that the inferred outputs are subtypes of - # the declared outputs. Thus, inferred_type.output should be a subtype - # declared_type.output. - try: - S = inferred_type.output.constrain_and_bind_subtype_variables( - declared_type.output, - S(recursion_env).free_type_variables(), - [], - )( - S - ) - except TypeError: - message = ( - 'declared function type {} is not compatible with ' - 'inferred type {}' - ) - raise TypeError( - message.format(declared_type, inferred_type) + if check_bodies: + phi1, inferred_type, _ = infer( + S(recursion_env), + node.body, + is_top_level=False, + extensions=extensions, + initial_stack=declared_type.input, + check_bodies=check_bodies ) + # We want to check that the inferred outputs are subtypes of + # the declared outputs. Thus, inferred_type.output should be a subtype + # declared_type.output. + try: + S = inferred_type.output.constrain_and_bind_subtype_variables( + declared_type.output, + S(recursion_env).free_type_variables(), + [], + )( + S + ) + except TypeError: + message = ( + 'declared function type {} is not compatible with ' + 'inferred type {}' + ) + raise TypeError( + message.format(declared_type, inferred_type) + ) effect = declared_type # we *mutate* the type environment gamma[name] = effect.generalized_wrt(S(gamma)) @@ -466,12 +480,13 @@ def infer( )(S) else: input_stack = o - S1, (i1, o1) = infer( + S1, (i1, o1), _ = infer( gamma, [*quotation.children], extensions=extensions, source_dir=source_dir, initial_stack=input_stack, + check_bodies=check_bodies ) current_subs, current_effect = ( S1(S), @@ -517,14 +532,48 @@ def infer( current_effect.input, TypeSequence([SequenceVariable()]), ) + elif not check_bodies and isinstance(node, concat.parse.ClassdefStatementNode): + type_parameters = [] + temp_gamma = gamma + for param_node in node.type_parameters: + param, temp_gamma = param_node.to_type(temp_gamma) + type_parameters.append(param) + gamma[node.class_name] = ObjectType( + IndividualVariable(), + {}, + type_parameters, + (), + True, + ) else: raise UnhandledNodeTypeError( "don't know how to handle '{}'".format(node) ) - except TypeError as e: - e.set_location_if_missing(node.location) + except TypeError as error: + error.set_location_if_missing(node.location) raise - return current_subs, current_effect + return current_subs, current_effect, gamma + + +@functools.lru_cache(maxsize=None) +def _check_stub_resolved_path(path: pathlib.Path) -> 'Environment': + try: + source = path.read_text() + except FileNotFoundError as e: + raise TypeError(f'Type stubs at {path} do not exist') from e + except IOError as e: + raise TypeError(f'Failed to read type stubs at {path}') from e + tokens = concat.lex.tokenize(source) + from concat.transpile import parse + concat_ast = parse(tokens) + print(concat_ast) + print(list(concat_ast.parsing_failures)) + env = Environment() + return check(env, concat_ast.children, str(path.parent), check_bodies=False) + + +def _check_stub(path: pathlib.Path) -> 'Environment': + return _check_stub_resolved_path(path.resolve()) # Parsing type annotations @@ -601,6 +650,8 @@ def to_type(self, env: Environment) -> Tuple[IndividualType, Environment]: args.append(arg_as_type) generic_type, env = self._generic_type.to_type(env) if isinstance(generic_type, ObjectType): + if generic_type.is_variadic: + args = (TypeSequence(args), ) return generic_type[args], env raise TypeError('{} is not a generic type'.format(generic_type)) @@ -622,7 +673,12 @@ def to_type(self, env: Environment) -> Tuple[IndividualType, Environment]: if self._name is None: return self._type.to_type(env) elif self._type is None: - return env[self._name].to_type(env) + ty = env[self._name] + if not isinstance(ty, IndividualType): + raise TypeError( + f'an individual type was expected in this part of a type sequence, got {ty}' + ) + return ty, env elif self._name in env: raise TypeError( '{} is associated with a type more than once in this sequence of types'.format( @@ -657,16 +713,17 @@ def __init__( def to_type(self, env: Environment) -> Tuple[TypeSequence, Environment]: sequence: List[StackItemType] = [] + temp_env = env.copy() if self._sequence_variable is None: # implicit stack polymorphism sequence.append(SequenceVariable()) - elif self._sequence_variable.name not in env: - env = env.copy() + elif self._sequence_variable.name not in temp_env: + temp_env = temp_env.copy() var = SequenceVariable() - env[self._sequence_variable.name] = var + temp_env[self._sequence_variable.name] = var sequence.append(var) for type_node in self._individual_type_items: - type, env = type_node.to_type(env) + type, temp_env = type_node.to_type(temp_env) sequence.append(type) return TypeSequence(sequence), env @@ -708,6 +765,7 @@ def to_type(self, env: Environment) -> Tuple[StackEffect, Environment]: a_bar = SequenceVariable() b_bar = a_bar new_env = env.copy() + known_stack_item_names = Environment() if self.input_sequence_variable is not None: if self.input_sequence_variable.name in new_env: a_bar = cast( @@ -727,11 +785,21 @@ def to_type(self, env: Environment) -> Tuple[StackEffect, Environment]: in_types = [] for item in self.input: - type, new_env = _ensure_type(item[1], new_env, item[0]) + type, new_env = _ensure_type( + item[1], + new_env, + item[0], + known_stack_item_names, + ) in_types.append(type) out_types = [] for item in self.output: - type, new_env = _ensure_type(item[1], new_env, item[0]) + type, new_env = _ensure_type( + item[1], + new_env, + item[0], + known_stack_item_names, + ) out_types.append(type) return ( @@ -739,7 +807,7 @@ def to_type(self, env: Environment) -> Tuple[StackEffect, Environment]: TypeSequence([a_bar, *in_types]), TypeSequence([b_bar, *out_types]), ), - new_env, + env, ) @@ -974,7 +1042,11 @@ def forall_type_parser() -> Generator: return _ForallTypeNode(forall.start, type_variables, ty) - # TODO: Parse type variables + parsers['type-variable'] = concat.parser_combinators.alt( + sequence_type_variable_parser, + individual_type_variable_parser, + ) + parsers['nonparameterized-type'] = concat.parser_combinators.alt( named_type_parser.desc('named type'), parsers.ref_parser('stack-effect-type'), @@ -1079,11 +1151,14 @@ def _generate_module_type( def _ensure_type( - typename: Optional[TypeNode], env: Environment, obj_name: Optional[str], + typename: Optional[TypeNode], + env: Environment, + obj_name: Optional[str], + known_stack_item_names: Environment, ) -> Tuple[StackItemType, Environment]: type: StackItemType - if obj_name and obj_name in env: - type = cast(StackItemType, env[obj_name]) + if obj_name and obj_name in known_stack_item_names: + type = cast(StackItemType, known_stack_item_names[obj_name]) elif typename is None: # NOTE: This could lead type varibles in the output of a function that # are unconstrained. In other words, it would basically become an Any @@ -1091,14 +1166,15 @@ def _ensure_type( type = IndividualVariable() elif isinstance(typename, TypeNode): type, env = cast( - Tuple[StackItemType, Environment], typename.to_type(env) + Tuple[StackItemType, Environment], + typename.to_type(env), ) else: raise NotImplementedError( 'Cannot turn {!r} into a type'.format(typename) ) if obj_name: - env[obj_name] = type + known_stack_item_names[obj_name] = type return type, env diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 2270948..07390bd 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -127,6 +127,16 @@ def is_subtype_of(self, supertype: Type) -> bool: return False return super().is_subtype_of(supertype) + def instantiate(self) -> 'IndividualType': + return cast(IndividualType, super().instantiate()) + + @abc.abstractmethod + def apply_substitution( + self, + sub: 'concat.typecheck.Substitutions', + ) -> 'IndividualType': + pass + class _Variable(Type, abc.ABC): """Objects that represent type variables. @@ -906,7 +916,7 @@ def __init__( self, self_type: IndividualVariable, # Attributes can be universally quantified since ObjectType allows it. - attributes: Dict[str, IndividualType], + attributes: Mapping[str, IndividualType], type_parameters: Sequence[_Variable] = (), nominal_supertypes: Sequence[IndividualType] = (), nominal: bool = False, @@ -937,6 +947,7 @@ def __init__( del self._other_kwargs['_type_arguments'] if 'nominal' in self._other_kwargs: del self._other_kwargs['nominal'] + self.is_variadic = bool(self._other_kwargs.get('is_variadic')) self._instantiations: Dict[TypeArguments, ObjectType] = {} @@ -1312,7 +1323,8 @@ def __hash__(self) -> int: ) def __getitem__( - self, type_arguments: Sequence[StackItemType] + self, + type_arguments: TypeArguments, ) -> 'ObjectType': from concat.typecheck import Substitutions @@ -1955,7 +1967,8 @@ def _mapping_to_str(mapping: Mapping) -> str: _x, {'__getitem__': py_function_type}, [_element_types_var], - nominal=True + nominal=True, + is_variadic=True, # iterable_type is a structural supertype ) tuple_type.set_internal_name('tuple_type') From 5e30a720989f2d629782fbc740446a776b4d0da8 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Fri, 22 Mar 2024 17:50:10 -0700 Subject: [PATCH 03/61] Add snakeviz to dev dependencies --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 991ce2e..e6333aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,10 +27,11 @@ readme = "README.md" [project.optional-dependencies] dev = [ "axblack==20220330", + "hypothesis>=6.75.1,<7", "mypy>=1.1.1", "pre-commit>=2.6.0,<3", + "snakeviz", "tox>=4.5.1,<5", - "hypothesis>=6.75.1,<7" ] From 6ca2a859204102ca71acb3512ff52fad9a1d949c Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Sun, 24 Mar 2024 23:13:35 -0700 Subject: [PATCH 04/61] Back OrderedSet by 2-3 tree Hopefully increased sharing speeds things up. --- concat/orderedset.py | 466 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 454 insertions(+), 12 deletions(-) diff --git a/concat/orderedset.py b/concat/orderedset.py index dae23a5..7a3fd5c 100644 --- a/concat/orderedset.py +++ b/concat/orderedset.py @@ -1,6 +1,5 @@ -from itertools import repeat -from typing import AbstractSet, Iterable, Iterator, TypeVar - +import itertools +from typing import AbstractSet, Any, Iterable, Iterator, Tuple, TypeVar _T = TypeVar('_T', covariant=True) @@ -8,26 +7,469 @@ class OrderedSet(AbstractSet[_T]): def __init__(self, elements: Iterable[_T]) -> None: super().__init__() - # In Python 3.7+, dicts are guaranteed to use insertion order. - self._dictionary = dict(zip(elements, repeat(None))) + self._data = _Tree23.from_iterable(elements) def __sub__(self, other: object) -> 'OrderedSet[_T]': if not isinstance(other, AbstractSet): return NotImplemented - return OrderedSet( - element for element in self._dictionary if element not in other - ) + data = self._data + for el in other: + data = data.delete(el) + return OrderedSet(data) def __or__(self, other: object) -> 'OrderedSet[_T]': if not isinstance(other, AbstractSet): return NotImplemented - return OrderedSet([*self, *other]) + data = self._data + for el in other: + data = data.insert(el) + return OrderedSet(data) def __contains__(self, element: object) -> bool: - return element in self._dictionary + return element in self._data def __iter__(self) -> Iterator[_T]: - return iter(self._dictionary) + return iter(self._data) + + def __len__(self) -> int: + return len(self._data) + + +class _Tree23Hole: + pass + + +class _Tree23: + """Implementation based on https://www.cs.princeton.edu/~dpw/courses/cos326-12/ass/2-3-trees.pdf. + + Kinds of nodes: + Leaf: () + 2-node: (l, X, r) + 3-node: (l, X, m, Y, r) + + Intermediary nodes: + Kicked up node: ('up', a, w, b) + Hole node: (__hole, l)""" + + def __init__(self, data: tuple) -> None: + self._data = data + if self.is_leaf(): + self.height = 0 + elif self.is_2_node(): + # assert not isinstance(data[0], _Tree23Hole) and not isinstance( + # data[2], _Tree23Hole) + self.height = max(data[0].height, data[2].height) + 1 + elif self.is_3_node(): + # assert not isinstance(data[0], _Tree23Hole) and not isinstance( + # data[2], _Tree23Hole) and not isinstance(data[4], _Tree23Hole) + self.height = ( + max(data[0].height, data[2].height, data[4].height) + 1 + ) + elif self._is_kicked_up_node(): + assert not isinstance(data[2], _Tree23Hole) and not isinstance( + data[3], _Tree23Hole + ) + self.height = max(data[1].height, data[3].height) + elif self._is_hole_node(): + self.height = data[1].height + 1 + else: + raise ValueError('Invalid 2-3 tree') + + @classmethod + def from_iterable(cls, i: Iterable) -> '_Tree23': + if isinstance(i, cls): + return i + tree = _leaf_23_tree + for el in i: + tree = tree.insert(el) + return tree + + def search(self, d) -> Tuple[bool, Any]: + t = self + while True: + data = t._data + if t.is_leaf(): + return (False, None) + if t.is_2_node(): + p, a, q = data + if d < a: + t = p + elif d == a: + return (True, a) + else: + t = q + elif t.is_3_node(): + p, a, q, b, r = data + if d < a: + t = p + elif d == a: + return (True, a) + elif d < b: + t = q + elif d == b: + return (True, b) + else: + t = r + + def __iter__(self) -> Iterator: + if self.is_leaf(): + return + if self.is_2_node(): + yield from self._data[0] + yield self._data[1] + yield from self._data[2] + if self.is_3_node(): + yield self._data[3] + yield from self._data[4] def __len__(self) -> int: - return len(self._dictionary) + if self.is_leaf(): + return 0 + if self.is_2_node(): + return len(self._data[0]) + 1 + len(self._data[2]) + if self.is_3_node(): + return ( + len(self._data[0]) + + 1 + + len(self._data[2]) + + 1 + + len(self._data[4]) + ) + raise ValueError('Invalid 2-3 tree') + + def _insert(self, key) -> '_Tree23': + data = self._data + if self.is_leaf(): + return _Tree23(('up', _leaf_23_tree, key, _leaf_23_tree)) + if self.is_2_node(): + p, a, q = data + # print(key, a) + if key < a: + p_ = p._insert(key) + return _Tree23((p_, a, q))._insert_upwards_phase() + if key == a: + return _Tree23((p, key, q)) + q_ = q._insert(key) + return _Tree23((p, a, q_))._insert_upwards_phase() + if self.is_3_node(): + l, X, m, Y, r = data + if key < X: + return _Tree23( + (l._insert(key), X, m, Y, r) + )._insert_upwards_phase() + if key == X: + return _Tree23((l, key, m, Y, r)) + if key < Y: + return _Tree23( + (l, X, m._insert(key), Y, r) + )._insert_upwards_phase() + if key == Y: + return _Tree23((l, X, m, key, r)) + return _Tree23( + (l, X, m, Y, r._insert(key)) + )._insert_upwards_phase() + raise ValueError('Invalid 2-3 tree') + + def _insert_upwards_phase(self) -> '_Tree23': + if self.is_2_node(): + q, X, r = self._data + if q._is_kicked_up_node(): + # 2-node upstairs, kicked up node on left + _, l, w, m = q._data + return _Tree23((l, w, m, X, r)) + if r._is_kicked_up_node(): + # 2-node upstairs, kicked up node on right + _, m, w, r = r._data + l = q + return _Tree23((l, X, m, w, r)) + return self + if self.is_3_node(): + a, X, c, Y, d = self._data + if a._is_kicked_up_node(): + _, a, w, b = a._data + return _Tree23( + ('up', _Tree23((a, w, b)), X, _Tree23((c, Y, d))) + ) + if c._is_kicked_up_node(): + _, b, w, c = c._data + return _Tree23( + ('up', _Tree23((a, X, b)), w, _Tree23((c, Y, d))) + ) + if d._is_kicked_up_node(): + b = c + _, c, w, d = d._data + return _Tree23( + ('up', _Tree23((a, X, b)), Y, _Tree23((c, w, d))) + ) + return self + + def insert(self, key) -> '_Tree23': + """Insert key into the tree and return a new tree. + + This method should only be called on the root of a tree.""" + + tree = self._insert(key) + if tree._is_kicked_up_node(): + return _Tree23(tree._data[1:]) + return tree + + def _delete(self, key) -> '_Tree23': + data = self._data + if self.is_leaf(): + return _leaf_23_tree + if self.is_2_node(): + p, a, q = data + if key < a: + p_ = p._delete(key) + return _Tree23((p_, a, q))._delete_upwards_phase() + if key == a: + if self._is_2_node_terminal(): + return _Tree23((p, self.__hole, q))._delete_upwards_phase() + pred = p.max() + return _Tree23( + (p._delete(pred), pred, q) + )._delete_upwards_phase() + q_ = q._delete(key) + return _Tree23((p, a, q_))._delete_upwards_phase() + if self.is_3_node(): + l, X, m, Y, r = data + if key < X: + return _Tree23( + (l._delete(key), X, m, Y, r) + )._delete_upwards_phase() + if key == X: + if self._is_3_node_terminal(): + return _Tree23( + (l, self.__hole, m, Y, r) + )._delete_upwards_phase() + pred = l.max() + return _Tree23( + (l._delete(pred), pred, m, Y, r) + )._delete_upwards_phase() + if key < Y: + return _Tree23( + (l, X, m._delete(key), Y, r) + )._delete_upwards_phase() + if key == Y: + if self._is_3_node_terminal(): + return _Tree23( + (l, X, m, self.__hole, r) + )._delete_upwards_phase() + pred = m.max() + return _Tree23( + (l, X, m._delete(pred), pred, r) + )._delete_upwards_phase() + return _Tree23( + (l, X, m, Y, r._delete(key)) + )._delete_upwards_phase() + raise ValueError('Invalid 2-3 tree') + + def _delete_upwards_phase(self) -> '_Tree23': + if self.is_3_node(): + w, x, alpha, y, d = self._data + if self._is_3_node_terminal(): + if x is self.__hole: + return _Tree23((_leaf_23_tree, y, _leaf_23_tree)) + if y is self.__hole: + return _Tree23((_leaf_23_tree, x, _leaf_23_tree)) + if w._is_hole_node(): + a = w._data[1] + if alpha.is_2_node(): + z = y + b, y, c = alpha._data + if all( + map( + lambda h: h == d.height - 1, + (a.height, b.height, c.height), + ) + ): + # 3-node parent, 2-node sibling, hole on left, height condition + return _Tree23((_Tree23((a, x, b, y, c)), z, d)) + if alpha.is_3_node(): + w = x + z = y + e = d + b, x, c, y, d = alpha._data + if all( + map( + lambda h: h == e.height - 1, + (a.height, b.height, c.height, d.height), + ) + ): + # 3-node parent, 3-parent sibling in middle, hole on left, right condition + return _Tree23( + (_Tree23((a, w, b)), x, _Tree23((c, y, d)), z, e) + ) + if alpha._is_hole_node(): + if w.is_2_node(): + z = y + y = x + a, x, b = w._data + c = alpha._data[1] + if all( + map( + lambda h: h == d.height - 1, + (a.height, b.height, c.height), + ) + ): + # 3-node parent, 2-node sibling on left, hole in middle, height condition + return _Tree23((_Tree23((a, x, b, y, c)), z, d)) + if w.is_3_node(): + z = y + y = x + a, w, b, x, c = w._data + e = d + d = alpha._data[1] + if all( + map( + lambda h: h == e.height - 1, + (a.height, b.height, c.height, d.height), + ) + ): + # 3-node parent, 3-node sibling on left, hole in middle, height condition + return _Tree23( + (_Tree23((a, w, b)), x, _Tree23((c, y, d)), z, e) + ) + if d.is_2_node(): + a = w + b = alpha._data[1] + c, z, d = d._data + if all( + map( + lambda h: h == a.height - 1, + (b.height, c.height, d.height), + ) + ): + return _Tree23((a, x, _Tree23((b, y, c, z, d)))) + if d.is_3_node(): + a = w + w = x + b = alpha._data[1] + x = y + c, y, d, z, e = d._data + if all( + map( + lambda h: h == a.height - 1, + (b.height, c.height, d.height, e.height), + ) + ): + # 3-node parent, 3-node sibling on right, hole in middle, height condition + return _Tree23( + (a, w, _Tree23((b, x, c)), y, _Tree23((d, z, e))) + ) + if d._is_hole_node(): + a = w + if alpha.is_2_node(): + z = y + b, y, c = alpha._data + d = d._data[1] + if all( + map( + lambda h: h == a.height - 1, + (b.height, c.height, d.height), + ) + ): + # 3-node parent, 2-node sibling in middle, hole on right, height condition + return _Tree23((a, x, _Tree23((b, y, c, z, d)))) + if alpha.is_3_node(): + w = x + z = y + e = d._data[1] + b, x, c, y, d = alpha._data + if all( + map( + lambda h: h == a.height - 1, + (b.height, c.height, d.height, e.height), + ) + ): + # 3-node parent, 3-node sibling in middle, hole on right, height condition + return _Tree23( + (a, w, _Tree23((b, x, c)), y, _Tree23((d, z, e))) + ) + # 3-node that either has no in data or children, or has bad heights + return self + if self.is_2_node(): + left, x, right = self._data + if self._is_2_node_terminal(): + if x is self.__hole: + return _Tree23((x, _leaf_23_tree)) + if left._is_hole_node(): + if right.is_2_node(): + # 2-node parent, 2-node sibling, hole on left + l = left._data[1] + m, y, r = right._data + return _Tree23((self.__hole, _Tree23((l, x, m, y, r)))) + if right.is_3_node(): + # 2-node parent, 3-node sibling, hole on left + a = left._data[1] + b, y, c, z, d = right._data + return _Tree23((_Tree23((a, x, b)), y, _Tree23((c, z, d)))) + if right._is_hole_node(): + if left.is_2_node(): + # 2-node parent, 2-node sibling, hole on right + r = right._data[1] + y = x + l, x, m = left._data + return _Tree23((self.__hole, _Tree23((l, x, m, y, r)))) + if left.is_3_node(): + # 2-node parent, 3-node sibling, hole on right + z = x + d = right._data[1] + a, x, b, y, c = w._data + return _Tree23((_Tree23((a, x, b)), y, _Tree23((c, z, d)))) + # no hole in key or children + return self + raise RuntimeError(f'Missing case in delete for {self!r}') + + def delete(self, key) -> '_Tree23': + tree = self._delete(key) + if tree._is_hole_node(): + return tree._data[1] + return tree + + def max(self) -> Any: + tree = self + while not tree.is_leaf(): + if tree.is_2_node(): + if tree._is_2_node_terminal(): + return tree._data[1] + tree = tree._data[2] + if tree.is_3_node(): + if tree._is_3_node_terminal(): + return tree._data[3] + tree = tree._data[4] + raise ValueError('Empty 2-3 tree has no max') + + __hole = _Tree23Hole() + + def is_leaf(self) -> bool: + return len(self._data) == 0 + + def is_2_node(self) -> bool: + return len(self._data) == 3 + + def is_3_node(self) -> bool: + return len(self._data) == 5 + + def _is_kicked_up_node(self) -> bool: + return len(self._data) == 4 and self._data[0] == 'up' + + def _is_3_node_terminal(self) -> bool: + return all( + map( + lambda t: t.is_leaf(), + (self._data[0], self._data[2], self._data[4]), + ) + ) + + def _is_2_node_terminal(self) -> bool: + return all(map(lambda t: t.is_leaf(), (self._data[0], self._data[2]))) + + def _is_hole_node(self) -> bool: + return len(self._data) == 2 and self._data[0] is self.__hole + + def __repr__(self) -> str: + return f'{type(self).__qualname__}({self._data!r})' + + +_leaf_23_tree = _Tree23(()) From 50f18b4555b078227d645460df4618a370b3c92d Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Tue, 26 Mar 2024 00:40:31 -0700 Subject: [PATCH 05/61] Move error message formatting to new module --- concat/__main__.py | 36 +++++++++++------------------------- concat/error_reporting.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 25 deletions(-) create mode 100644 concat/error_reporting.py diff --git a/concat/__main__.py b/concat/__main__.py index ffbb8f5..221ad15 100644 --- a/concat/__main__.py +++ b/concat/__main__.py @@ -4,6 +4,7 @@ import argparse from concat.transpile import parse, transpile_ast, typecheck import concat.astutils +from concat.error_reporting import get_line_at, create_parsing_failure_message import concat.execute import concat.lex import concat.parser_combinators @@ -31,29 +32,6 @@ def func(name: str) -> IO[AnyStr]: return func -def get_line_at(file: TextIO, location: concat.astutils.Location) -> str: - file.seek(0, io.SEEK_SET) - lines = [*file] - return lines[location[0] - 1] - - -def create_parsing_failure_message( - file: TextIO, - stream: Sequence[concat.lex.Token], - failure: concat.parser_combinators.FailureTree, -) -> str: - location = stream[failure.furthest_index].start - line = get_line_at(file, location) - message = f'Expected {failure.expected} at line {location[0]}, column {location[1] + 1}:\n{line.rstrip()}\n{" " * location[1] + "^"}' - if failure.children: - message += '\nbecause:' - for f in failure.children: - message += '\n' + textwrap.indent( - create_parsing_failure_message(file, stream, f), ' ' - ) - return message - - arg_parser = argparse.ArgumentParser(description='Run a Concat program.') arg_parser.add_argument( 'file', @@ -103,10 +81,18 @@ def create_parsing_failure_message( typecheck(concat_ast, source_dir) python_ast = transpile_ast(concat_ast) except concat.typecheck.StaticAnalysisError as e: - print('Static Analysis Error:\n') + if e.path is None: + in_path = '' + else: + in_path = ' in file ' + str(e.path) + print(f'Static Analysis Error{in_path}:\n') print(e, 'in line:') if e.location: - print(get_line_at(args.file, e.location), end='') + if e.path is not None: + with e.path.open() as f: + print(get_line_at(f, e.location), end='') + else: + print(get_line_at(args.file, e.location), end='') print(' ' * e.location[1] + '^') except concat.parser_combinators.ParseError as e: print('Parse Error:') diff --git a/concat/error_reporting.py b/concat/error_reporting.py new file mode 100644 index 0000000..f44b75f --- /dev/null +++ b/concat/error_reporting.py @@ -0,0 +1,28 @@ +import concat.astutils +import concat.parser_combinators +import io +import textwrap +from typing import Sequence, TextIO + + +def get_line_at(file: TextIO, location: concat.astutils.Location) -> str: + file.seek(0, io.SEEK_SET) + lines = [*file] + return lines[location[0] - 1] + + +def create_parsing_failure_message( + file: TextIO, + stream: Sequence[concat.lex.Token], + failure: concat.parser_combinators.FailureTree, +) -> str: + location = stream[failure.furthest_index].start + line = get_line_at(file, location) + message = f'Expected {failure.expected} at line {location[0]}, column {location[1] + 1}:\n{line.rstrip()}\n{" " * location[1] + "^"}' + if failure.children: + message += '\nbecause:' + for f in failure.children: + message += '\n' + textwrap.indent( + create_parsing_failure_message(file, stream, f), ' ' + ) + return message From 7a72b3ebebf149f5c4f487c1a9a8ea7c04d182eb Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Tue, 26 Mar 2024 18:47:31 -0700 Subject: [PATCH 06/61] Add type stubs for Python builtins --- concat/examples/symmetric_tree.cat | 1 - concat/orderedset.py | 3 +- concat/parse.py | 81 ++++-- concat/typecheck/__init__.py | 446 ++++++++++++++++++++--------- concat/typecheck/preamble_types.py | 17 +- concat/typecheck/py_builtins.cati | 94 ++++++ concat/typecheck/types.py | 396 ++++++++++++++++++++++--- 7 files changed, 832 insertions(+), 206 deletions(-) create mode 100644 concat/typecheck/py_builtins.cati diff --git a/concat/examples/symmetric_tree.cat b/concat/examples/symmetric_tree.cat index 5adf33e..6e216cb 100644 --- a/concat/examples/symmetric_tree.cat +++ b/concat/examples/symmetric_tree.cat @@ -6,7 +6,6 @@ # Tree input format: [, , ] # None is the empty tree -from builtins import eval from concat.stdlib.pyinterop import getitem from concat.stdlib.pyinterop import to_str from concat.stdlib.pyinterop import to_dict diff --git a/concat/orderedset.py b/concat/orderedset.py index 7a3fd5c..4a36d8b 100644 --- a/concat/orderedset.py +++ b/concat/orderedset.py @@ -1,4 +1,3 @@ -import itertools from typing import AbstractSet, Any, Iterable, Iterator, Tuple, TypeVar _T = TypeVar('_T', covariant=True) @@ -415,7 +414,7 @@ def _delete_upwards_phase(self) -> '_Tree23': # 2-node parent, 3-node sibling, hole on right z = x d = right._data[1] - a, x, b, y, c = w._data + a, x, b, y, c = left._data return _Tree23((_Tree23((a, x, b)), y, _Tree23((c, z, d)))) # no hole in key or children return self diff --git a/concat/parse.py b/concat/parse.py index 95d47aa..74578d6 100644 --- a/concat/parse.py +++ b/concat/parse.py @@ -29,19 +29,20 @@ def word_ext(parsers): import abc import operator from typing import ( - Iterable, - Iterator, - Optional, - Type, - TypeVar, Any, - Sequence, - Tuple, + Callable, Dict, Generator, + Iterable, + Iterator, List, - Callable, + Optional, + Sequence, TYPE_CHECKING, + Tuple, + Type, + TypeVar, + Union, ) import concat.lex import concat.astutils @@ -212,6 +213,9 @@ def __str__(self) -> str: ) return '(' + input_stack_type + ' '.join(map(str, self.children)) + ')' + def __repr__(self) -> str: + return f'{type(self).__qualname__}(children={self.children!r}, location={self.location!r}, input_stack_type={self.input_stack_type!r})' + class NameWordNode(WordNode): def __init__(self, name: 'concat.lex.Token'): @@ -260,6 +264,9 @@ def parsing_failures( assert self.result.failures is not None yield self.result.failures + def __repr__(self) -> str: + return f'{type(self).__qualname__}(result={self.result!r})' + class BytesWordNode(WordNode): def __init__(self, bytes: 'concat.lex.Token'): @@ -353,6 +360,7 @@ def __init__( bases: Iterable['Words'] = (), keyword_args: Iterable[Tuple[str, WordNode]] = (), type_parameters: Iterable[Node] = (), + is_variadic: bool = False, ): super().__init__() self.location = location @@ -362,6 +370,7 @@ def __init__( self.bases = bases self.keyword_args = keyword_args self.type_parameters = type_parameters + self.is_variadic = is_variadic def token(typ: str) -> concat.parser_combinators.Parser: @@ -594,7 +603,7 @@ def suite(): statement = concat.parser_combinators.seq(parsers['statement']) block_content = ( parsers['word'] << token('NEWLINE').optional() - | parsers['statement'] << token('NEWLINE') + | parsers['statement'] << token('NEWLINE').optional() ).at_least(1) indented_block = token('NEWLINE').optional() >> bracketed( token('INDENT'), block_content, token('DEDENT') @@ -661,19 +670,56 @@ def classdef_statement_parser(): """This parses a class definition statement. classdef statement = CLASS, NAME, - [ LSQB, type variable, (COMMA, type variable)*, [ COMMA ], RSQB ], + [ LSQB, ((type variable, (COMMA, type variable)*, [ COMMA ]) | (type variable, NAME=...)), RSQB) ], decorator*, [ bases ], keyword arg*, COLON, suite ; bases = tuple word ; keyword arg = NAME, EQUAL, word ;""" location = (yield token('CLASS')).start name_token = yield token('NAME') - type_parameters = yield bracketed( - token('LSQB'), - parsers['type-variable'].sep_by(token('COMMA')) - << token('COMMA').optional(), - token('RSQB'), - ).optional() + is_variadic = False + + def ellispis_verify( + tok: concat.lex.Token, + ) -> concat.parser_combinators.Parser[concat.lex.Token, Any]: + nonlocal is_variadic + + if tok.value == '...': + is_variadic = True + return concat.parser_combinators.success(None) + return concat.parser_combinators.fail('a literal ellispis (...)') + + def handle_recovery( + x: Union[ + Sequence[Node], + Tuple[Any, concat.parser_combinators.Result[Any]], + ] + ) -> Sequence[Node]: + if ( + isinstance(x, tuple) + and len(x) > 1 + and isinstance(x[1], concat.parser_combinators.Result) + ): + return [ParseError(x[1])] + return x + + ellispis_parser = token('NAME').bind(ellispis_verify) + type_parameters = ( + yield bracketed( + token('LSQB'), + ( + concat.parser_combinators.seq(parsers['type-variable']) + << ellispis_parser + ) + | ( + parsers['type-variable'].sep_by(token('COMMA')) + << token('COMMA').optional() + ), + token('RSQB'), + ) + .map(handle_recovery) + .optional() + ) decorators = yield decorator.many() bases_list = yield bases.optional() keyword_args = yield keyword_arg.map(tuple).many() @@ -686,7 +732,8 @@ def classdef_statement_parser(): decorators, bases_list, keyword_args, - type_parameters=type_parameters or [] + type_parameters=type_parameters or [], + is_variadic=is_variadic, ) parsers['classdef-statement'] = classdef_statement_parser diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index fd79ebe..eec3c51 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -4,33 +4,129 @@ "Robert Kleffner: A Foundation for Typed Concatenative Languages, April 2017." """ -import abc -import builtins + import collections.abc -from concat.lex import Token -import functools -import importlib -import importlib.util -import itertools -import pathlib -import sys from typing import ( + Any, + Callable, + Dict, Generator, Iterable, + Iterator, List, - Set, - Tuple, - Dict, - Union, + Mapping, Optional, - Callable, Sequence, - TypeVar, + Set, TYPE_CHECKING, - overload, + Tuple, + TypeVar, + Union, cast, + overload, ) from typing_extensions import Protocol + + +_Result = TypeVar('_Result', covariant=True) + + +class _Substitutable(Protocol[_Result]): + def apply_substitution(self, sub: 'Substitutions') -> _Result: + pass + + +class Substitutions(collections.abc.Mapping, Mapping['_Variable', 'Type']): + def __init__(self, sub: Iterable[Tuple['_Variable', 'Type']] = {}) -> None: + self._sub = dict(sub) + self._by_identity_cache: Dict[int, Any] = {} + + def __getitem__(self, var: '_Variable') -> 'Type': + return self._sub[var] + + def __iter__(self) -> Iterator['_Variable']: + return iter(self._sub) + + def __len__(self) -> int: + return len(self._sub) + + def __call__(self, arg: _Substitutable[_Result]) -> _Result: + # if id(arg) not in self._by_identity_cache: + # self._by_identity_cache[id(arg)] = arg.apply_substitution(self) + # return self._by_identity_cache[id(arg)] + # Using the id is wrong because a different object can have the same id later + return arg.apply_substitution(self) + + def _dom(self) -> Set['_Variable']: + return {*self} + + def __str__(self) -> str: + return ( + '{' + + ',\n'.join( + map(lambda i: '{}: {}'.format(i[0], i[1]), self.items()) + ) + + '}' + ) + + def apply_substitution(self, sub: 'Substitutions') -> 'Substitutions': + return Substitutions( + { + **sub, + **{a: sub(i) for a, i in self.items() if a not in sub._dom()}, + } + ) + + def __hash__(self) -> int: + return hash(tuple(self.items())) + + +from concat.typecheck.types import ( + ForwardTypeReference, + ForAll, + GenericTypeKind, + IndividualKind, + IndividualType, + IndividualVariable, + ObjectType, + PythonFunctionType, + QuotationType, + SequenceKind, + SequenceVariable, + StackEffect, + StackItemType, + Type, + TypeSequence, + bool_type, + context_manager_type, + ellipsis_type, + free_type_variables_of_mapping, + init_primitives, + int_type, + invertible_type, + iterable_type, + list_type, + module_type, + none_type, + not_implemented_type, + object_type, + py_function_type, + slice_type, + str_type, + subscriptable_type, + subtractable_type, + tuple_type, +) +import abc +import builtins +from concat.error_reporting import create_parsing_failure_message +from concat.lex import Token +import functools +import importlib +import importlib.util +import itertools +import pathlib +import sys import concat.parser_combinators import concat.parse @@ -45,6 +141,7 @@ class StaticAnalysisError(Exception): def __init__(self, message: str) -> None: self.message = message self.location: Optional['concat.astutils.Location'] = None + self.path: Optional[pathlib.Path] = None def set_location_if_missing( self, location: 'concat.astutils.Location' @@ -52,6 +149,10 @@ def set_location_if_missing( if not self.location: self.location = location + def set_path_if_missing(self, path: pathlib.Path) -> None: + if self.path is None: + self.path = path + def __str__(self) -> str: return '{} at {}'.format(self.message, self.location) @@ -106,73 +207,6 @@ class UnhandledNodeTypeError(builtins.NotImplementedError): pass -_Result = TypeVar('_Result', covariant=True) - - -class _Substitutable(Protocol[_Result]): - def apply_substitution(self, sub: 'Substitutions') -> _Result: - pass - - -class Substitutions(Dict['_Variable', 'Type']): - def __call__(self, arg: _Substitutable[_Result]) -> _Result: - return arg.apply_substitution(self) - - def _dom(self) -> Set['_Variable']: - return {*self} - - def __str__(self) -> str: - return ( - '{' - + ',\n'.join( - map(lambda i: '{}: {}'.format(i[0], i[1]), self.items()) - ) - + '}' - ) - - def apply_substitution(self, sub: 'Substitutions') -> 'Substitutions': - return Substitutions( - { - **sub, - **{a: sub(i) for a, i in self.items() if a not in sub._dom()}, - } - ) - - -from concat.typecheck.types import ( - Type, - IndividualVariable, - StackEffect, - ForAll, - IndividualType, - ObjectType, - PythonFunctionType, - SequenceVariable, - TypeSequence, - StackItemType, - QuotationType, - bool_type, - context_manager_type, - ellipsis_type, - free_type_variables_of_mapping, - int_type, - init_primitives, - invertible_type, - iterable_type, - list_type, - module_type, - none_type, - not_implemented_type, - object_type, - py_function_type, - slice_type, - str_type, - subscriptable_type, - subtractable_type, - tuple_type, -) - - class Environment(Dict[str, Type]): def copy(self) -> 'Environment': return Environment(super().copy()) @@ -183,19 +217,43 @@ def apply_substitution(self, sub: 'Substitutions') -> 'Environment': def free_type_variables(self) -> 'OrderedSet[_Variable]': return free_type_variables_of_mapping(self) + def resolve_forward_references(self) -> None: + for name, t in self.items(): + self[name] = t.resolve_forward_references() + def check( environment: Environment, program: concat.astutils.WordsOrStatements, source_dir: str = '.', - check_bodies: bool = True + _should_check_bodies: bool = True, + _should_load_builtins: bool = True, ) -> Environment: import concat.typecheck.preamble_types + if _should_load_builtins: + builtins_stub_env = _check_stub( + pathlib.Path(__file__).with_name('py_builtins.cati'), + is_builtins=True, + ) + else: + builtins_stub_env = Environment() environment = Environment( - {**concat.typecheck.preamble_types.types, **environment} + { + **builtins_stub_env, + **concat.typecheck.preamble_types.types, + **environment, + } + ) + res = infer( + environment, + program, + None, + True, + source_dir, + check_bodies=_should_check_bodies, ) - res = infer(environment, program, None, True, source_dir, check_bodies=check_bodies) + # res[2].resolve_forward_references() return res[2] @@ -208,7 +266,7 @@ def infer( is_top_level=False, source_dir='.', initial_stack: Optional[TypeSequence] = None, - check_bodies: bool = True + check_bodies: bool = True, ) -> Tuple[Substitutions, StackEffect, Environment]: """The infer function described by Kleffner.""" e = list(e) @@ -219,6 +277,26 @@ def infer( ) current_effect = StackEffect(initial_stack, initial_stack) + # Prepare for forward references. + # TODO: Do this in a more principled way with scope graphs. + gamma = gamma.copy() + for node in e: + if isinstance(node, concat.parse.ClassdefStatementNode): + type_name = node.class_name + kind = IndividualKind() + type_parameters = [] + if node.is_variadic: + parameter_kinds = [SequenceKind()] + else: + for param in node.type_parameters: + type_parameters.append(param.to_type(gamma)[0]) + if type_parameters: + parameter_kinds = [ + variable.kind for variable in type_parameters + ] + kind = GenericTypeKind(parameter_kinds) + gamma[type_name] = ForwardTypeReference(kind, type_name, gamma) + for node in e: try: S, (i, o) = current_subs, current_effect @@ -253,6 +331,8 @@ def infer( if child.value not in gamma: raise NameError(child) name_type = gamma[child.value] + if isinstance(name_type, ForwardTypeReference): + raise NameError(child) if not isinstance(name_type, IndividualType): raise TypeError( f'{name_type} must be an individual type' @@ -278,7 +358,7 @@ def infer( extensions=extensions, source_dir=source_dir, initial_stack=input_stack, - check_bodies=check_bodies + check_bodies=check_bodies, ) current_subs, current_effect = ( S2(S1), @@ -304,7 +384,7 @@ def infer( extensions=extensions, source_dir=source_dir, initial_stack=collected_type, - check_bodies=check_bodies + check_bodies=check_bodies, ) collected_type = fun_type.output # FIXME: Infer the type of elements in the list based on @@ -337,7 +417,7 @@ def infer( extensions=extensions, source_dir=source_dir, initial_stack=collected_type, - check_bodies=check_bodies + check_bodies=check_bodies, ) collected_type = fun_type.output assert isinstance(collected_type[-1], IndividualType) @@ -364,7 +444,9 @@ def infer( module_parts = node.value.split('.') module_spec = None path = None - for module_prefix in itertools.accumulate(module_parts, lambda a, b: f'{a}.{b}'): + for module_prefix in itertools.accumulate( + module_parts, lambda a, b: f'{a}.{b}' + ): for finder in sys.meta_path: module_spec = finder.find_spec(module_prefix, path) if module_spec is not None: @@ -376,11 +458,11 @@ def infer( stub_env = _check_stub(stub_path) imported_type = stub_env.get(node.imported_name) if imported_type is None: - raise TypeError(f'Cannot find {node.imported_name} in module {node.value}') + raise TypeError( + f'Cannot find {node.imported_name} in module {node.value}' + ) # TODO: Support star imports - gamma[imported_name] = current_subs( - imported_type - ) + gamma[imported_name] = current_subs(imported_type) elif isinstance(node, concat.parse.ImportStatementNode): # TODO: Support all types of import correctly. if node.asname is not None: @@ -404,12 +486,19 @@ def infer( S = current_subs f = current_effect name = node.name - # NOTE: To continue the "bidirectional" bent, we will require a + # NOTE: To continue the "bidirectional" bent, we will require a ghg # type annotation. # TODO: Make the return types optional? + print('node', repr(node.stack_effect)) declared_type, _ = node.stack_effect.to_type(S(gamma)) + print('type from node', declared_type) declared_type = S(declared_type) + print('subbed', declared_type) recursion_env = gamma.copy() + if not isinstance(declared_type, StackEffect): + raise TypeError( + f'declared type of {name} must be a stack effect, got {declared_type}' + ) recursion_env[name] = declared_type.generalized_wrt(S(gamma)) if check_bodies: phi1, inferred_type, _ = infer( @@ -418,7 +507,7 @@ def infer( is_top_level=False, extensions=extensions, initial_stack=declared_type.input, - check_bodies=check_bodies + check_bodies=check_bodies, ) # We want to check that the inferred outputs are subtypes of # the declared outputs. Thus, inferred_type.output should be a subtype @@ -431,17 +520,40 @@ def infer( )( S ) - except TypeError: + except TypeError as e: message = ( 'declared function type {} is not compatible with ' 'inferred type {}' ) raise TypeError( message.format(declared_type, inferred_type) - ) + ) from e effect = declared_type + # type check decorators + _, final_type_stack, _ = infer( + gamma, + node.decorators, + is_top_level=False, + extensions=extensions, + initial_stack=TypeSequence([effect]), + check_bodies=check_bodies, + ) + final_type_stack_output = final_type_stack.output.as_sequence() + if len(final_type_stack_output) != 1: + raise TypeError( + f'Decorators produce too many stack items: only 1 should be left. Stack: {final_type_stack.output}' + ) + final_type = final_type_stack_output[0] + if not isinstance(final_type, IndividualType): + raise TypeError( + f'Decorators should produce something of individual type, got {final_type}' + ) # we *mutate* the type environment - gamma[name] = effect.generalized_wrt(S(gamma)) + gamma[name] = ( + final_type.generalized_wrt(S(gamma)) + if isinstance(final_type, StackEffect) + else final_type + ) elif isinstance(node, concat.parse.NumberWordNode): if isinstance(node.value, int): current_effect = StackEffect( @@ -453,7 +565,10 @@ def infer( (i1, o1) = current_effect if node.value not in current_subs(gamma): raise NameError(node) - type_of_name = current_subs(gamma)[node.value].instantiate() + type_of_name = current_subs(gamma)[node.value] + if isinstance(type_of_name, ForwardTypeReference): + raise NameError(node) + type_of_name = type_of_name.instantiate() type_of_name = type_of_name.get_type_of_attribute( '__call__' ).instantiate() @@ -486,7 +601,7 @@ def infer( extensions=extensions, source_dir=source_dir, initial_stack=input_stack, - check_bodies=check_bodies + check_bodies=check_bodies, ) current_subs, current_effect = ( S1(S), @@ -529,22 +644,39 @@ def infer( ) elif isinstance(node, concat.parse.ParseError): current_effect = StackEffect( - current_effect.input, - TypeSequence([SequenceVariable()]), + current_effect.input, TypeSequence([SequenceVariable()]), ) - elif not check_bodies and isinstance(node, concat.parse.ClassdefStatementNode): + elif not check_bodies and isinstance( + node, concat.parse.ClassdefStatementNode + ): type_parameters = [] temp_gamma = gamma - for param_node in node.type_parameters: - param, temp_gamma = param_node.to_type(temp_gamma) - type_parameters.append(param) + if node.is_variadic: + type_parameters.append(SequenceVariable()) + else: + for param_node in node.type_parameters: + param, temp_gamma = param_node.to_type(temp_gamma) + type_parameters.append(param) + + _, _, body_attrs = infer( + temp_gamma, + node.children, + extensions=extensions, + source_dir=source_dir, + initial_stack=TypeSequence([]), + check_bodies=check_bodies, + ) gamma[node.class_name] = ObjectType( IndividualVariable(), - {}, + body_attrs, type_parameters, (), True, + is_variadic=node.is_variadic, ) + gamma[node.class_name].set_internal_name(node.class_name) + # elif isinstance(node, concat.parse.TypeAliasStatementNode): + # gamma[node.name], _ = node.type_node.to_type(gamma) else: raise UnhandledNodeTypeError( "don't know how to handle '{}'".format(node) @@ -556,7 +688,9 @@ def infer( @functools.lru_cache(maxsize=None) -def _check_stub_resolved_path(path: pathlib.Path) -> 'Environment': +def _check_stub_resolved_path( + path: pathlib.Path, is_builtins: bool = False +) -> 'Environment': try: source = path.read_text() except FileNotFoundError as e: @@ -564,16 +698,45 @@ def _check_stub_resolved_path(path: pathlib.Path) -> 'Environment': except IOError as e: raise TypeError(f'Failed to read type stubs at {path}') from e tokens = concat.lex.tokenize(source) - from concat.transpile import parse - concat_ast = parse(tokens) - print(concat_ast) - print(list(concat_ast.parsing_failures)) + # print(tokens) env = Environment() - return check(env, concat_ast.children, str(path.parent), check_bodies=False) + from concat.transpile import parse + try: + concat_ast = parse(tokens) + except concat.parser_combinators.ParseError as e: + print('Parse Error:') + with path.open() as file: + print( + create_parsing_failure_message( + file, tokens, e.args[0].failures + ) + ) + return env + # print(concat_ast) + recovered_parsing_failures = concat_ast.parsing_failures + with path.open() as file: + for failure in recovered_parsing_failures: + print('Parse Error:') + print(create_parsing_failure_message(file, tokens, failure)) + return check( + env, + concat_ast.children, + str(path.parent), + _should_check_bodies=False, + _should_load_builtins=not is_builtins, + ) -def _check_stub(path: pathlib.Path) -> 'Environment': - return _check_stub_resolved_path(path.resolve()) + +def _check_stub( + path: pathlib.Path, is_builtins: bool = False +) -> 'Environment': + path = path.resolve() + try: + return _check_stub_resolved_path(path, is_builtins) + except StaticAnalysisError as e: + e.set_path_if_missing(path) + raise # Parsing type annotations @@ -651,7 +814,9 @@ def to_type(self, env: Environment) -> Tuple[IndividualType, Environment]: generic_type, env = self._generic_type.to_type(env) if isinstance(generic_type, ObjectType): if generic_type.is_variadic: - args = (TypeSequence(args), ) + args = (TypeSequence(args),) + return generic_type[args], env + if isinstance(generic_type.kind, GenericTypeKind): return generic_type[args], env raise TypeError('{} is not a generic type'.format(generic_type)) @@ -786,19 +951,13 @@ def to_type(self, env: Environment) -> Tuple[StackEffect, Environment]: in_types = [] for item in self.input: type, new_env = _ensure_type( - item[1], - new_env, - item[0], - known_stack_item_names, + item[1], new_env, item[0], known_stack_item_names, ) in_types.append(type) out_types = [] for item in self.output: type, new_env = _ensure_type( - item[1], - new_env, - item[0], - known_stack_item_names, + item[1], new_env, item[0], known_stack_item_names, ) out_types.append(type) @@ -975,8 +1134,8 @@ def sequence_type_variable_parser() -> Generator: return _SequenceVariableNode(name) @concat.parser_combinators.generate - def type_sequence_parser() -> Generator: - type = parsers['type'] | individual_type_variable_parser + def stack_effect_type_sequence_parser() -> Generator: + type = parsers['type'] # TODO: Allow type-only items item = concat.parser_combinators.seq( @@ -999,14 +1158,35 @@ def type_sequence_parser() -> Generator: return TypeSequenceNode(location, seq_var_parsed, i) + @concat.parser_combinators.generate + def type_sequence_parser() -> Generator: + type = parsers['type'] + + item = type.map(lambda t: _TypeSequenceIndividualTypeNode((None, t))) + items = item.many() + + seq_var = sequence_type_variable_parser + seq_var_parsed: Optional[_SequenceVariableNode] + seq_var_parsed = yield seq_var.optional() + i = yield items + + if seq_var_parsed is None and i: + location = i[0].location + elif seq_var_parsed is not None: + location = seq_var_parsed.location + else: + location = None + + return TypeSequenceNode(location, seq_var_parsed, i) + @concat.parser_combinators.generate def stack_effect_type_parser() -> Generator: separator = concat.parse.token('MINUSMINUS') location = (yield concat.parse.token('LPAR')).start - i = yield parsers['type-sequence'] << separator - o = yield parsers['type-sequence'] + i = yield stack_effect_type_sequence_parser << separator + o = yield stack_effect_type_sequence_parser yield concat.parse.token('RPAR') @@ -1043,8 +1223,7 @@ def forall_type_parser() -> Generator: return _ForallTypeNode(forall.start, type_variables, ty) parsers['type-variable'] = concat.parser_combinators.alt( - sequence_type_variable_parser, - individual_type_variable_parser, + sequence_type_variable_parser, individual_type_variable_parser, ) parsers['nonparameterized-type'] = concat.parser_combinators.alt( @@ -1166,8 +1345,7 @@ def _ensure_type( type = IndividualVariable() elif isinstance(typename, TypeNode): type, env = cast( - Tuple[StackItemType, Environment], - typename.to_type(env), + Tuple[StackItemType, Environment], typename.to_type(env), ) else: raise NotImplementedError( diff --git a/concat/typecheck/preamble_types.py b/concat/typecheck/preamble_types.py index 8ca8923..585077a 100644 --- a/concat/typecheck/preamble_types.py +++ b/concat/typecheck/preamble_types.py @@ -17,6 +17,7 @@ init_primitives, int_type, iterable_type, + iterator_type, leq_comparable_type, lt_comparable_type, list_type, @@ -27,6 +28,7 @@ object_type, optional_type, py_function_type, + py_overloaded_type, str_type, subscriptable_type, subtractable_type, @@ -188,10 +190,6 @@ TypeSequence([_rest_var]), ), ), - # Python builtins - 'print': py_function_type, - 'Exception': py_function_type, - 'input': py_function_type, 'if_then': ObjectType( _x, { @@ -259,22 +257,15 @@ TypeSequence([_stack_type_var, int_type]), ), 'iterable': iterable_type, - 'tuple': tuple_type, - 'BaseException': base_exception_type, 'NoReturn': no_return_type, 'subscriptable': subscriptable_type, 'subtractable': subtractable_type, - 'bool': bool_type, - 'object': object_type, 'context_manager': context_manager_type, - 'dict': dict_type, + 'iterator': iterator_type, 'module': module_type, - 'list': list_type, - 'str': str_type, 'py_function': py_function_type, + 'py_overloaded': py_overloaded_type, 'Optional': optional_type, - 'int': int_type, - 'float': float_type, 'file': file_type, 'none': none_type, 'None': ForAll( diff --git a/concat/typecheck/py_builtins.cati b/concat/typecheck/py_builtins.cati new file mode 100644 index 0000000..37f43c8 --- /dev/null +++ b/concat/typecheck/py_builtins.cati @@ -0,0 +1,94 @@ +# NOTE: I will probably need to mark this as "THE object type" in the future +# since other classes implicitly inherit from it. +class object: + () + +class bool: + () + +class int: + def __add__(--) @cast (py_function[(object), str]): + () + + def __invert__(--) @cast (py_function[(), int]): + () + + def __sub__(--) @cast (py_function[(object), str]): + () + + def __le__(--) @cast (py_function[(int), bool]): + () + + def __lt__(--) @cast (py_function[(int), bool]): + () + + def __ge__(--) @cast (py_function[(int), bool]): + () + +class float: + () + +class slice[`start, `stop, `step]: + () + +class str: + def __getitem__(--) @cast (py_overloaded[ + py_function[(int), str], + py_function[ + (slice[Optional[int], Optional[int], Optional[int]]), + str + ] + ]): + () + + def __add__(--) @cast (py_function[(object), str]): + () + + def find(--) @cast (py_function[(str Optional[int] Optional[int]), int]): + () + + def join(--) @cast (py_function[(str iterable[str]), str]): + () + + def __iter__(--) @cast (py_function[(), iterator[str]]): + () + + def index(--) @cast (py_function[(str Optional[int] Optional[int]), int]): + () + +def eval(--) @cast (py_function[(str), object]): + () + +def print(--) @cast (py_function[(), none]): + () + +class BaseException: + () + +class Exception($BaseException,): + () + +def input(--) @cast (py_function[(str), str]): + () + +class tuple[`t...]: + # NOTE: Will need a new feature to get the type of __getitem__ right. + def __getitem__(--) @cast (py_function[(int), object]): + () + +class dict[`key, `value]: + def __iter__(--) @cast (py_function[(), iterator[`key]]): + () + +class list[`element]: + def __getitem__(--) @cast (py_overloaded[ + py_function[(int), `element], + py_function[ + (slice[Optional[int], Optional[int], Optional[int]]), + list[`element] + ] + ]): + () + + def __iter__(--) @cast (py_function[(), iterator[`element]]): + () diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 07390bd..f772e23 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -1,28 +1,25 @@ from concat.orderedset import OrderedSet -from concat.typecheck import ( - AttributeError, - StackMismatchError, - TypeError, -) import concat.typecheck +import functools from typing import ( AbstractSet, - Optional, + Callable, Dict, Iterable, Iterator, + Iterator, + List, + Mapping, + NoReturn, + Optional, Sequence, + Set, + TYPE_CHECKING, Tuple, TypeVar, Union, - List, - Iterator, - Set, - Mapping, - NoReturn, cast, overload, - TYPE_CHECKING, ) from typing_extensions import Literal import abc @@ -35,6 +32,11 @@ class Type(abc.ABC): + def __init__(self) -> None: + self._free_type_variables_cached: Optional[ + OrderedSet[_Variable] + ] = None + # TODO: Fully replace with <=. def is_subtype_of(self, supertype: 'Type') -> bool: return ( @@ -70,9 +72,14 @@ def attributes(self) -> Mapping[str, 'Type']: pass @abc.abstractmethod - def free_type_variables(self) -> OrderedSet['_Variable']: + def _free_type_variables(self) -> OrderedSet['_Variable']: pass + def free_type_variables(self) -> OrderedSet['_Variable']: + if self._free_type_variables_cached is None: + self._free_type_variables_cached = self._free_type_variables() + return self._free_type_variables_cached + @abc.abstractmethod def apply_substitution( self, _: 'concat.typecheck.Substitutions' @@ -110,6 +117,14 @@ def constrain(self, supertype: 'Type') -> None: def instantiate(self) -> 'Type': return self + @abc.abstractmethod + def resolve_forward_references(self) -> 'Type': + pass + + @abc.abstractproperty + def kind(self) -> 'Kind': + pass + class IndividualType(Type, abc.ABC): def to_for_all(self) -> Type: @@ -132,11 +147,14 @@ def instantiate(self) -> 'IndividualType': @abc.abstractmethod def apply_substitution( - self, - sub: 'concat.typecheck.Substitutions', + self, sub: 'concat.typecheck.Substitutions', ) -> 'IndividualType': pass + @property + def kind(self) -> 'Kind': + return IndividualKind() + class _Variable(Type, abc.ABC): """Objects that represent type variables. @@ -149,12 +167,22 @@ def apply_substitution( self, sub: 'concat.typecheck.Substitutions' ) -> Union[IndividualType, '_Variable', 'TypeSequence']: if self in sub: - return sub[self] # type: ignore + result = sub[self] + assert self.kind == result.kind, f'{self!r} --> {result!r}' + return result # type: ignore return self - def free_type_variables(self) -> OrderedSet['_Variable']: + def _free_type_variables(self) -> OrderedSet['_Variable']: return OrderedSet({self}) + def __lt__(self, other) -> bool: + """Comparator for storing variables in OrderedSets.""" + return id(self) < id(other) + + def __gt__(self, other) -> bool: + """Comparator for storing variables in OrderedSets.""" + return id(self) > id(other) + class IndividualVariable(_Variable, IndividualType): def __init__(self) -> None: @@ -245,6 +273,13 @@ def attributes(self) -> NoReturn: ) ) + def resolve_forward_references(self) -> 'IndividualVariable': + return self + + @property + def kind(self) -> 'Kind': + return IndividualKind() + class SequenceVariable(_Variable): def __init__(self) -> None: @@ -319,9 +354,17 @@ def attributes(self) -> NoReturn: 'the sequence type {} does not hold attributes'.format(self) ) + def resolve_forward_references(self) -> 'SequenceVariable': + return self + + @property + def kind(self) -> 'Kind': + return SequenceKind() + class TypeSequence(Type, Iterable['StackItemType']): def __init__(self, sequence: Sequence['StackItemType']) -> None: + super().__init__() self._rest: Optional[SequenceVariable] if sequence and isinstance(sequence[0], SequenceVariable): self._rest = sequence[0] @@ -412,7 +455,7 @@ def constrain_and_bind_supertype_variables( and self._rest and not self._individual_types ): - raise StackMismatchError(self, supertype) + raise concat.typecheck.StackMismatchError(self, supertype) elif not self._individual_types: if ( self._is_empty() @@ -435,7 +478,7 @@ def constrain_and_bind_supertype_variables( ): return Substitutions({supertype._rest: self}) else: - raise StackMismatchError(self, supertype) + raise concat.typecheck.StackMismatchError(self, supertype) elif ( not supertype._individual_types and supertype._rest @@ -479,11 +522,11 @@ def constrain_and_bind_supertype_variables( sub ) return sub - except StackMismatchError: - raise StackMismatchError(self, supertype) + except concat.typecheck.StackMismatchError: + raise concat.typecheck.StackMismatchError(self, supertype) else: # TODO: Add info about occurs check and rigid variables. - raise StackMismatchError(self, supertype) + raise concat.typecheck.StackMismatchError(self, supertype) else: raise TypeError( '{} must be a sequence type, not {}'.format(self, supertype) @@ -509,7 +552,7 @@ def constrain_and_bind_subtype_variables( and not supertype._individual_types and supertype._rest not in rigid_variables ): - raise StackMismatchError(self, supertype) + raise concat.typecheck.StackMismatchError(self, supertype) elif ( supertype._is_empty() and self._rest @@ -536,16 +579,16 @@ def constrain_and_bind_subtype_variables( and not supertype._individual_types ): # QUESTION: Should this be allowed? I'm being defensive here. - raise StackMismatchError(self, supertype) + raise concat.typecheck.StackMismatchError(self, supertype) else: - raise StackMismatchError(self, supertype) + raise concat.typecheck.StackMismatchError(self, supertype) elif ( not supertype._individual_types and supertype._rest and supertype._rest not in self.free_type_variables() and supertype._rest not in rigid_variables ): - raise StackMismatchError(self, supertype) + raise concat.typecheck.StackMismatchError(self, supertype) elif self._individual_types and supertype._individual_types: sub = self._individual_types[ -1 @@ -575,16 +618,16 @@ def constrain_and_bind_subtype_variables( subtyping_assumptions, )(sub) return sub - except StackMismatchError: - raise StackMismatchError(self, supertype) + except concat.typecheck.StackMismatchError: + raise concat.typecheck.StackMismatchError(self, supertype) else: - raise StackMismatchError(self, supertype) + raise concat.typecheck.StackMismatchError(self, supertype) else: raise TypeError( '{} must be a sequence type, not {}'.format(self, supertype) ) - def free_type_variables(self) -> OrderedSet['_Variable']: + def _free_type_variables(self) -> OrderedSet['_Variable']: ftv: OrderedSet[_Variable] = OrderedSet([]) for t in self: ftv |= t.free_type_variables() @@ -629,6 +672,16 @@ def __iter__(self) -> Iterator['StackItemType']: def __hash__(self) -> int: return hash(tuple(self.as_sequence())) + def resolve_forward_references(self) -> 'TypeSequence': + self._individual_types = [ + t.resolve_forward_references() for t in self._individual_types + ] + return self + + @property + def kind(self) -> 'Kind': + return SequenceKind() + # TODO: Actually get rid of ForAll uses. This is a temporary measure since I # don't want to do that work right now. @@ -641,6 +694,16 @@ class _Function(IndividualType): def __init__( self, input_types: TypeSequence, output_types: TypeSequence, ) -> None: + for ty in input_types[1:]: + if ty.kind != IndividualKind(): + raise concat.typeheck.TypeError( + f'{ty} must be an individual type' + ) + for ty in output_types[1:]: + if ty.kind != IndividualKind(): + raise concat.typeheck.TypeError( + f'{ty} must be an individual type' + ) super().__init__() self.input = input_types self.output = output_types @@ -764,7 +827,7 @@ def constrain_and_bind_subtype_variables( )(sub) return sub - def free_type_variables(self) -> OrderedSet['_Variable']: + def _free_type_variables(self) -> OrderedSet['_Variable']: return ( self.input.free_type_variables() | self.output.free_type_variables() @@ -783,7 +846,9 @@ def _rename_sequence_variable( and isinstance(subtype_list[0], SequenceVariable) ): if supertype_list[0] not in sub: - sub[supertype_list[0]] = subtype_list[0] + # FIXME: Treat sub immutably, or better yet, don't use + # substitutions here if possible + sub._sub[supertype_list[0]] = subtype_list[0] else: if sub(supertype_list[0]) is not subtype_list[0]: return False @@ -816,6 +881,11 @@ def apply_substitution( def bind(self) -> '_Function': return _Function(self.input[:-1], self.output) + def resolve_forward_references(self) -> 'StackEffect': + self.input = self.input.resolve_forward_references() + self.output = self.output.resolve_forward_references() + return self + class QuotationType(_Function): def __init__(self, fun_type: _Function) -> None: @@ -924,6 +994,7 @@ def __init__( _head: Optional['ObjectType'] = None, **_other_kwargs, ) -> None: + super().__init__() # There should be no need to make the self_type variable unique because # it is treated as a bound variable in apply_substitution. In other # words, it is removed from any substitution received. @@ -951,6 +1022,25 @@ def __init__( self._instantiations: Dict[TypeArguments, ObjectType] = {} + def resolve_forward_references(self) -> 'ObjectType': + self._attributes = { + attr: t.resolve_forward_references() + for attr, t in self._attributes.items() + } + self._nominal_supertypes = [ + t.resolve_forward_references() for t in self._nominal_supertypes + ] + self._type_arguments = [ + t.resolve_forward_references() for t in self._type_arguments + ] + return self + + @property + def kind(self) -> 'Kind': + if len(self._type_parameters) == 0: + return IndividualKind() + return GenericTypeKind([var.kind for var in self._type_parameters]) + def apply_substitution( self, sub: 'concat.typecheck.Substitutions', @@ -1268,7 +1358,7 @@ def __repr__(self) -> str: None if self._head is self else self._head, ) - def free_type_variables(self) -> OrderedSet[_Variable]: + def _free_type_variables(self) -> OrderedSet[_Variable]: ftv = free_type_variables_of_mapping(self.attributes) for arg in self.type_arguments: ftv |= arg.free_type_variables() @@ -1309,6 +1399,7 @@ def __hash__(self) -> int: if ObjectType._hash_variable is None: ObjectType._hash_variable = IndividualVariable() sub = Substitutions({self._self_type: ObjectType._hash_variable}) + # Avoid sub(self) since the lru cache on that will hash self type_to_hash = sub(self) return hash( ( @@ -1322,10 +1413,7 @@ def __hash__(self) -> int: ) ) - def __getitem__( - self, - type_arguments: TypeArguments, - ) -> 'ObjectType': + def __getitem__(self, type_arguments: TypeArguments,) -> 'ObjectType': from concat.typecheck import Substitutions if self._arity != len(type_arguments): @@ -1430,12 +1518,26 @@ def __init__( ) if self._arity == 0: assert isinstance(self.input, collections.abc.Sequence) + assert self._type_arguments[1].kind == IndividualKind() self._args = list(args) self._overloads = _overloads if '_head' in self._kwargs: del self._kwargs['_head'] self._head: PythonFunctionType + def resolve_forward_references(self) -> 'PythonFunctionType': + super().resolve_forward_references() + overloads = [] + for args, ret in overloads: + overloads.append( + ( + [arg.resolve_forward_references() for arg in args], + ret.resolve_forward_references(), + ) + ) + self._overloads = overloads + return self + def __str__(self) -> str: if not self._type_arguments: return 'py_function_type' @@ -1496,7 +1598,7 @@ def input(self) -> Sequence[StackItemType]: @property def output(self) -> IndividualType: assert self._arity == 0 - assert isinstance(self._type_arguments[1], IndividualType) + assert self._type_arguments[1].kind == IndividualKind() return self._type_arguments[1] def select_overload( @@ -1677,6 +1779,71 @@ def constrain_and_bind_subtype_variables( ) +class _PythonOverloadedType(Type): + def __init__(self) -> None: + super().__init__() + + def __getitem__(self, args: Sequence[Type]) -> '_PythonOverloadedType': + import concat.typecheck + + if len(args) == 0: + raise concat.typecheck.TypeError( + 'py_overloaded must be applied to at least one argument' + ) + for arg in args: + if not isinstance(arg, PythonFunctionType): + raise concat.typecheck.TypeError( + 'Arguments to py_overloaded must be Python function types' + ) + fun_type = args[0] + for arg in args[1:]: + fun_type = fun_type.with_overload(arg.input, arg.output) + return fun_type + + def attributes(self) -> Mapping[str, 'Type']: + raise TypeError('py_overloaded does not have attributes') + + def _free_type_variables(self) -> OrderedSet['_Variable']: + return OrderedSet([]) + + def apply_substitution( + self, _: 'concat.typecheck.Substitutions' + ) -> '_PythonOverloadedType': + return self + + def constrain_and_bind_supertype_variables( + self, + supertype: 'Type', + rigid_variables: AbstractSet['_Variable'], + subtyping_assumptions: List[Tuple['IndividualType', 'IndividualType']], + ) -> 'Substitutions': + raise concat.typecheck.TypeError('py_overloaded is a generic type') + + def constrain_and_bind_subtype_variables( + self, + supertype: 'Type', + rigid_variables: AbstractSet['_Variable'], + subtyping_assumptions: List[Tuple['IndividualType', 'IndividualType']], + ) -> 'Substitutions': + raise concat.typecheck.TypeError('py_overloaded is a generic type') + + def resolve_forward_references(self) -> '_PythonOverloadedType': + return self + + @property + def kind(self) -> 'Kind': + return GenericTypeKind([SequenceKind()]) + + def __hash__(self) -> int: + return hash(type(self).__qualname__) + + def __eq__(self, other: object) -> bool: + return isinstance(other, type(self)) + + +py_overloaded_type = _PythonOverloadedType() + + class _NoReturnType(ObjectType): def __init__(self) -> None: x = IndividualVariable() @@ -1716,6 +1883,158 @@ def apply_substitution( return _OptionalType(tuple(sub(TypeSequence(self._type_arguments)))) +class Kind(abc.ABC): + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + pass + + @abc.abstractmethod + def __hash__(self) -> int: + pass + + +class IndividualKind(Kind): + def __eq__(self, other: object) -> bool: + return isinstance(other, IndividualKind) + + def __hash__(self) -> int: + return hash(type(self).__qualname__) + + +class SequenceKind(Kind): + def __eq__(self, other: object) -> bool: + return isinstance(other, SequenceKind) + + def __hash__(self) -> int: + return hash(type(self).__qualname__) + + +class GenericTypeKind(Kind): + def __init__(self, parameter_kinds: Sequence[Kind]) -> None: + assert parameter_kinds + self.parameter_kinds = parameter_kinds + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, GenericTypeKind) + and self.parameter_kinds == other.parameter_kinds + ) + + def __hash__(self) -> int: + return hash(tuple(self.parameter_kinds)) + + +class ForwardTypeReference(Type): + def __init__( + self, + kind: Kind, + name_to_resolve: str, + resolution_env: 'Environment', + _type_arguments: TypeArguments = (), + ) -> None: + super().__init__() + self._kind = kind + self._name_to_resolve = name_to_resolve + self._resolution_env = resolution_env + self._resolved_type: Optional[Type] = None + self._type_arguments = _type_arguments + + def _resolve(self) -> Type: + ty = self._resolution_env[self._name_to_resolve] + if self._type_arguments: + ty = ty[self._type_arguments] + return ty + + def _as_hashable_tuple(self) -> tuple: + return ( + self._kind, + id(self._resolution_env), + self._name_to_resolve, + tuple(self._type_arguments), + ) + + def __hash__(self) -> int: + return hash(self._as_hashable_tuple()) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Type): + return NotImplemented + if not isinstance(other, ForwardTypeReference): + return False + return self._as_hashable_tuple() == other._as_hashable_tuple() + + def __getitem__(self, args: TypeArguments) -> IndividualType: + import concat.typecheck + + if isinstance(self.kind, GenericTypeKind): + if len(self.kind.parameter_kinds) != len(args): + raise concat.typecheck.TypeError( + 'Wrong number of arguments to generic type' + ) + for kind, arg in zip(self.kind.parameter_kinds, args): + if kind != arg.kind: + raise concat.typecheck.TypeError( + f'Type argument has kind {arg.kind}, expected kind {kind}' + ) + return ForwardTypeReference( + IndividualKind(), + self._name_to_resolve, + self._resolution_env, + _type_arguments=args, + ) + raise TypeError(f'{self} is not a generic type') + + def resolve_forward_references(self) -> Type: + if self._resolved_type is None: + self._resolved_type = self._resolve() + return self._resolved_type + + def apply_substitution( + self, sub: 'concat.typecheck.Substitutions' + ) -> 'ForwardTypeReference': + return self + # return ForwardTypeReference(self.kind, lambda: sub(self._resolve())) + + @property + def attributes(self) -> Mapping[str, Type]: + import concat.typecheck + + raise concat.typecheck.TypeError( + 'Cannot access attributes of type before they are defined' + ) + + def constrain_and_bind_subtype_variables( + self, + supertype: Type, + rigid_variables: AbstractSet['_Variable'], + subtyping_assumptions: List[Tuple[IndividualType, IndividualType]], + ) -> 'Substitutions': + import concat.typecheck + + raise concat.typecheck.TypeError( + 'Supertypes of type are not known before its definition' + ) + + def constrain_and_bind_supertype_variables( + self, + supertype: Type, + rigid_variables: AbstractSet['_Variable'], + subtyping_assumptions: List[Tuple[IndividualType, IndividualType]], + ) -> 'Substitutions': + import concat.typecheck + + raise concat.typecheck.TypeError( + 'Supertypes of type are not known before its definition' + ) + + def _free_type_variables(self) -> OrderedSet[_Variable]: + return OrderedSet([]) + + @property + def kind(self) -> Kind: + return self._kind + + def _iterable_to_str(iterable: Iterable) -> str: return '[' + ', '.join(map(str, iterable)) + ']' @@ -1815,7 +2134,6 @@ def _mapping_to_str(mapping: Mapping) -> str: '__add__': _int_add_type, '__invert__': py_function_type[TypeSequence([]), _x], '__sub__': _int_add_type, - '__invert__': py_function_type[TypeSequence([]), _x], '__le__': py_function_type[TypeSequence([_x]), bool_type], '__lt__': py_function_type[TypeSequence([_x]), bool_type], '__ge__': py_function_type[TypeSequence([_x]), bool_type], From 1edc14e2dc07799525deb1738674595a7ad81803 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Tue, 26 Mar 2024 20:47:03 -0700 Subject: [PATCH 07/61] Add type stubs for to_str and to_py_function --- concat/stdlib/pyinterop/__init__.cati | 6 ++++++ concat/stdlib/pyinterop/__init__.py | 14 -------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/concat/stdlib/pyinterop/__init__.cati b/concat/stdlib/pyinterop/__init__.cati index e13f030..d8f2d26 100644 --- a/concat/stdlib/pyinterop/__init__.cati +++ b/concat/stdlib/pyinterop/__init__.cati @@ -3,3 +3,9 @@ def getitem(*stack_type_var obj:subscriptable[`x, `y] index:`x -- *stack_type_va def to_dict(*stack_type_var it:Optional[iterable[tuple[`x, `y]]] -- *stack_type_var d:dict[`x, `y]): () + +def to_str(*stack_type_var errors:object encoding:object obj:object -- *stack_type_var string:str): + () + +def to_py_function(*rest_var f:(*rest_var_2 -- *rest_var_3) -- *rest_var py_f:py_function[*any_args, `any_ret]): + () diff --git a/concat/stdlib/pyinterop/__init__.py b/concat/stdlib/pyinterop/__init__.py index 890e49f..fb7ee67 100644 --- a/concat/stdlib/pyinterop/__init__.py +++ b/concat/stdlib/pyinterop/__init__.py @@ -10,11 +10,8 @@ TypeSequence, dict_type, iterable_type, - object_type, optional_type, - py_function_type, slice_type, - str_type, subscriptable_type, tuple_type, ) @@ -100,17 +97,6 @@ TypeSequence([_stack_type_var, slice_type[_z, _y, _x]]), ), ), - 'to_str': StackEffect( - TypeSequence([_stack_type_var, object_type, object_type, object_type]), - TypeSequence([_stack_type_var, str_type]), - ), - 'to_py_function': ForAll( - [_rest_var, _rest_var_2, _rest_var_3], - StackEffect( - [_rest_var, StackEffect([_rest_var_2], [_rest_var_3])], - [_rest_var, py_function_type], - ), - ), } From ab89d753b617640e5bdc684f225c0f14cbfd1221 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Tue, 26 Mar 2024 21:48:17 -0700 Subject: [PATCH 08/61] Do more kind checking and allow forward references to forward operations to resolved type --- concat/typecheck/__init__.py | 14 ++-- concat/typecheck/types.py | 131 +++++++++++++++++++++++++---------- 2 files changed, 103 insertions(+), 42 deletions(-) diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index eec3c51..9dde0ea 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -39,7 +39,11 @@ def apply_substitution(self, sub: 'Substitutions') -> _Result: class Substitutions(collections.abc.Mapping, Mapping['_Variable', 'Type']): def __init__(self, sub: Iterable[Tuple['_Variable', 'Type']] = {}) -> None: self._sub = dict(sub) - self._by_identity_cache: Dict[int, Any] = {} + for variable, ty in self._sub.items(): + if variable.kind != ty.kind: + raise TypeError( + f'{variable} is being substituted by {ty}, which has the wrong kind ({variable.kind} vs {ty.kind})' + ) def __getitem__(self, var: '_Variable') -> 'Type': return self._sub[var] @@ -51,10 +55,10 @@ def __len__(self) -> int: return len(self._sub) def __call__(self, arg: _Substitutable[_Result]) -> _Result: - # if id(arg) not in self._by_identity_cache: - # self._by_identity_cache[id(arg)] = arg.apply_substitution(self) - # return self._by_identity_cache[id(arg)] - # Using the id is wrong because a different object can have the same id later + # Previously I tried caching results by the id of the argument. But + # since the id is the memory address of the object in CPython, another + # object might have the same id later. I think this was leading to + # nondeterministic Concat type errors from the type checker. return arg.apply_substitution(self) def _dom(self) -> Set['_Variable']: diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index f772e23..686c0e7 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -419,9 +419,7 @@ def is_subtype_of(self, supertype: Type) -> bool: else: return False else: - raise TypeError( - '{} must be a sequence type, not {}'.format(self, supertype) - ) + return False def constrain_and_bind_supertype_variables( self, @@ -994,6 +992,7 @@ def __init__( _head: Optional['ObjectType'] = None, **_other_kwargs, ) -> None: + assert isinstance(self_type, IndividualVariable) super().__init__() # There should be no need to make the self_type variable unique because # it is treated as a bound variable in apply_substitution. In other @@ -1122,20 +1121,24 @@ def is_subtype_of(self, supertype: 'Type') -> bool: return False # instantiate these types in a way such that alpha equivalence is not # an issue - # NOTE: I assume the parameters are in the same order, which is - # fragile. - parameter_pairs = zip(self.type_parameters, supertype.type_parameters) - if not all(type(a) == type(b) for a, b in parameter_pairs): - return False - self = self.instantiate() - supertype = supertype.instantiate() - parameter_sub = Substitutions( - { - **dict(zip(self.type_arguments, supertype.type_arguments)), - self.self_type: supertype.self_type, - } - ) - self = parameter_sub(self) + if self._arity > 0: + parameter_pairs = zip( + self.type_parameters, supertype.type_parameters + ) + if not all(a.kind == b.kind for a, b in parameter_pairs): + return False + self = self.instantiate() + supertype = supertype.instantiate() + argument_pairs = dict( + zip(self.type_arguments, supertype.type_arguments) + ) + assert all( + a.kind == b.kind for a, b in argument_pairs.items() + ), str(repr(argument_pairs)) + parameter_sub = Substitutions( + {**argument_pairs, self.self_type: supertype.self_type,} + ) + self = parameter_sub(self) for attr, attr_type in supertype._attributes.items(): if attr not in self._attributes: @@ -1166,7 +1169,7 @@ def constrain_and_bind_supertype_variables( ): return Substitutions({supertype: self}) elif isinstance(supertype, (SequenceVariable, TypeSequence)): - raise TypeError( + raise concat.typecheck.TypeError( '{} is an individual type, but {} is a sequence type'.format( self, supertype ) @@ -1189,7 +1192,7 @@ def constrain_and_bind_supertype_variables( if not isinstance(supertype, ObjectType): raise NotImplementedError(supertype) if self._arity < supertype._arity: - raise TypeError( + raise concat.typecheck.TypeError( '{} is not as polymorphic as {}'.format(self, supertype) ) # every object type is a subtype of object_type @@ -1202,7 +1205,7 @@ def constrain_and_bind_supertype_variables( and supertype != self and self._head != supertype._head ): - raise TypeError( + raise concat.typecheck.TypeError( '{} is not a subtype of {}'.format(self, supertype) ) @@ -1218,7 +1221,7 @@ def constrain_and_bind_supertype_variables( return self.constrain_and_bind_supertype_variables( none_type, rigid_variables, subtyping_assumptions ) - except TypeError: + except concat.typecheck.TypeError: return self.constrain_and_bind_supertype_variables( supertype._type_arguments[0], rigid_variables, @@ -1235,9 +1238,10 @@ def constrain_and_bind_supertype_variables( instantiated_self = self.instantiate() supertype = supertype.instantiate() for name in supertype._attributes: - type = instantiated_self.get_type_of_attribute(name) + # FIXME: Really types of attributes should not be higher-kinded + type = instantiated_self.get_type_of_attribute(name).instantiate() sub = sub(type).constrain_and_bind_supertype_variables( - sub(supertype.get_type_of_attribute(name)), + sub(supertype.get_type_of_attribute(name).instantiate()), rigid_variables, subtyping_assumptions, )(sub) @@ -1284,7 +1288,7 @@ def constrain_and_bind_subtype_variables( if not isinstance(supertype, ObjectType): raise NotImplementedError(supertype) if self._arity < supertype._arity: - raise TypeError( + raise concat.typecheck.TypeError( '{} is not as polymorphic as {}'.format(self, supertype) ) # every object type is a subtype of object_type @@ -1331,8 +1335,11 @@ def constrain_and_bind_subtype_variables( supertype = supertype.instantiate() for name in supertype._attributes: type = instantiated_self.get_type_of_attribute(name) + # FIXME: Really types of attributes should not be higher-kinded + type = type.instantiate() + # print('in constrain_and_bind_subtype_variables, objecttype', repr(type)) sub = type.constrain_and_bind_subtype_variables( - supertype.get_type_of_attribute(name), + supertype.get_type_of_attribute(name).instantiate(), rigid_variables, subtyping_assumptions, )(sub) @@ -1416,13 +1423,23 @@ def __hash__(self) -> int: def __getitem__(self, type_arguments: TypeArguments,) -> 'ObjectType': from concat.typecheck import Substitutions + if not isinstance(self.kind, GenericTypeKind): + raise concat.typecheck.TypeError(f'{self} is not a generic type') + if self._arity != len(type_arguments): - raise TypeError( + raise concat.typecheck.TypeError( 'type constructor {} given {} arguments, expected {} arguments'.format( self, len(type_arguments), self._arity ) ) + expected_kinds = tuple(self.kind.parameter_kinds) + given_kinds = tuple(ty.kind for ty in type_arguments) + if expected_kinds != given_kinds: + raise concat.typecheck.TypeError( + f'wrong kinds of arguments given to type: expected {expected_kinds}, given {given_kinds}' + ) + type_arguments = tuple(type_arguments) if type_arguments in self._instantiations: return self._instantiations[type_arguments] @@ -1557,9 +1574,22 @@ def get_type_of_attribute(self, attribute: str) -> IndividualType: def __getitem__( self, arguments: Tuple[TypeSequence, IndividualType] ) -> 'PythonFunctionType': - assert self._arity == 2 + if self._arity != 2: + raise concat.typecheck.TypeError(f'{self} is not a generic type') + if len(arguments) != 2: + raise concat.typecheck.TypeError( + f'{self} takes two arguments, got {len(arguments)}' + ) input = arguments[0] output = arguments[1] + if input.kind != SequenceKind(): + raise concat.typecheck.TypeError( + f'First argument to {self} must be a sequence type of function arguments' + ) + if output.kind != IndividualKind(): + raise concat.typecheck.TypeError( + f'Second argument to {self} must be an individual type for the return type' + ) return PythonFunctionType( self._self_type, *self._args, @@ -1644,6 +1674,8 @@ def bind(self) -> 'PythonFunctionType': return self._head[TypeSequence(inputs), output] def is_subtype_of(self, supertype: Type) -> bool: + if super().is_subtype_of(supertype): + return True if isinstance(supertype, PythonFunctionType): # NOTE: make sure types are of same kind (arity) if len(self._type_parameters) != len(supertype._type_parameters): @@ -1651,14 +1683,11 @@ def is_subtype_of(self, supertype: Type) -> bool: if len(self._type_parameters) == 2: # both are py_function_type return True - assert isinstance(supertype._type_arguments[0], TypeSequence) - assert isinstance(self._type_arguments[1], IndividualType) return ( supertype._type_arguments[0] <= self._type_arguments[0] and self._type_arguments[1] <= supertype._type_arguments[1] ) - else: - return super().is_subtype_of(supertype) + return False def constrain_and_bind_supertype_variables( self, @@ -1783,7 +1812,7 @@ class _PythonOverloadedType(Type): def __init__(self) -> None: super().__init__() - def __getitem__(self, args: Sequence[Type]) -> '_PythonOverloadedType': + def __getitem__(self, args: Sequence[Type]) -> 'PythonFunctionType': import concat.typecheck if len(args) == 0: @@ -1800,6 +1829,7 @@ def __getitem__(self, args: Sequence[Type]) -> '_PythonOverloadedType': fun_type = fun_type.with_overload(arg.input, arg.output) return fun_type + @property def attributes(self) -> Mapping[str, 'Type']: raise TypeError('py_overloaded does not have attributes') @@ -1811,6 +1841,11 @@ def apply_substitution( ) -> '_PythonOverloadedType': return self + def instantiate(self) -> PythonFunctionType: + return self[ + py_function_type.instantiate(), + ] + def constrain_and_bind_supertype_variables( self, supertype: 'Type', @@ -1954,17 +1989,24 @@ def _as_hashable_tuple(self) -> tuple: ) def __hash__(self) -> int: + if self._resolved_type is not None: + return hash(self._resolved_type) return hash(self._as_hashable_tuple()) def __eq__(self, other: object) -> bool: + if super().__eq__(other): + return True if not isinstance(other, Type): return NotImplemented + if self._resolved_type is not None: + return self._resolved_type == other if not isinstance(other, ForwardTypeReference): return False return self._as_hashable_tuple() == other._as_hashable_tuple() def __getitem__(self, args: TypeArguments) -> IndividualType: - import concat.typecheck + if self._resolved_type is not None: + return self._resolved_type[args] if isinstance(self.kind, GenericTypeKind): if len(self.kind.parameter_kinds) != len(args): @@ -1992,12 +2034,15 @@ def resolve_forward_references(self) -> Type: def apply_substitution( self, sub: 'concat.typecheck.Substitutions' ) -> 'ForwardTypeReference': + if self._resolved_type is not None: + return sub(self._resolved_type) + return self - # return ForwardTypeReference(self.kind, lambda: sub(self._resolve())) @property def attributes(self) -> Mapping[str, Type]: - import concat.typecheck + if self._resolved_type is not None: + return self._resolved_type.attributes raise concat.typecheck.TypeError( 'Cannot access attributes of type before they are defined' @@ -2009,7 +2054,13 @@ def constrain_and_bind_subtype_variables( rigid_variables: AbstractSet['_Variable'], subtyping_assumptions: List[Tuple[IndividualType, IndividualType]], ) -> 'Substitutions': - import concat.typecheck + if self is supertype: + return concat.typecheck.Substitutions() + + if self._resolved_type is not None: + return self._resolved_type.constrain_and_bind_subtype_variables( + supertype, rigid_variables, subtyping_assumptions + ) raise concat.typecheck.TypeError( 'Supertypes of type are not known before its definition' @@ -2021,7 +2072,13 @@ def constrain_and_bind_supertype_variables( rigid_variables: AbstractSet['_Variable'], subtyping_assumptions: List[Tuple[IndividualType, IndividualType]], ) -> 'Substitutions': - import concat.typecheck + if self is supertype: + return concat.typecheck.Substitutions() + + if self._resolved_type is not None: + return self._resolved_type.constrain_and_bind_supertype_variables( + supertype, rigid_variables, subtyping_assumptions + ) raise concat.typecheck.TypeError( 'Supertypes of type are not known before its definition' From aef0d9348cd789873bbd27093b75288bd292e514 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Wed, 27 Mar 2024 23:26:52 -0700 Subject: [PATCH 09/61] Implement pragmas that tell type checker about fundamental builtin types --- concat/lex.py | 2 + concat/parse.py | 40 ++++++++-- concat/typecheck/__init__.py | 31 +++++--- concat/typecheck/py_builtins.cati | 10 ++- concat/typecheck/types.py | 123 +++++++++++++----------------- 5 files changed, 119 insertions(+), 87 deletions(-) diff --git a/concat/lex.py b/concat/lex.py index 11a3e48..09ba568 100644 --- a/concat/lex.py +++ b/concat/lex.py @@ -127,6 +127,8 @@ def _tokens(self) -> Iterator['Token']: continue elif tok.value == '$': tok.type = 'DOLLARSIGN' + elif tok.value == '!': + tok.type = 'EXCLAMATIONMARK' elif tok.value in {'def', 'import', 'from'}: tok.type = tok.value.upper() tok.is_keyword = True diff --git a/concat/parse.py b/concat/parse.py index 74578d6..0df932d 100644 --- a/concat/parse.py +++ b/concat/parse.py @@ -373,6 +373,16 @@ def __init__( self.is_variadic = is_variadic +class PragmaNode(Node): + def __init__( + self, location: 'Location', pragma_name: str, args: Sequence[str] + ) -> None: + super().__init__() + self.location = location + self.pragma = pragma_name + self.args = args + + def token(typ: str) -> concat.parser_combinators.Parser: description = f'{typ} token' return concat.parser_combinators.test_item( @@ -393,9 +403,7 @@ def top_level_parser() -> Generator[ newline = token('NEWLINE') statement = parsers['statement'] word = parsers['word'] - children = yield recover( - (word | statement | newline).many(), skip_until(token('ENDMARKER')) - ).map(lambda w: [ParseError(w[1])] if isinstance(w, tuple) else w) + children = yield (word | statement | newline).commit().many() children = [ child for child in children @@ -445,10 +453,10 @@ def top_level_parser() -> Generator[ @concat.parser_combinators.generate def quote_word_contents() -> Generator: - if 'type-sequence' in parsers: - input_stack_type_parser = parsers['type-sequence'] << token( - 'COLON' - ) + if 'stack-effect-type-sequence' in parsers: + input_stack_type_parser = parsers[ + 'stack-effect-type-sequence' + ] << token('COLON') input_stack_type = yield input_stack_type_parser.optional() else: input_stack_type = None @@ -747,6 +755,24 @@ def handle_recovery( parsers.ref_parser('word'), ) + @concat.parser_combinators.generate('internal pragma') + def pragma_parser() -> Generator: + """This parses a pragma for internal use. + + pragma = EXCLAMATIONMARK, @, @, qualified name+ + qualified name = module""" + location = (yield token('EXCLAMATIONMARK')).start + for _ in range(2): + name_token = yield token('NAME') + if name_token.value != '@': + return concat.parser_combinators.fail('a literal at sign (@)') + pragma_name = yield module + args = yield module.many() + return PragmaNode(location, pragma_name, args) + + parsers['pragma'] = pragma_parser + parsers['statement'] |= parsers.ref_parser('pragma') + parsers['word'] |= parsers.ref_parser('cast-word') @concat.parser_combinators.generate diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index 9dde0ea..4ecaa4a 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -86,8 +86,8 @@ def __hash__(self) -> int: from concat.typecheck.types import ( - ForwardTypeReference, ForAll, + ForwardTypeReference, GenericTypeKind, IndividualKind, IndividualType, @@ -105,18 +105,19 @@ def __hash__(self) -> int: context_manager_type, ellipsis_type, free_type_variables_of_mapping, + get_list_type, + get_object_type, init_primitives, int_type, invertible_type, iterable_type, - list_type, module_type, + no_return_type, none_type, not_implemented_type, - object_type, py_function_type, slice_type, - str_type, + get_str_type, subscriptable_type, subtractable_type, tuple_type, @@ -305,7 +306,17 @@ def infer( try: S, (i, o) = current_subs, current_effect - if isinstance(node, concat.parse.PushWordNode): + if isinstance(node, concat.parse.PragmaNode): + if node.pragma == 'concat.typecheck.builtin_object': + name = node.args[0] + concat.typecheck.types.set_object_type(gamma[name]) + if node.pragma == 'concat.typecheck.builtin_list': + name = node.args[0] + concat.typecheck.types.set_list_type(gamma[name]) + if node.pragma == 'concat.typecheck.builtin_str': + name = node.args[0] + concat.typecheck.types.set_str_type(gamma[name]) + elif isinstance(node, concat.parse.PushWordNode): S1, (i1, o1) = S, (i, o) child = node.children[0] if isinstance(child, concat.parse.FreezeWordNode): @@ -405,7 +416,10 @@ def infer( StackEffect( i, TypeSequence( - [*collected_type, list_type[element_type,]] + [ + *collected_type, + get_list_type()[element_type,], + ] ), ) ), @@ -616,7 +630,7 @@ def infer( S, StackEffect( current_effect.input, - TypeSequence([*current_effect.output, str_type]), + TypeSequence([*current_effect.output, get_str_type()]), ), ) elif isinstance(node, concat.parse.AttributeWordNode): @@ -717,7 +731,6 @@ def _check_stub_resolved_path( ) ) return env - # print(concat_ast) recovered_parsing_failures = concat_ast.parsing_failures with path.open() as file: for failure in recovered_parsing_failures: @@ -1287,7 +1300,7 @@ def _generate_type_of_innermost_module( sys.path = old_path module_attributes = {} for name in dir(module): - attribute_type = object_type + attribute_type = get_object_type() if isinstance(getattr(module, name), int): attribute_type = int_type elif callable(getattr(module, name)): diff --git a/concat/typecheck/py_builtins.cati b/concat/typecheck/py_builtins.cati index 37f43c8..c1ee206 100644 --- a/concat/typecheck/py_builtins.cati +++ b/concat/typecheck/py_builtins.cati @@ -1,8 +1,8 @@ -# NOTE: I will probably need to mark this as "THE object type" in the future -# since other classes implicitly inherit from it. class object: () +!@@concat.typecheck.builtin_object object + class bool: () @@ -54,7 +54,9 @@ class str: () def index(--) @cast (py_function[(str Optional[int] Optional[int]), int]): - () + () + +!@@concat.typecheck.builtin_str str def eval(--) @cast (py_function[(str), object]): () @@ -92,3 +94,5 @@ class list[`element]: def __iter__(--) @cast (py_function[(), iterator[`element]]): () + +!@@concat.typecheck.builtin_list list diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 686c0e7..b26b411 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -42,7 +42,7 @@ def is_subtype_of(self, supertype: 'Type') -> bool: return ( supertype is self or isinstance(self, IndividualType) - and supertype is object_type + and supertype is get_object_type() ) def __le__(self, other: object) -> bool: @@ -183,6 +183,9 @@ def __gt__(self, other) -> bool: """Comparator for storing variables in OrderedSets.""" return id(self) > id(other) + def __eq__(self, other) -> bool: + return id(self) == id(other) + class IndividualVariable(_Variable, IndividualType): def __init__(self) -> None: @@ -196,7 +199,7 @@ def constrain_and_bind_supertype_variables( ) -> 'Substitutions': from concat.typecheck import Substitutions - if self is supertype or supertype == object_type: + if self is supertype or supertype == get_object_type(): return Substitutions() if not isinstance(supertype, IndividualType): raise TypeError( @@ -233,8 +236,8 @@ def constrain_and_bind_subtype_variables( if self is supertype: return Substitutions() - if supertype == object_type: - return Substitutions({self: object_type}) + if supertype == get_object_type(): + return Substitutions({self: get_object_type()}) if not isinstance(supertype, IndividualType): raise TypeError( '{} must be an individual type: expected {}'.format( @@ -1115,7 +1118,7 @@ def is_subtype_of(self, supertype: 'Type') -> bool: return super().is_subtype_of(supertype) if self._arity != supertype._arity: return False - if self._arity == 0 and supertype is object_type: + if self._arity == 0 and supertype is get_object_type(): return True if supertype._nominal and self._head is not supertype._head: return False @@ -1196,7 +1199,7 @@ def constrain_and_bind_supertype_variables( '{} is not as polymorphic as {}'.format(self, supertype) ) # every object type is a subtype of object_type - if supertype == object_type: + if supertype == get_object_type(): return Substitutions() # Don't forget that there's nominal subtyping too. if supertype._nominal: @@ -1292,7 +1295,7 @@ def constrain_and_bind_subtype_variables( '{} is not as polymorphic as {}'.format(self, supertype) ) # every object type is a subtype of object_type - if supertype == object_type: + if supertype == get_object_type(): return Substitutions() # Don't forget that there's nominal subtyping too. if supertype._nominal: @@ -1337,7 +1340,6 @@ def constrain_and_bind_subtype_variables( type = instantiated_self.get_type_of_attribute(name) # FIXME: Really types of attributes should not be higher-kinded type = type.instantiate() - # print('in constrain_and_bind_subtype_variables, objecttype', repr(type)) sub = type.constrain_and_bind_subtype_variables( supertype.get_type_of_attribute(name).instantiate(), rigid_variables, @@ -2113,8 +2115,46 @@ def _mapping_to_str(mapping: Mapping) -> str: float_type = ObjectType(_x, {}, nominal=True) no_return_type = _NoReturnType() -object_type = ObjectType(_x, {}, nominal=True) -object_type.set_internal_name('object_type') + + +_object_type: Optional[Type] = None + + +def get_object_type() -> Type: + assert _object_type is not None + return _object_type + + +def set_object_type(ty: Type) -> None: + global _object_type + _object_type = ty + + +_list_type: Optional[Type] = None + + +def get_list_type() -> Type: + assert _list_type is not None + return _list_type + + +def set_list_type(ty: Type) -> None: + global _list_type + _list_type = ty + + +_str_type: Optional[Type] = None + + +def get_str_type() -> Type: + assert _str_type is not None + return _str_type + + +def set_str_type(ty: Type) -> None: + global _str_type + _str_type = ty + _arg_type_var = SequenceVariable() _return_type_var = IndividualVariable() @@ -2149,7 +2189,9 @@ def _mapping_to_str(mapping: Mapping) -> str: _x, { '__add__': py_function_type[ - TypeSequence([object_type]), _add_result_type + # FIXME: object should be the parameter type + TypeSequence([_x]), + _add_result_type, ] }, [_add_result_type], @@ -2183,7 +2225,8 @@ def _mapping_to_str(mapping: Mapping) -> str: ) lt_comparable_type.set_internal_name('lt_comparable_type') -_int_add_type = py_function_type[TypeSequence([object_type]), _x] +# FIXME: The parameter type should be object. +_int_add_type = py_function_type[TypeSequence([_x]), _x] int_type = ObjectType( _x, @@ -2278,62 +2321,6 @@ def _mapping_to_str(mapping: Mapping) -> str: ) slice_type.set_internal_name('slice_type') -_element_type_var = IndividualVariable() -_list_getitem_type = py_function_type[ - TypeSequence([int_type]), _element_type_var -].with_overload((slice_type[(optional_type[int_type,],) * 3],), _x) -list_type = ObjectType( - _x, - { - '__getitem__': _list_getitem_type, - '__iter__': py_function_type[ - TypeSequence([]), iterator_type[_element_type_var,] - ], - }, - [_element_type_var], - nominal=True, -) -list_type.set_internal_name('list_type') - -_str_getitem_type = py_function_type[ - TypeSequence([int_type]), _x -].with_overload( - [ - slice_type[ - optional_type[int_type,], - optional_type[int_type,], - optional_type[int_type,], - ] - ], - _x, -) -str_type = ObjectType( - _x, - { - '__getitem__': _str_getitem_type, - '__add__': py_function_type[TypeSequence([object_type]), _x], - # FIXME: str doesn't have an __radd__. I added it only to help with - # type checking. - '__radd__': py_function_type[TypeSequence([object_type]), _x], - 'find': py_function_type[ - TypeSequence( - [_x, optional_type[int_type,], optional_type[int_type,]] - ), - int_type, - ], - 'join': py_function_type[TypeSequence([_x, iterable_type[_x,]]), _x], - '__iter__': py_function_type[TypeSequence([]), iterator_type[_x,]], - 'index': py_function_type[ - TypeSequence( - [_x, optional_type[int_type,], optional_type[int_type,]] - ), - int_type, - ], - }, - nominal=True, -) -str_type.set_internal_name('str_type') - ellipsis_type = ObjectType(_x, {}) not_implemented_type = ObjectType(_x, {}) From dc5267c9fbcb57869c07ee56abc25de3e5134aa2 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Sat, 25 May 2024 17:43:23 -0700 Subject: [PATCH 10/61] Fix order of variables in generalization --- concat/linked_list.py | 108 +++++++++++++++++++++++++++++++ concat/orderedset.py | 104 ++++++++++++++++++++++++++--- concat/tests/test_linked_list.py | 85 ++++++++++++++++++++++++ concat/tests/test_orderedset.py | 37 +++++++++++ concat/typecheck/__init__.py | 4 +- concat/typecheck/types.py | 51 +++++++-------- 6 files changed, 350 insertions(+), 39 deletions(-) create mode 100644 concat/linked_list.py create mode 100644 concat/tests/test_linked_list.py create mode 100644 concat/tests/test_orderedset.py diff --git a/concat/linked_list.py b/concat/linked_list.py new file mode 100644 index 0000000..c528713 --- /dev/null +++ b/concat/linked_list.py @@ -0,0 +1,108 @@ +from typing import ( + Any, + Callable, + Iterator, + List, + Optional, + Reversible, + Sequence, + Tuple, + TypeVar, + Union, + overload, + cast, +) +from typing_extensions import Never + +_T_co = TypeVar('_T_co', covariant=True) +_T = TypeVar('_T') + + +class LinkedList(Sequence[_T_co]): + def __init__( + self, _val: Optional[Tuple[_T_co, 'LinkedList[_T_co]']] + ) -> None: + self._val = _val + if _val is None: + self._length = 0 + else: + self._length = 1 + len(_val[1]) + + @classmethod + def from_iterable(cls, iterable: Reversible[_T_co]) -> 'LinkedList[_T_co]': + if isinstance(iterable, cls): + return iterable + l: LinkedList[_T_co] = empty_list + for el in reversed(iterable): + l = cls((el, l)) + return l + + @overload + def __getitem__(self, i: int) -> _T_co: + pass + + @overload + def __getitem__(self, i: slice) -> 'LinkedList[_T_co]': + pass + + def __getitem__( + self, i: Union[slice, int] + ) -> Union['LinkedList[_T_co]', _T_co]: + if isinstance(i, slice): + raise NotImplementedError + for _ in range(i): + if self._val is None: + raise IndexError + self = self._val[1] + if self._val is None: + raise IndexError + return self._val[0] + + def __len__(self) -> int: + return self._length + + def __add__(self, other: object) -> 'LinkedList[Any]': + if not isinstance(other, LinkedList): + return NotImplemented + for el in reversed(list(self)): + other = LinkedList((el, other)) + return other + + def filter(self, p: Callable[[_T_co], bool]) -> 'LinkedList[_T_co]': + if self._val is None: + return self + # FIXME: Stack safety + # TODO: Reuse as much of tail as possible + if p(self._val[0]): + tail = self._val[1].filter(p) + return LinkedList((self._val[0], tail)) + return self._val[1].filter(p) + + def _tails(self) -> 'List[LinkedList[_T_co]]': + res: List[LinkedList[_T_co]] = [] + while self._val is not None: + res.append(self) + self = self._val[1] + return res + + def __iter__(self) -> Iterator[_T_co]: + while self._val is not None: + yield self._val[0] + self = self._val[1] + + def __str__(self) -> str: + return str(list(self)) + + def __repr__(self) -> str: + return f'LinkedList.from_iterable({list(self)!r})' + + def __eq__(self, other: object) -> bool: + if not isinstance(other, LinkedList): + return NotImplemented + for a, b in zip(self, other): + if a != b: + return False + return True + + +empty_list = LinkedList[Never](None) diff --git a/concat/orderedset.py b/concat/orderedset.py index 4a36d8b..8195ffe 100644 --- a/concat/orderedset.py +++ b/concat/orderedset.py @@ -1,4 +1,16 @@ -from typing import AbstractSet, Any, Iterable, Iterator, Tuple, TypeVar +from concat.linked_list import LinkedList, empty_list +from typing import ( + AbstractSet, + Any, + Generic, + Iterable, + Iterator, + Optional, + Reversible, + Tuple, + TypeVar, + Union, +) _T = TypeVar('_T', covariant=True) @@ -8,7 +20,7 @@ def __init__(self, elements: Iterable[_T]) -> None: super().__init__() self._data = _Tree23.from_iterable(elements) - def __sub__(self, other: object) -> 'OrderedSet[_T]': + def __sub__(self, other: object) -> 'OrderedSet[Any]': if not isinstance(other, AbstractSet): return NotImplemented data = self._data @@ -16,7 +28,7 @@ def __sub__(self, other: object) -> 'OrderedSet[_T]': data = data.delete(el) return OrderedSet(data) - def __or__(self, other: object) -> 'OrderedSet[_T]': + def __or__(self, other: object) -> 'OrderedSet[Any]': if not isinstance(other, AbstractSet): return NotImplemented data = self._data @@ -30,10 +42,71 @@ def __contains__(self, element: object) -> bool: def __iter__(self) -> Iterator[_T]: return iter(self._data) + def __reversed__(self) -> Iterator[_T]: + return reversed(self._data) + def __len__(self) -> int: return len(self._data) +# Inspired by Java's LinkedHashSet +# https://github.com/anjbur/java-immutable-collections/blob/master/src/main/java/org/javimmutable/collections/inorder/JImmutableInsertOrderSet.java +class InsertionOrderedSet(AbstractSet[_T]): + def __init__( + self, + elements: 'Reversible[_T]', + _order: Optional[LinkedList[_T]] = None, + ) -> None: + super().__init__() + self._data = OrderedSet(elements) + self._order = ( + LinkedList.from_iterable(elements) if _order is None else _order + ) + + def __sub__(self, other: object) -> 'InsertionOrderedSet[_T]': + if not isinstance(other, AbstractSet): + return NotImplemented + data = self._data - other + new_set = InsertionOrderedSet[_T]( + data, self._order.filter(lambda x: x not in other) + ) + return new_set + + def __or__(self, other: object) -> 'InsertionOrderedSet[Any]': + if not isinstance(other, AbstractSet): + return NotImplemented + data = self._data | other + if isinstance(other, InsertionOrderedSet): + order = self._order + other._order + else: + order = self._order + LinkedList.from_iterable(list(other)) + order = filter_duplicates(order) + new_set = InsertionOrderedSet(data, order) + return new_set + + def __contains__(self, element: object) -> bool: + return element in self._data + + def __iter__(self) -> Iterator[_T]: + return iter(self._order) + + def __len__(self) -> int: + return len(self._data) + + +def filter_duplicates(xs: LinkedList[_T]) -> LinkedList[_T]: + found = set() + + def predicate(x: _T) -> bool: + nonlocal found + if x in found: + return False + found.add(x) + return True + + return xs.filter(predicate) + + class _Tree23Hole: pass @@ -113,14 +186,23 @@ def search(self, d) -> Tuple[bool, Any]: def __iter__(self) -> Iterator: if self.is_leaf(): return - if self.is_2_node(): - yield from self._data[0] - yield self._data[1] - yield from self._data[2] + yield from self._data[0] + yield self._data[1] + yield from self._data[2] if self.is_3_node(): yield self._data[3] yield from self._data[4] + def __reversed__(self) -> Iterator: + if self.is_leaf(): + return + if self.is_3_node(): + yield from reversed(self._data[4]) + yield self._data[3] + yield from reversed(self._data[2]) + yield self._data[1] + yield from reversed(self._data[0]) + def __len__(self) -> int: if self.is_leaf(): return 0 @@ -385,7 +467,7 @@ def _delete_upwards_phase(self) -> '_Tree23': return _Tree23( (a, w, _Tree23((b, x, c)), y, _Tree23((d, z, e))) ) - # 3-node that either has no in data or children, or has bad heights + # 3-node that either has no data or children, or has bad heights return self if self.is_2_node(): left, x, right = self._data @@ -426,7 +508,7 @@ def delete(self, key) -> '_Tree23': return tree._data[1] return tree - def max(self) -> Any: + def max(self, default: object = None) -> Any: tree = self while not tree.is_leaf(): if tree.is_2_node(): @@ -437,7 +519,9 @@ def max(self) -> Any: if tree._is_3_node_terminal(): return tree._data[3] tree = tree._data[4] - raise ValueError('Empty 2-3 tree has no max') + if default is None: + raise ValueError('Empty 2-3 tree has no max') + return default __hole = _Tree23Hole() diff --git a/concat/tests/test_linked_list.py b/concat/tests/test_linked_list.py new file mode 100644 index 0000000..8826d51 --- /dev/null +++ b/concat/tests/test_linked_list.py @@ -0,0 +1,85 @@ +from concat.linked_list import LinkedList, empty_list +from hypothesis import example, given +import hypothesis.strategies as st +from typing import Callable, List +import unittest + + +linked_lists = st.lists(st.integers()).map(LinkedList.from_iterable) + + +class TestMonoid(unittest.TestCase): + def test_empty(self) -> None: + self.assertEqual(0, len(empty_list)) + self.assertFalse(empty_list) + self.assertListEqual([], list(empty_list)) + + @given(st.lists(st.integers()), st.lists(st.integers())) + def test_add(self, a: List[int], b: List[int]) -> None: + self.assertListEqual( + a + b, + list(LinkedList.from_iterable(a) + LinkedList.from_iterable(b)), + ) + + @given(linked_lists, linked_lists, linked_lists) + def test_assoc( + self, a: LinkedList[int], b: LinkedList[int], c: LinkedList[int] + ) -> None: + self.assertEqual((a + b) + c, a + (b + c)) + + @given(linked_lists) + def test_id(self, a: LinkedList[int]) -> None: + self.assertEqual(a, a + empty_list) + self.assertEqual(a, empty_list + a) + + +predicates = st.functions( + like=lambda _: True, returns=st.booleans(), pure=True +) + + +class TestFilter(unittest.TestCase): + @given(predicates) + def test_empty(self, p: Callable[[int], bool]) -> None: + self.assertEqual(empty_list, empty_list.filter(p)) + + @given(linked_lists) + def test_remove_all(self, l: LinkedList[int]) -> None: + self.assertEqual(empty_list, l.filter(lambda _: False)) + + @given(linked_lists) + def test_keep_all(self, l: LinkedList[int]) -> None: + self.assertEqual(l, l.filter(lambda _: True)) + + @given(linked_lists, predicates) + def test_idempotency( + self, l: LinkedList[int], p: Callable[[int], bool] + ) -> None: + self.assertEqual(l.filter(p), l.filter(p).filter(p)) + + @given(linked_lists, predicates) + def test_excluded_middle( + self, l: LinkedList[int], p: Callable[[int], bool] + ) -> None: + self.assertSetEqual( + set(l), set(l.filter(p) + l.filter(lambda x: not p(x))) + ) + + @given(linked_lists, predicates) + def test_subset( + self, l: LinkedList[int], p: Callable[[int], bool] + ) -> None: + self.assertLessEqual(set(l.filter(p)), set(l)) + + @given(linked_lists, predicates) + def test_forward_observable_order( + self, l: LinkedList[int], p: Callable[[int], bool] + ) -> None: + observed_order = [] + + def pred(x: int) -> bool: + observed_order.append(x) + return p(x) + + l.filter(pred) + self.assertListEqual(list(l), observed_order) diff --git a/concat/tests/test_orderedset.py b/concat/tests/test_orderedset.py new file mode 100644 index 0000000..7c4f2ab --- /dev/null +++ b/concat/tests/test_orderedset.py @@ -0,0 +1,37 @@ +from concat.orderedset import InsertionOrderedSet +from hypothesis import given # type: ignore +import hypothesis.strategies as st # type: ignore +from typing import List, Set +import unittest + + +class TestInsertionOrderedSet(unittest.TestCase): + @given(st.sets(st.integers()).map(list)) + def test_insertion_from_list_preserves_order(self, l: List[int]) -> None: + self.assertListEqual(l, list(InsertionOrderedSet(l))) + + @given(st.sets(st.integers()), st.sets(st.integers())) + def test_set_difference_preserves_order( + self, original: Set[int], to_remove: Set[int] + ) -> None: + insertion_order_set = InsertionOrderedSet[int](list(original)) + expected_order = list( + x for x in insertion_order_set if x not in to_remove + ) + insertion_order_set -= to_remove + actual_order = list(insertion_order_set) + self.assertListEqual(expected_order, actual_order) + + @given(st.sets(st.integers()), st.sets(st.integers())) + def test_union_preserves_order( + self, original: Set[int], to_add: Set[int] + ) -> None: + insertion_order_set = InsertionOrderedSet[int](list(original)) + insertion_order_to_add = InsertionOrderedSet[int](list(to_add)) + expected_order = list(insertion_order_set) + for x in insertion_order_to_add: + if x not in expected_order: + expected_order.append(x) + insertion_order_set = insertion_order_set | insertion_order_to_add + actual_order = list(insertion_order_set) + self.assertListEqual(expected_order, actual_order) diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index 4ecaa4a..a481386 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -138,7 +138,7 @@ def __hash__(self) -> int: if TYPE_CHECKING: import concat.astutils - from concat.orderedset import OrderedSet + from concat.orderedset import InsertionOrderedSet from concat.typecheck.types import _Variable @@ -219,7 +219,7 @@ def copy(self) -> 'Environment': def apply_substitution(self, sub: 'Substitutions') -> 'Environment': return Environment({name: sub(t) for name, t in self.items()}) - def free_type_variables(self) -> 'OrderedSet[_Variable]': + def free_type_variables(self) -> 'InsertionOrderedSet[_Variable]': return free_type_variables_of_mapping(self) def resolve_forward_references(self) -> None: diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index b26b411..210fe0a 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -1,4 +1,4 @@ -from concat.orderedset import OrderedSet +from concat.orderedset import InsertionOrderedSet import concat.typecheck import functools from typing import ( @@ -34,7 +34,7 @@ class Type(abc.ABC): def __init__(self) -> None: self._free_type_variables_cached: Optional[ - OrderedSet[_Variable] + InsertionOrderedSet[_Variable] ] = None # TODO: Fully replace with <=. @@ -72,10 +72,10 @@ def attributes(self) -> Mapping[str, 'Type']: pass @abc.abstractmethod - def _free_type_variables(self) -> OrderedSet['_Variable']: + def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: pass - def free_type_variables(self) -> OrderedSet['_Variable']: + def free_type_variables(self) -> InsertionOrderedSet['_Variable']: if self._free_type_variables_cached is None: self._free_type_variables_cached = self._free_type_variables() return self._free_type_variables_cached @@ -172,8 +172,8 @@ def apply_substitution( return result # type: ignore return self - def _free_type_variables(self) -> OrderedSet['_Variable']: - return OrderedSet({self}) + def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: + return InsertionOrderedSet([self]) def __lt__(self, other) -> bool: """Comparator for storing variables in OrderedSets.""" @@ -628,8 +628,8 @@ def constrain_and_bind_subtype_variables( '{} must be a sequence type, not {}'.format(self, supertype) ) - def _free_type_variables(self) -> OrderedSet['_Variable']: - ftv: OrderedSet[_Variable] = OrderedSet([]) + def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: + ftv: InsertionOrderedSet[_Variable] = InsertionOrderedSet([]) for t in self: ftv |= t.free_type_variables() return ftv @@ -697,12 +697,12 @@ def __init__( ) -> None: for ty in input_types[1:]: if ty.kind != IndividualKind(): - raise concat.typeheck.TypeError( + raise concat.typecheck.TypeError( f'{ty} must be an individual type' ) for ty in output_types[1:]: if ty.kind != IndividualKind(): - raise concat.typeheck.TypeError( + raise concat.typecheck.TypeError( f'{ty} must be an individual type' ) super().__init__() @@ -713,10 +713,11 @@ def __iter__(self) -> Iterator['TypeSequence']: return iter((self.input, self.output)) def generalized_wrt(self, gamma: 'Environment') -> Type: + parameters = list( + self.free_type_variables() - gamma.free_type_variables() + ) return ObjectType( - IndividualVariable(), - {'__call__': self,}, - list(self.free_type_variables() - gamma.free_type_variables()), + IndividualVariable(), {'__call__': self,}, parameters, ) def __hash__(self) -> int: @@ -828,7 +829,7 @@ def constrain_and_bind_subtype_variables( )(sub) return sub - def _free_type_variables(self) -> OrderedSet['_Variable']: + def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: return ( self.input.free_type_variables() | self.output.free_type_variables() @@ -962,8 +963,8 @@ def apply_substitution( def free_type_variables_of_mapping( attributes: Mapping[str, Type] -) -> OrderedSet[_Variable]: - ftv: OrderedSet[_Variable] = OrderedSet([]) +) -> InsertionOrderedSet[_Variable]: + ftv: InsertionOrderedSet[_Variable] = InsertionOrderedSet([]) for sigma in attributes.values(): ftv |= sigma.free_type_variables() return ftv @@ -1178,16 +1179,12 @@ def constrain_and_bind_supertype_variables( ) ) - # To support higher-rank polymorphism, polymorphic types are subtypes - # of their instances. - if isinstance(supertype, StackEffect): subtyping_assumptions.append((self, supertype)) - instantiated_self = self.instantiate() # We know instantiated_self is not a type constructor here, so # there's no need to worry about variable binding - return instantiated_self.get_type_of_attribute( + return self.get_type_of_attribute( '__call__' ).constrain_and_bind_supertype_variables( supertype, rigid_variables, subtyping_assumptions @@ -1349,7 +1346,7 @@ def constrain_and_bind_subtype_variables( def get_type_of_attribute(self, attribute: str) -> IndividualType: if attribute not in self._attributes: - raise AttributeError(self, attribute) + raise concat.typecheck.AttributeError(self, attribute) self_sub = concat.typecheck.Substitutions({self._self_type: self}) @@ -1367,7 +1364,7 @@ def __repr__(self) -> str: None if self._head is self else self._head, ) - def _free_type_variables(self) -> OrderedSet[_Variable]: + def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: ftv = free_type_variables_of_mapping(self.attributes) for arg in self.type_arguments: ftv |= arg.free_type_variables() @@ -1835,8 +1832,8 @@ def __getitem__(self, args: Sequence[Type]) -> 'PythonFunctionType': def attributes(self) -> Mapping[str, 'Type']: raise TypeError('py_overloaded does not have attributes') - def _free_type_variables(self) -> OrderedSet['_Variable']: - return OrderedSet([]) + def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: + return InsertionOrderedSet([]) def apply_substitution( self, _: 'concat.typecheck.Substitutions' @@ -2086,8 +2083,8 @@ def constrain_and_bind_supertype_variables( 'Supertypes of type are not known before its definition' ) - def _free_type_variables(self) -> OrderedSet[_Variable]: - return OrderedSet([]) + def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: + return InsertionOrderedSet([]) @property def kind(self) -> Kind: From 333e8160ddf5c3beb19fd105e7268a723dab0530 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Wed, 29 May 2024 20:59:48 -0700 Subject: [PATCH 11/61] Ignore flake8 shadowing warning --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index aad4e97..cb4c9eb 100644 --- a/tox.ini +++ b/tox.ini @@ -21,3 +21,6 @@ passenv = CARGO_BUILD_TARGET setenv = PYTHONWARNINGS = default + +[flake8] +ignore = F402 From 57c0991cd34501de788c8de0d89d698ae4b6a136 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Wed, 29 May 2024 21:27:58 -0700 Subject: [PATCH 12/61] Fix LinkedList.__eq__ --- concat/linked_list.py | 7 ++++--- tox.ini | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/concat/linked_list.py b/concat/linked_list.py index c528713..ab3a9cc 100644 --- a/concat/linked_list.py +++ b/concat/linked_list.py @@ -1,5 +1,4 @@ from typing import ( - Any, Callable, Iterator, List, @@ -10,7 +9,6 @@ TypeVar, Union, overload, - cast, ) from typing_extensions import Never @@ -61,7 +59,7 @@ def __getitem__( def __len__(self) -> int: return self._length - def __add__(self, other: object) -> 'LinkedList[Any]': + def __add__(self, other: 'LinkedList[_T_co]') -> 'LinkedList[_T_co]': if not isinstance(other, LinkedList): return NotImplemented for el in reversed(list(self)): @@ -96,9 +94,12 @@ def __str__(self) -> str: def __repr__(self) -> str: return f'LinkedList.from_iterable({list(self)!r})' + # "supertype defines the argument type as object" def __eq__(self, other: object) -> bool: if not isinstance(other, LinkedList): return NotImplemented + if len(self) != len(other): + return False for a, b in zip(self, other): if a != b: return False diff --git a/tox.ini b/tox.ini index cb4c9eb..d9a42c5 100644 --- a/tox.ini +++ b/tox.ini @@ -23,4 +23,4 @@ setenv = PYTHONWARNINGS = default [flake8] -ignore = F402 +ignore = E741, F402 From 272905c79be7d098b188765f890ad875aabfddd3 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Wed, 29 May 2024 22:17:32 -0700 Subject: [PATCH 13/61] Fix parsing in the REPL after adding error tolerance --- concat/parse.py | 6 +++++- concat/stdlib/repl.py | 2 ++ concat/tests/stdlib/test_repl.py | 3 ++- concat/tests/test_parse.py | 3 +++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/concat/parse.py b/concat/parse.py index 0df932d..a083b18 100644 --- a/concat/parse.py +++ b/concat/parse.py @@ -57,11 +57,15 @@ def word_ext(parsers): class Node(abc.ABC): - @abc.abstractmethod def __init__(self): self.location = (0, 0) self.children: Iterable[Node] = [] + def assert_no_parse_errors(self) -> None: + failures = list(self.parsing_failures) + if failures: + raise concat.parser_combinators.ParseError(failures) + @property def parsing_failures( self, diff --git a/concat/stdlib/repl.py b/concat/stdlib/repl.py index 71b9e38..e7e214d 100644 --- a/concat/stdlib/repl.py +++ b/concat/stdlib/repl.py @@ -77,8 +77,10 @@ def read_form(stack: List[object], stash: List[object]) -> None: string = _read_until_complete_line() try: ast = _parse('$(' + string + ')') + ast.assert_no_parse_errors() except concat.parser_combinators.ParseError: ast = _parse(string) + ast.assert_no_parse_errors() concat.typecheck.check(caller_globals['@@extra_env'], ast.children) # I don't think it makes sense for us to get multiple children if what # we got was a statement, so we assert. diff --git a/concat/tests/stdlib/test_repl.py b/concat/tests/stdlib/test_repl.py index e30bcce..9b66bb9 100644 --- a/concat/tests/stdlib/test_repl.py +++ b/concat/tests/stdlib/test_repl.py @@ -3,6 +3,7 @@ import sys import contextlib import concat.parse +import concat.parser_combinators import concat.typecheck from concat.typecheck.types import SequenceVariable, StackEffect, TypeSequence import concat.stdlib.types @@ -62,5 +63,5 @@ def test_catch_parse_errors(self): with replace_stdin(io.StringIO('drg nytu y,i.')): try: concat.stdlib.repl.repl([], []) - except concat.parse.ParseError: + except concat.parser_combinators.ParseError: self.fail('repl must recover from parser failures') diff --git a/concat/tests/test_parse.py b/concat/tests/test_parse.py index 3a28725..cfb5d35 100644 --- a/concat/tests/test_parse.py +++ b/concat/tests/test_parse.py @@ -23,6 +23,9 @@ def test_examples(self) -> None: >> concat.parse.token('NAME').many() >> concat.parse.token('RPAR') ) + parsers['type-variable'] = concat.parser_combinators.fail( + 'not implemented' + ) # for example programs, we only test acceptance From 4b251be221b5c2eae2f2a258ba82847eab8a4517 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Wed, 29 May 2024 23:29:13 -0700 Subject: [PATCH 14/61] Improve .many() error messages by being able to commit to an alternative --- concat/parser_combinators/__init__.py | 70 ++++++++++++++++++++------- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/concat/parser_combinators/__init__.py b/concat/parser_combinators/__init__.py index ea257d4..ef18190 100644 --- a/concat/parser_combinators/__init__.py +++ b/concat/parser_combinators/__init__.py @@ -92,6 +92,7 @@ def __init__( current_index: int, is_success: bool, failures: Optional[FailureTree] = None, + is_committed: bool = False, ) -> None: self.output = output self.current_index = current_index @@ -101,9 +102,10 @@ def __init__( ) self.is_success = is_success self.failures = failures + self.is_committed = is_committed def __repr__(self) -> str: - return f'{type(self).__qualname__}({self.output!r}, {self.current_index!r}, {self.is_success!r}, {self.failures!r})' + return f'{type(self).__qualname__}({self.output!r}, {self.current_index!r}, {self.is_success!r}, {self.failures!r}, is_committed={self.is_committed!r})' def __eq__(self, other: object) -> bool: if isinstance(other, Result): @@ -112,17 +114,25 @@ def __eq__(self, other: object) -> bool: self.current_index, self.is_success, self.failures, + self.is_committed, ) == ( other.output, other.current_index, other.is_success, other.failures, + other.is_committed, ) return NotImplemented def __hash__(self) -> int: return hash( - (self.output, self.current_index, self.is_success, self.failures) + ( + self.output, + self.current_index, + self.is_success, + self.failures, + self.is_committed, + ) ) @@ -142,29 +152,37 @@ def new_parser( stream: Sequence[_T_contra], index: int ) -> Result[Union[_U_co, _V]]: left_result = self(stream, index) - if left_result.is_success: + if ( + left_result.is_success + or left_result.is_committed + and left_result.current_index > index + ): return left_result right_result = other(stream, index) new_failure: Optional[FailureTree] if right_result.is_success: if left_result.current_index > right_result.current_index: - if ( - left_result.failures is not None - and right_result.failures is not None - ): + if right_result.failures is not None: + assert left_result.failures is not None new_failure = FailureTree( f'{left_result.failures.expected} or {right_result.failures.expected}', left_result.failures.furthest_index, left_result.failures.children + right_result.failures.children, ) - return Result( - right_result.output, - right_result.current_index, - True, - new_failure, - ) - raise Exception('todo') + else: + new_failure = left_result.failures + return Result( + right_result.output, + right_result.current_index, + True, + new_failure, + ) + return right_result + if ( + right_result.is_committed + and right_result.current_index > index + ): return right_result assert left_result.failures is not None assert right_result.failures is not None @@ -321,14 +339,30 @@ def new_parser() -> Generator: return new_parser def optional(self) -> 'Parser[_T_contra, Optional[_U_co]]': + return self | success(None) + + # See + # https://elixirforum.com/t/parser-combinators-how-to-know-when-many-should-return-an-error/46048/12 + # on the problem this solves. + def commit(self) -> 'Parser[_T_contra, _U_co]': + """Do not try alteratives adjacent to this parser if it consumes input before failure. + + This is useful with combinators like many(): parser.many() succeeds + even if `parser` fails in a way that you know is an error and should be + reported for a better error message.""" + @Parser def new_parser( stream: Sequence[_T_contra], index: int - ) -> Result[Optional[_U_co]]: + ) -> Result[_U_co]: result = self(stream, index) - if result.is_success: - return result - return Result(None, index, True, result.failures) + return Result( + result.output, + result.current_index, + result.is_success, + result.failures, + is_committed=True, + ) return new_parser From 4aa49aa20908fd0ffa401ddcf290eab350540b85 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Thu, 30 May 2024 18:24:42 -0700 Subject: [PATCH 15/61] Use Python function as argument to cont.map in map_cont --- concat/stdlib/continuation.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/concat/stdlib/continuation.py b/concat/stdlib/continuation.py index 12bdc6e..c53980b 100644 --- a/concat/stdlib/continuation.py +++ b/concat/stdlib/continuation.py @@ -1,11 +1,7 @@ from concat.common_types import ConcatFunction from concat.typecheck.types import ( - ForAll, IndividualVariable, SequenceVariable, - StackEffect, - TypeSequence, - continuation_monad_type, ) from typing import Any, Callable, Generic, List, NoReturn, Type, TypeVar, cast @@ -135,12 +131,12 @@ def map_cont(stack: List[Any], stash: List[Any]) -> None: cast(ContinuationMonad, stack.pop()), ) - def python_function(b: _B) -> _C: + def python_function(b: Any) -> Any: stack.append(b) f(stack, stash) return stack.pop() - result = cont.map(f) + result = cont.map(python_function) stack.append(result) @@ -156,7 +152,7 @@ def python_function(b: _B) -> ContinuationMonad[_R, _C]: f(stack, stash) return cast(ContinuationMonad[_R, _C], stack.pop()) - result = cont.bind(python_function) + result: ContinuationMonad = cont.bind(python_function) stack.append(result) @@ -173,5 +169,5 @@ def concat_k(stack: List[object], _: List[object]) -> None: concat_run(stack, stash) return cast(_R, stack.pop()) - result = ContinuationMonad(python_run) + result: ContinuationMonad = ContinuationMonad(python_run) stack.append(result) From 19fe9ffe73cd0223d648547caa70c080696aaeb7 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Fri, 31 May 2024 00:51:13 -0700 Subject: [PATCH 16/61] Add preamble type stubs --- concat/stdlib/pyinterop/__init__.py | 8 +- concat/tests/strategies.py | 47 +- concat/tests/test_typecheck.py | 79 +- concat/tests/typecheck/__init__.py | 0 concat/tests/typecheck/test_types.py | 108 ++ concat/typecheck/__init__.py | 110 +- concat/typecheck/preamble.cati | 88 ++ concat/typecheck/preamble_types.py | 279 +---- concat/typecheck/py_builtins.cati | 2 +- concat/typecheck/types.py | 1669 ++++++++++---------------- 10 files changed, 989 insertions(+), 1401 deletions(-) create mode 100644 concat/tests/typecheck/__init__.py create mode 100644 concat/tests/typecheck/test_types.py create mode 100644 concat/typecheck/preamble.cati diff --git a/concat/stdlib/pyinterop/__init__.py b/concat/stdlib/pyinterop/__init__.py index fb7ee67..c06c189 100644 --- a/concat/stdlib/pyinterop/__init__.py +++ b/concat/stdlib/pyinterop/__init__.py @@ -2,7 +2,7 @@ from concat.common_types import ConcatFunction import concat.stdlib.ski from concat.typecheck.types import ( - ForAll, + GenericType, IndividualVariable, ObjectType, SequenceVariable, @@ -43,7 +43,7 @@ _y = IndividualVariable() _z = IndividualVariable() globals()['@@types'] = { - 'getitem': ForAll( + 'getitem': GenericType( [_stack_type_var, _x, _y], StackEffect( TypeSequence([_stack_type_var, subscriptable_type[_x, _y], _x,]), @@ -69,7 +69,7 @@ }, [_rest_var, _y, _z], ), - 'to_dict': ForAll( + 'to_dict': GenericType( [_stack_type_var, _x, _y], StackEffect( TypeSequence( @@ -83,7 +83,7 @@ TypeSequence([_stack_type_var, dict_type[_x, _y]]), ), ), - 'to_slice': ForAll( + 'to_slice': GenericType( [_stack_type_var, _x, _y, _z], StackEffect( TypeSequence( diff --git a/concat/tests/strategies.py b/concat/tests/strategies.py index 13399d2..932c06d 100644 --- a/concat/tests/strategies.py +++ b/concat/tests/strategies.py @@ -1,38 +1,38 @@ -from concat.lex import Token from concat.typecheck.types import ( IndividualType, IndividualVariable, ObjectType, + PythonFunctionType, SequenceVariable, StackEffect, TypeSequence, + none_type, + optional_type, + py_function_type, ) -from hypothesis.strategies import ( +from hypothesis.strategies import ( # type: ignore SearchStrategy, builds, dictionaries, from_type, - just, lists, none, recursive, register_type_strategy, text, + tuples, ) -from typing import ( - Iterable, - Sequence, - Type, -) +from typing import Type def _type_sequence_strategy( individual_type_strategy: SearchStrategy[IndividualType], + no_rest_var: bool = False, ) -> SearchStrategy[TypeSequence]: return builds( lambda maybe_seq_var, rest: TypeSequence(maybe_seq_var + rest), - lists(from_type(SequenceVariable), max_size=1), - lists(individual_type_strategy, max_size=10), + lists(from_type(SequenceVariable), max_size=0 if no_rest_var else 1), + lists(individual_type_strategy, max_size=5), ) @@ -44,27 +44,15 @@ def _object_type_strategy( ObjectType, attributes=dictionaries(text(), individual_type_strategy), nominal_supertypes=lists(individual_type_strategy), - _type_arguments=lists( - from_type(SequenceVariable) - | individual_type_strategy - | _type_sequence_strategy(individual_type_strategy) - ), _head=none(), - _other_kwargs=just({}), ), lambda children: builds( ObjectType, attributes=dictionaries(text(), individual_type_strategy), nominal_supertypes=lists(individual_type_strategy), - _type_arguments=lists( - from_type(SequenceVariable) - | individual_type_strategy - | _type_sequence_strategy(individual_type_strategy) - ), _head=children, - _other_kwargs=just({}), ), - max_leaves=50, + max_leaves=10, ) @@ -93,6 +81,19 @@ def _mark_individual_type_strategy( ) | _mark_individual_type_strategy( _object_type_strategy(children), ObjectType + ) + | _mark_individual_type_strategy( + builds( + lambda args: py_function_type[args], + tuples( + _type_sequence_strategy(children, no_rest_var=True), children + ), + ), + PythonFunctionType, + ) + | _mark_individual_type_strategy( + builds(lambda args: optional_type[args], tuples(children),), + type(optional_type[none_type,]), ), max_leaves=50, ) diff --git a/concat/tests/test_typecheck.py b/concat/tests/test_typecheck.py index d4c7b70..971e496 100644 --- a/concat/tests/test_typecheck.py +++ b/concat/tests/test_typecheck.py @@ -11,18 +11,16 @@ StackEffect, Type as ConcatType, TypeSequence, - int_type, - dict_type, + addable_type, ellipsis_type, - file_type, float_type, - list_type, - none_type, + get_object_type, + int_type, no_return_type, + none_type, not_implemented_type, - object_type, + optional_type, py_function_type, - str_type, ) import concat.typecheck.preamble_types import concat.astutils @@ -41,6 +39,10 @@ ) +default_env = concat.typecheck.load_builtins_and_preamble() +default_env.resolve_forward_references() + + def lex_string(string: str) -> List[concat.lex.Token]: return lex.tokenize(string) @@ -84,14 +86,14 @@ def test_attribute_word(self, attr_word) -> None: def test_add_operator_inference(self, a: int, b: int) -> None: try_prog = '{!r} {!r} +\n'.format(a, b) tree = parse(try_prog) - _, type, _ = concat.typecheck.infer( - concat.typecheck.Environment( - {'+': concat.typecheck.preamble_types.types['+']} - ), + sub, type, _ = concat.typecheck.infer( + concat.typecheck.Environment({'+': default_env['+']}), tree.children, is_top_level=True, ) - note(str(type)) + note(repr(type)) + note(str(sub)) + note(repr(default_env['+'])) self.assertEqual( type, StackEffect(TypeSequence([]), TypeSequence([int_type])) ) @@ -101,7 +103,7 @@ def test_if_then_inference(self) -> None: tree = parse(try_prog) _, type, _ = concat.typecheck.infer( concat.typecheck.Environment( - concat.typecheck.preamble_types.types + {**default_env, **concat.typecheck.preamble_types.types,} ), tree.children, is_top_level=True, @@ -165,7 +167,7 @@ def seek_file(file:file offset:int whence:int --): ) ) env = concat.typecheck.Environment( - concat.typecheck.preamble_types.types + {**default_env, **concat.typecheck.preamble_types.types,} ) concat.typecheck.infer(env, tree.children, None, True) # If we get here, we passed @@ -174,7 +176,9 @@ def test_cast_word(self) -> None: """Test that the type checker properly checks casts.""" tree = parse('"str" cast (int)') _, type, _ = concat.typecheck.infer( - Environment(concat.typecheck.preamble_types.types), + Environment( + {**default_env, **concat.typecheck.preamble_types.types} + ), tree.children, is_top_level=True, ) @@ -199,8 +203,8 @@ class TestStackEffectParser(unittest.TestCase): TypeSequence([_a_bar, _b]), TypeSequence([_a_bar]) ), 'a:object b:object -- b a': StackEffect( - TypeSequence([_a_bar, object_type, object_type,]), - TypeSequence([_a_bar, *[object_type] * 2]), + TypeSequence([_a_bar, get_object_type(), get_object_type(),]), + TypeSequence([_a_bar, *[get_object_type()] * 2]), ), 'a:`t -- a a': StackEffect( TypeSequence([_a_bar, _b]), TypeSequence([_a_bar, _b, _b]) @@ -232,9 +236,13 @@ def test_examples(self) -> None: effect = build_parsers()['stack-effect-type'].parse(tokens) except concat.parser_combinators.ParseError as e: self.fail(f'could not parse {effect_string}\n{e}') - env = Environment(concat.typecheck.preamble_types.types) + env = Environment( + {**default_env, **concat.typecheck.preamble_types.types} + ) actual = effect.to_type(env)[0].generalized_wrt(env) expected = self.examples[example].generalized_wrt(env) + print(str(actual)) + print(str(expected)) self.assertEqual( actual, expected, ) @@ -306,19 +314,19 @@ def test_int_not_subtype_of_float(self) -> None: def test_stack_effect_subtyping(self, type1, type2) -> None: fun1 = StackEffect(TypeSequence([type1]), TypeSequence([type2])) fun2 = StackEffect( - TypeSequence([no_return_type]), TypeSequence([object_type]) + TypeSequence([no_return_type]), TypeSequence([get_object_type()]) ) - self.assertLessEqual(fun1, fun2) + self.assertTrue(fun1.is_subtype_of(fun2)) @given(from_type(IndividualType)) @settings(suppress_health_check=(HealthCheck.filter_too_much,)) def test_no_return_is_bottom_type(self, type) -> None: - self.assertLessEqual(no_return_type, type) + self.assertTrue(no_return_type.is_subtype_of(type)) @given(from_type(IndividualType)) @settings(suppress_health_check=(HealthCheck.filter_too_much,)) def test_object_is_top_type(self, type) -> None: - self.assertLessEqual(type, object_type) + self.assertTrue(type.is_subtype_of(get_object_type())) __attributes_generator = dictionaries( text(max_size=25), from_type(IndividualType), max_size=5 # type: ignore @@ -337,7 +345,7 @@ def test_object_structural_subtyping( x1, x2 = IndividualVariable(), IndividualVariable() object1 = ObjectType(x1, {**other_attributes, **attributes}) object2 = ObjectType(x2, attributes) - self.assertLessEqual(object1, object2) + self.assertTrue(object1.is_subtype_of(object2)) @given(__attributes_generator, __attributes_generator) @settings(suppress_health_check=(HealthCheck.filter_too_much,)) @@ -347,14 +355,14 @@ def test_class_structural_subtyping( x1, x2 = IndividualVariable(), IndividualVariable() object1 = ClassType(x1, {**other_attributes, **attributes}) object2 = ClassType(x2, attributes) - self.assertLessEqual(object1, object2) + self.assertTrue(object1.is_subtype_of(object2)) @given(from_type(StackEffect)) @settings(suppress_health_check=(HealthCheck.filter_too_much,)) def test_object_subtype_of_stack_effect(self, effect) -> None: x = IndividualVariable() object = ObjectType(x, {'__call__': effect}) - self.assertLessEqual(object, effect) + self.assertTrue(object.is_subtype_of(effect)) @given(from_type(IndividualType), from_type(IndividualType)) @settings( @@ -367,7 +375,7 @@ def test_object_subtype_of_py_function(self, type1, type2) -> None: x = IndividualVariable() py_function = py_function_type[TypeSequence([type1]), type2] object = ObjectType(x, {'__call__': py_function}) - self.assertLessEqual(object, py_function) + self.assertTrue(object.is_subtype_of(py_function)) @given(from_type(StackEffect)) def test_class_subtype_of_stack_effect(self, effect) -> None: @@ -377,7 +385,7 @@ def test_class_subtype_of_stack_effect(self, effect) -> None: TypeSequence([*effect.input, x]), effect.output ) cls = ClassType(x, {'__init__': unbound_effect}) - self.assertLessEqual(cls, effect) + self.assertTrue(cls.is_subtype_of(effect)) @given(from_type(IndividualType), from_type(IndividualType)) @settings( @@ -391,4 +399,19 @@ def test_class_subtype_of_py_function(self, type1, type2) -> None: py_function = py_function_type[TypeSequence([type1]), type2] unbound_py_function = py_function_type[TypeSequence([x, type1]), type2] cls = ClassType(x, {'__init__': unbound_py_function}) - self.assertLessEqual(cls, py_function) + self.assertTrue(cls.is_subtype_of(py_function)) + + @given(from_type(IndividualType)) + def test_none_subtype_of_optional(self, ty: IndividualType) -> None: + opt_ty = optional_type[ + ty, + ] + self.assertTrue(none_type.is_subtype_of(opt_ty)) + + @given(from_type(IndividualType)) + def test_type_subtype_of_optional(self, ty: IndividualType) -> None: + opt_ty = optional_type[ + ty, + ] + note(str(ty)) + self.assertTrue(ty.is_subtype_of(opt_ty)) diff --git a/concat/tests/typecheck/__init__.py b/concat/tests/typecheck/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/concat/tests/typecheck/test_types.py b/concat/tests/typecheck/test_types.py new file mode 100644 index 0000000..c3e67fd --- /dev/null +++ b/concat/tests/typecheck/test_types.py @@ -0,0 +1,108 @@ +from concat.typecheck import ( + TypeError as ConcatTypeError, + load_builtins_and_preamble, +) +from concat.typecheck.types import ( + IndividualVariable, + ObjectType, + SequenceVariable, + StackEffect, + TypeSequence, + addable_type, + int_type, + py_function_type, +) +import unittest + + +load_builtins_and_preamble() + + +class TestIndividualVariableConstrain(unittest.TestCase): + def test_individual_variable_subtype(self) -> None: + v = IndividualVariable() + ty = int_type + sub = v.constrain_and_bind_variables(ty, set(), []) + self.assertEqual(ty, sub(v)) + + def test_individual_variable_supertype(self) -> None: + v = IndividualVariable() + ty = int_type + sub = ty.constrain_and_bind_variables(v, set(), []) + self.assertEqual(ty, sub(v)) + + def test_attribute_subtype(self) -> None: + v = IndividualVariable() + attr_ty = ObjectType(IndividualVariable(), {'__add__': v}) + ty = int_type + with self.assertRaises(ConcatTypeError): + attr_ty.constrain_and_bind_variables(ty, set(), []) + + def test_attribute_supertype(self) -> None: + v = IndividualVariable() + attr_ty = ObjectType(IndividualVariable(), {'__add__': v}) + ty = int_type + sub = ty.constrain_and_bind_variables(attr_ty, set(), []) + self.assertEqual(ty.get_type_of_attribute('__add__'), sub(v)) + + def test_py_function_return_subtype(self) -> None: + v = IndividualVariable() + py_fun_ty = py_function_type[TypeSequence([int_type]), v] + ty = int_type.get_type_of_attribute('__add__') + sub = py_fun_ty.constrain_and_bind_variables(ty, set(), []) + self.assertEqual(int_type, sub(v)) + + def test_py_function_return_supertype(self) -> None: + v = IndividualVariable() + py_fun_ty = py_function_type[TypeSequence([int_type]), v] + ty = int_type.get_type_of_attribute('__add__') + sub = ty.constrain_and_bind_variables(py_fun_ty, set(), []) + self.assertEqual(int_type, sub(v)) + + def test_type_sequence_subtype(self) -> None: + v = IndividualVariable() + seq_ty = TypeSequence([v]) + ty = TypeSequence([int_type]) + sub = seq_ty.constrain_and_bind_variables(ty, set(), []) + self.assertEqual(int_type, sub(v)) + + def test_type_sequence_supertype(self) -> None: + v = IndividualVariable() + seq_ty = TypeSequence([v]) + ty = TypeSequence([int_type]) + sub = ty.constrain_and_bind_variables(seq_ty, set(), []) + self.assertEqual(int_type, sub(v)) + + def test_int_addable(self) -> None: + v = IndividualVariable() + sub = int_type.constrain_and_bind_variables( + addable_type[v, v], set(), [] + ) + self.assertEqual(int_type, sub(v)) + + def test_int__add__addable__add__(self) -> None: + v = IndividualVariable() + int_add = int_type.get_type_of_attribute('__add__') + addable_add = addable_type[v, v].get_type_of_attribute('__add__') + sub = int_add.constrain_and_bind_variables(addable_add, set(), []) + print(v) + print(int_add) + print(addable_add) + print(sub) + self.assertEqual(int_type, sub(v)) + + +class TestSequenceVariableConstrain(unittest.TestCase): + def test_stack_effect_input_subtype(self) -> None: + v = SequenceVariable() + effect_ty = StackEffect(TypeSequence([v]), TypeSequence([])) + ty = StackEffect(TypeSequence([]), TypeSequence([])) + sub = effect_ty.constrain_and_bind_variables(ty, set(), []) + self.assertEqual(TypeSequence([]), sub(v)) + + def test_stack_effect_input_supertype(self) -> None: + v = SequenceVariable() + effect_ty = StackEffect(TypeSequence([v]), TypeSequence([])) + ty = StackEffect(TypeSequence([]), TypeSequence([])) + sub = ty.constrain_and_bind_variables(effect_ty, set(), []) + self.assertEqual(TypeSequence([]), sub(v)) diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index a481386..0756574 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -7,7 +7,6 @@ import collections.abc from typing import ( - Any, Callable, Dict, Generator, @@ -23,7 +22,6 @@ TypeVar, Union, cast, - overload, ) from typing_extensions import Protocol @@ -37,7 +35,12 @@ def apply_substitution(self, sub: 'Substitutions') -> _Result: class Substitutions(collections.abc.Mapping, Mapping['_Variable', 'Type']): - def __init__(self, sub: Iterable[Tuple['_Variable', 'Type']] = {}) -> None: + def __init__( + self, + sub: Union[ + Iterable[Tuple['_Variable', 'Type']], Mapping['_Variable', 'Type'] + ] = {}, + ) -> None: self._sub = dict(sub) for variable, ty in self._sub.items(): if variable.kind != ty.kind: @@ -86,8 +89,8 @@ def __hash__(self) -> int: from concat.typecheck.types import ( - ForAll, ForwardTypeReference, + GenericType, GenericTypeKind, IndividualKind, IndividualType, @@ -107,7 +110,6 @@ def __hash__(self) -> int: free_type_variables_of_mapping, get_list_type, get_object_type, - init_primitives, int_type, invertible_type, iterable_type, @@ -227,25 +229,48 @@ def resolve_forward_references(self) -> None: self[name] = t.resolve_forward_references() +def load_builtins_and_preamble() -> Environment: + env = _check_stub( + pathlib.Path(__file__).with_name('py_builtins.cati'), is_builtins=True, + ) + env = Environment( + { + **env, + **_check_stub( + pathlib.Path(__file__).with_name('preamble.cati'), + is_preamble=True, + ), + } + ) + return env + + def check( environment: Environment, program: concat.astutils.WordsOrStatements, source_dir: str = '.', _should_check_bodies: bool = True, _should_load_builtins: bool = True, + _should_load_preamble: bool = True, ) -> Environment: import concat.typecheck.preamble_types + builtins_stub_env = Environment() + preamble_stub_env = Environment() if _should_load_builtins: builtins_stub_env = _check_stub( pathlib.Path(__file__).with_name('py_builtins.cati'), is_builtins=True, ) - else: - builtins_stub_env = Environment() + if _should_load_preamble: + preamble_stub_env = _check_stub( + pathlib.Path(__file__).with_name('preamble.cati'), + is_preamble=True, + ) environment = Environment( { **builtins_stub_env, + **preamble_stub_env, **concat.typecheck.preamble_types.types, **environment, } @@ -391,7 +416,7 @@ def infer( elif isinstance(node, concat.parse.ListWordNode): phi = S collected_type = o - element_type: IndividualType = object_type + element_type: IndividualType = no_return_type for item in node.list_children: phi1, fun_type, _ = infer( phi(gamma), @@ -404,7 +429,7 @@ def infer( collected_type = fun_type.output # FIXME: Infer the type of elements in the list based on # ALL the elements. - if element_type == object_type: + if element_type == no_return_type: assert isinstance(collected_type[-1], IndividualType) element_type = collected_type[-1] # drop the top of the stack to use as the item @@ -507,11 +532,8 @@ def infer( # NOTE: To continue the "bidirectional" bent, we will require a ghg # type annotation. # TODO: Make the return types optional? - print('node', repr(node.stack_effect)) declared_type, _ = node.stack_effect.to_type(S(gamma)) - print('type from node', declared_type) declared_type = S(declared_type) - print('subbed', declared_type) recursion_env = gamma.copy() if not isinstance(declared_type, StackEffect): raise TypeError( @@ -531,13 +553,11 @@ def infer( # the declared outputs. Thus, inferred_type.output should be a subtype # declared_type.output. try: - S = inferred_type.output.constrain_and_bind_subtype_variables( + S = inferred_type.output.constrain_and_bind_variables( declared_type.output, S(recursion_env).free_type_variables(), [], - )( - S - ) + )(S) except TypeError as e: message = ( 'declared function type {} is not compatible with ' @@ -596,9 +616,11 @@ def infer( node.value, type_of_name, type_of_name ) ) - constraint_subs = o1.constrain_and_bind_supertype_variables( + constraint_subs = o1.constrain_and_bind_variables( type_of_name.input, set(), [] ) + print(repr(o1)) + print(constraint_subs) current_subs = constraint_subs(current_subs) current_effect = current_subs( StackEffect(i1, type_of_name.output) @@ -608,9 +630,9 @@ def infer( # make sure any annotation matches the current stack if quotation.input_stack_type is not None: input_stack, _ = quotation.input_stack_type.to_type(gamma) - S = o.constrain_and_bind_supertype_variables( - input_stack, set(), [] - )(S) + S = o.constrain_and_bind_variables(input_stack, set(), [])( + S + ) else: input_stack = o S1, (i1, o1), _ = infer( @@ -645,7 +667,7 @@ def infer( node.value, attr_function_type, attr_function_type ) ) - R = out_types.constrain_and_bind_supertype_variables( + R = out_types.constrain_and_bind_variables( attr_function_type.input, set(), [] ) current_subs, current_effect = ( @@ -684,14 +706,25 @@ def infer( initial_stack=TypeSequence([]), check_bodies=check_bodies, ) - gamma[node.class_name] = ObjectType( - IndividualVariable(), - body_attrs, - type_parameters, - (), - True, - is_variadic=node.is_variadic, + # TODO: Introduce scopes into the environment object + body_attrs = Environment( + { + name: ty + for name, ty in body_attrs.items() + if name not in temp_gamma + } + ) + ty = ObjectType( + self_type=IndividualVariable(), + attributes=body_attrs, + nominal_supertypes=(), + nominal=True, ) + if type_parameters: + ty = GenericType( + type_parameters, ty, is_variadic=node.is_variadic + ) + gamma[node.class_name] = ty gamma[node.class_name].set_internal_name(node.class_name) # elif isinstance(node, concat.parse.TypeAliasStatementNode): # gamma[node.name], _ = node.type_node.to_type(gamma) @@ -707,7 +740,7 @@ def infer( @functools.lru_cache(maxsize=None) def _check_stub_resolved_path( - path: pathlib.Path, is_builtins: bool = False + path: pathlib.Path, is_builtins: bool = False, is_preamble: bool = False ) -> 'Environment': try: source = path.read_text() @@ -742,15 +775,16 @@ def _check_stub_resolved_path( str(path.parent), _should_check_bodies=False, _should_load_builtins=not is_builtins, + _should_load_preamble=not is_preamble and not is_builtins, ) def _check_stub( - path: pathlib.Path, is_builtins: bool = False + path: pathlib.Path, is_builtins: bool = False, is_preamble: bool = False ) -> 'Environment': path = path.resolve() try: - return _check_stub_resolved_path(path, is_builtins) + return _check_stub_resolved_path(path, is_builtins, is_preamble) except StaticAnalysisError as e: e.set_path_if_missing(path) raise @@ -829,7 +863,7 @@ def to_type(self, env: Environment) -> Tuple[IndividualType, Environment]: arg_as_type, env = arg.to_type(env) args.append(arg_as_type) generic_type, env = self._generic_type.to_type(env) - if isinstance(generic_type, ObjectType): + if isinstance(generic_type, GenericType): if generic_type.is_variadic: args = (TypeSequence(args),) return generic_type[args], env @@ -1073,7 +1107,7 @@ def to_type(self, env: Environment) -> Tuple[Type, Environment]: parameter, temp_env = var.to_type(temp_env) variables.append(parameter) ty, _ = self._type.to_type(temp_env) - forall_type = ForAll(variables, ty) + forall_type = GenericType(variables, ty) return forall_type, env @@ -1114,7 +1148,7 @@ def non_star_name_parser() -> Generator: @concat.parser_combinators.generate def named_type_parser() -> Generator: - name_token = yield concat.parse.token('NAME') + name_token = yield non_star_name_parser return NamedTypeNode(name_token.start, name_token.value) @concat.parser_combinators.generate @@ -1175,6 +1209,8 @@ def stack_effect_type_sequence_parser() -> Generator: return TypeSequenceNode(location, seq_var_parsed, i) + parsers['stack-effect-type-sequence'] = stack_effect_type_sequence_parser + @concat.parser_combinators.generate def type_sequence_parser() -> Generator: type = parsers['type'] @@ -1268,8 +1304,6 @@ def object_type_parser() -> Generator: individual_type_variable_parser, ).desc('individual type') - # TODO: Parse sequence type variables - parsers['type'] = concat.parser_combinators.alt( # NOTE: There's a parsing ambiguity that might come back to bite me... forall_type_parser.desc('forall type'), @@ -1277,6 +1311,7 @@ def object_type_parser() -> Generator: concat.parse.token('LPAR') >> parsers.ref_parser('type-sequence') << concat.parse.token('RPAR'), + sequence_type_variable_parser, ) parsers['type-sequence'] = type_sequence_parser.desc('type sequence') @@ -1371,6 +1406,3 @@ def _ensure_type( if obj_name: known_stack_item_names[obj_name] = type return type, env - - -init_primitives() diff --git a/concat/typecheck/preamble.cati b/concat/typecheck/preamble.cati new file mode 100644 index 0000000..1c45a4d --- /dev/null +++ b/concat/typecheck/preamble.cati @@ -0,0 +1,88 @@ +def to_list(*rest_var i:iterable[`a_var] -- *rest_var l:list[`a_var]): + () + +def py_call(*rest_var kwargs:iterable[object] args:iterable[object] f:py_function[(*seq_var), `a_var] -- *rest_var res:`a_var): + () + +def nip(*rest_var a:object b:`a_var -- *rest_var b:`a_var): + () + +def nip_2(*rest_var a:object b:object c:`a_var -- *rest_var c:`a_var): + () + +def drop(*rest_var a:object -- *rest_var): + () + +def open(*rest_var kwargs:dict[str, object] path:str -- *rest_var f:file): + () + +def to_int(*stack_type_var base:Optional[int] x:object -- *stack_type_var i:int): + () + +# Addition type rules: +# Mypy allows specifying a restrictive operand type and leaves the whole +# NotImplemented thing up to the programmer. See linked_list.py for an example. +# FIXME: Make the rules safer... somehow +# ... a b => (... {__add__(t) -> s} t) +# --- +# a b + => (... s) +# ... a b => (... t {__radd__(t) -> s}) +# --- +# a b + => (... s) +# FIXME: Implement the second type rule +def +(*stack_type_var x:addable[`other, `c_var] y:`other -- *stack_type_var res:`c_var): + () + +# FIXME: We should check if the other operand supports __rsub__ if the +# first operand doesn't support __sub__. +def -(*stack_type_var x:subtractable[`b_var, `c_var] y:`b_var -- *stack_type_var res:`c_var): + () + +def is(*stack_type_var a:object b:object -- *stack_type_var res:bool): + () + +def and(*stack_type_var a:object b:object -- *stack_type_var res:bool): + () + +def or(*stack_type_var a:object b:object -- *stack_type_var res:bool): + () + +# TODO: I should be more careful here, since at least __eq__ can be +# deleted, if I remember correctly. +def ==(*stack_type_var a:object b:object -- *stack_type_var res:bool): + () + +def False(*rest_var -- *rest_var false:bool): + () + +def True(*rest_var -- *rest_var true:bool): + () + +def loop(*rest_var body:(*rest_var -- *rest_var flag:bool) -- *rest_var): + () + +# Rule 1: first operand has __ge__(type(second operand)) +# Rule 2: second operand has __le__(type(first operand)) +# FIXME: Implement the second type rule +def >=(*stack_type_var a:geq_comparable[`b_var] b:`b_var -- *stack_type_var res:bool): + () + +# Rule 1: first operand has __lt__(type(second operand)) +# Rule 2: second operand has __gt__(type(first operand)) +# FIXME: Implement the second type rule +# Also look at Python's note about when reflected method get's priority. +def <(*stack_type_var a:lt_comparable[`b_var] b:`b_var -- *stack_type_var res:bool): + () + +# FIXME: Implement the second type rule +def <=(*stack_type_var a:leq_comparable[`b_var] b:`b_var -- *stack_type_var res:bool): + () + +def choose(*rest_var b:bool t:(*rest_var -- *seq_var) f:(*rest_var -- *seq_var) -- *seq_var): + () + +def if_not(*rest_var b:bool body:(*rest_var -- *rest_var) -- *rest_var): + () + +def if_then(*rest_var b:bool body:(*rest_var -- *rest_var) -- *rest_var): + () diff --git a/concat/typecheck/preamble_types.py b/concat/typecheck/preamble_types.py index 585077a..c8c26d0 100644 --- a/concat/typecheck/preamble_types.py +++ b/concat/typecheck/preamble_types.py @@ -1,41 +1,29 @@ from concat.typecheck.types import ( + GenericType, IndividualVariable, SequenceVariable, - ForAll, - ObjectType, StackEffect, TypeSequence, addable_type, - base_exception_type, - bool_type, context_manager_type, - dict_type, ellipsis_type, file_type, - float_type, geq_comparable_type, - init_primitives, - int_type, iterable_type, iterator_type, leq_comparable_type, lt_comparable_type, - list_type, module_type, - none_type, no_return_type, + none_type, not_implemented_type, - object_type, optional_type, py_function_type, py_overloaded_type, - str_type, subscriptable_type, subtractable_type, - tuple_type, ) -init_primitives() _rest_var = SequenceVariable() _seq_var = SequenceVariable() @@ -47,94 +35,39 @@ _x = IndividualVariable() types = { - 'py_call': ForAll( - [_rest_var, _seq_var, _a_var], - StackEffect( - TypeSequence( - [ - _rest_var, - iterable_type[object_type,], - iterable_type[object_type,], - py_function_type[TypeSequence([_seq_var]), _a_var], - ] - ), - TypeSequence([_rest_var, _a_var]), - ), - ), - 'swap': ForAll( + 'addable': addable_type, + 'leq_comparable': leq_comparable_type, + 'lt_comparable': lt_comparable_type, + 'geq_comparable': geq_comparable_type, + 'swap': GenericType( [_rest_var, _a_var, _b_var], StackEffect( TypeSequence([_rest_var, _a_var, _b_var]), TypeSequence([_rest_var, _b_var, _a_var]), ), ), - 'pick': ForAll( + 'pick': GenericType( [_rest_var, _a_var, _b_var, _c_var], StackEffect( TypeSequence([_rest_var, _a_var, _b_var, _c_var]), TypeSequence([_rest_var, _a_var, _b_var, _c_var, _a_var]), ), ), - 'nip': ForAll( - [_rest_var, _a_var], - StackEffect( - TypeSequence([_rest_var, object_type, _a_var]), - TypeSequence([_rest_var, _a_var]), - ), - ), - 'nip_2': ObjectType( - _a_var, - { - '__call__': StackEffect( - TypeSequence([_rest_var, object_type, object_type, _b_var]), - TypeSequence([_rest_var, _b_var]), - ) - }, - [_rest_var, _b_var], - ), - 'drop': ForAll( - [_rest_var], - StackEffect( - TypeSequence([_rest_var, object_type]), TypeSequence([_rest_var]) - ), - ), - 'dup': ForAll( + 'dup': GenericType( [_rest_var, _a_var], StackEffect( TypeSequence([_rest_var, _a_var]), TypeSequence([_rest_var, _a_var, _a_var]), ), ), - 'open': ForAll( - [_rest_var], - StackEffect( - TypeSequence( - [_rest_var, dict_type[str_type, object_type], str_type] - ), - TypeSequence([_rest_var, file_type]), - ), - ), - 'over': ForAll( + 'over': GenericType( [_rest_var, _a_var, _b_var], StackEffect( TypeSequence([_rest_var, _a_var, _b_var]), TypeSequence([_rest_var, _a_var, _b_var, _a_var]), ), ), - 'to_list': ForAll( - [_rest_var], - StackEffect( - TypeSequence([_rest_var, iterable_type[_a_var,]]), - TypeSequence([_rest_var, list_type[_a_var,]]), - ), - ), - 'False': ForAll( - [_rest_var], - StackEffect( - TypeSequence([_rest_var]), TypeSequence([_rest_var, bool_type]) - ), - ), - 'curry': ForAll( + 'curry': GenericType( [_rest_var, _seq_var, _stack_var, _a_var], StackEffect( TypeSequence( @@ -157,16 +90,12 @@ ), ), ), - 'choose': ForAll( + 'call': GenericType( [_rest_var, _seq_var], StackEffect( TypeSequence( [ _rest_var, - bool_type, - StackEffect( - TypeSequence([_rest_var]), TypeSequence([_seq_var]) - ), StackEffect( TypeSequence([_rest_var]), TypeSequence([_seq_var]) ), @@ -175,87 +104,7 @@ TypeSequence([_seq_var]), ), ), - 'if_not': ForAll( - [_rest_var], - StackEffect( - TypeSequence( - [ - _rest_var, - bool_type, - StackEffect( - TypeSequence([_rest_var]), TypeSequence([_rest_var]) - ), - ] - ), - TypeSequence([_rest_var]), - ), - ), - 'if_then': ObjectType( - _x, - { - '__call__': StackEffect( - TypeSequence( - [ - _rest_var, - bool_type, - StackEffect( - TypeSequence([_rest_var]), - TypeSequence([_rest_var]), - ), - ] - ), - TypeSequence([_rest_var]), - ) - }, - [_rest_var], - ), - 'call': ObjectType( - _x, - { - '__call__': StackEffect( - TypeSequence( - [ - _rest_var, - StackEffect( - TypeSequence([_rest_var]), TypeSequence([_seq_var]) - ), - ] - ), - TypeSequence([_seq_var]), - ) - }, - [_rest_var, _seq_var], - ), - 'loop': ForAll( - [_rest_var], - StackEffect( - TypeSequence( - [ - _rest_var, - StackEffect( - TypeSequence([_rest_var]), - TypeSequence([_rest_var, bool_type]), - ), - ] - ), - TypeSequence([_rest_var]), - ), - ), - 'True': ObjectType( - _a_var, - { - '__call__': StackEffect( - TypeSequence([_rest_var]), TypeSequence([_rest_var, bool_type]) - ) - }, - [_rest_var], - ), # TODO: Separate type-check-time environment from runtime environment. - # XXX: generalize to_int over the stack - 'to_int': StackEffect( - TypeSequence([_stack_type_var, optional_type[int_type,], object_type]), - TypeSequence([_stack_type_var, int_type]), - ), 'iterable': iterable_type, 'NoReturn': no_return_type, 'subscriptable': subscriptable_type, @@ -268,128 +117,32 @@ 'Optional': optional_type, 'file': file_type, 'none': none_type, - 'None': ForAll( + 'None': GenericType( [_stack_type_var], StackEffect( TypeSequence([_stack_type_var]), TypeSequence([_stack_type_var, none_type]), ), ), - '...': ForAll( + '...': GenericType( [_stack_type_var], StackEffect( TypeSequence([_stack_type_var]), TypeSequence([_stack_type_var, ellipsis_type]), ), ), - 'Ellipsis': ForAll( + 'Ellipsis': GenericType( [_stack_type_var], StackEffect( TypeSequence([_stack_type_var]), TypeSequence([_stack_type_var, ellipsis_type]), ), ), - 'NotImplemented': ForAll( + 'NotImplemented': GenericType( [_stack_type_var], StackEffect( TypeSequence([_stack_type_var]), TypeSequence([_stack_type_var, not_implemented_type]), ), ), - # Addition type rules: - # require object_type because the methods should return - # NotImplemented for most types - # FIXME: Make the rules safer... somehow - # ... a b => (... {__add__(object) -> s} t) - # --- - # a b + => (... s) - # ... a b => (... t {__radd__(object) -> s}) - # --- - # a b + => (... s) - # FIXME: Implement the second type rule - '+': ForAll( - [_stack_type_var, _c_var], - StackEffect( - TypeSequence( - [_stack_type_var, addable_type[_c_var,], object_type] - ), - TypeSequence([_stack_type_var, _c_var]), - ), - ), - # FIXME: We should check if the other operand supports __rsub__ if the - # first operand doesn't support __sub__. - '-': ForAll( - [_stack_type_var, _b_var, _c_var], - StackEffect( - TypeSequence( - [_stack_type_var, subtractable_type[_b_var, _c_var], _b_var] - ), - TypeSequence([_stack_type_var, _c_var]), - ), - ), - # Rule 1: first operand has __ge__(type(second operand)) - # Rule 2: second operand has __le__(type(first operand)) - # FIXME: Implement the second type rule - '>=': ForAll( - [_stack_type_var, _b_var], - StackEffect( - TypeSequence( - [_stack_type_var, geq_comparable_type[_b_var,], _b_var] - ), - TypeSequence([_stack_type_var, bool_type]), - ), - ), - # Rule 1: first operand has __lt__(type(second operand)) - # Rule 2: second operand has __gt__(type(first operand)) - # FIXME: Implement the second type rule - # Also look at Python's note about when reflected method get's priority. - '<': ForAll( - [_stack_type_var, _b_var], - StackEffect( - TypeSequence( - [_stack_type_var, lt_comparable_type[_b_var,], _b_var] - ), - TypeSequence([_stack_type_var, bool_type]), - ), - ), - # FIXME: Implement the second type rule - '<=': ForAll( - [_stack_type_var, _b_var], - StackEffect( - TypeSequence( - [_stack_type_var, leq_comparable_type[_b_var,], _b_var] - ), - TypeSequence([_stack_type_var, bool_type]), - ), - ), - 'is': ForAll( - [_stack_type_var], - StackEffect( - TypeSequence([_stack_type_var, object_type, object_type]), - TypeSequence([_stack_type_var, bool_type]), - ), - ), - 'and': ForAll( - [_stack_type_var], - StackEffect( - TypeSequence([_stack_type_var, object_type, object_type]), - TypeSequence([_stack_type_var, bool_type]), - ), - ), - 'or': ForAll( - [_stack_type_var], - StackEffect( - TypeSequence([_stack_type_var, object_type, object_type]), - TypeSequence([_stack_type_var, bool_type]), - ), - ), - # TODO: I should be more careful here, since at least __eq__ can be - # deleted, if I remember correctly. - '==': ForAll( - [_stack_type_var], - StackEffect( - TypeSequence([_stack_type_var, object_type, object_type]), - TypeSequence([_stack_type_var, bool_type]), - ), - ), } diff --git a/concat/typecheck/py_builtins.cati b/concat/typecheck/py_builtins.cati index c1ee206..e8f75f5 100644 --- a/concat/typecheck/py_builtins.cati +++ b/concat/typecheck/py_builtins.cati @@ -7,7 +7,7 @@ class bool: () class int: - def __add__(--) @cast (py_function[(object), str]): + def __add__(--) @cast (py_function[(object), int]): () def __invert__(--) @cast (py_function[(), int]): diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 210fe0a..11ecf6b 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -1,19 +1,15 @@ from concat.orderedset import InsertionOrderedSet import concat.typecheck -import functools from typing import ( AbstractSet, - Callable, Dict, Iterable, Iterator, - Iterator, List, Mapping, NoReturn, Optional, Sequence, - Set, TYPE_CHECKING, Tuple, TypeVar, @@ -21,10 +17,9 @@ cast, overload, ) -from typing_extensions import Literal +from typing_extensions import Literal, Self import abc import collections.abc -from collections import defaultdict if TYPE_CHECKING: @@ -36,28 +31,29 @@ def __init__(self) -> None: self._free_type_variables_cached: Optional[ InsertionOrderedSet[_Variable] ] = None + self._internal_name: Optional[str] = None + self._forward_references_resolved = False - # TODO: Fully replace with <=. + # QUESTION: Do I need this? def is_subtype_of(self, supertype: 'Type') -> bool: - return ( - supertype is self - or isinstance(self, IndividualType) - and supertype is get_object_type() - ) + try: + sub = self.constrain_and_bind_variables(supertype, set(), []) + except concat.typecheck.TypeError: + return False + return not sub - def __le__(self, other: object) -> bool: - if not isinstance(other, Type): - return NotImplemented - return self.is_subtype_of(other) + # No <= implementation using subtyping, because variables overload that for + # sort by identity. def __eq__(self, other: object) -> bool: if self is other: return True if not isinstance(other, Type): return NotImplemented - return self <= other and other <= self + # QUESTION: Define == separately from is_subtype_of? + return self.is_subtype_of(other) and other.is_subtype_of(self) - def get_type_of_attribute(self, name: str) -> 'IndividualType': + def get_type_of_attribute(self, name: str) -> 'Type': raise AttributeError(self, name) def has_attribute(self, name: str) -> bool: @@ -87,22 +83,13 @@ def apply_substitution( pass @abc.abstractmethod - def constrain_and_bind_supertype_variables( + def constrain_and_bind_variables( self, supertype: 'Type', rigid_variables: AbstractSet['_Variable'], subtyping_assumptions: List[Tuple['IndividualType', 'IndividualType']], ) -> 'Substitutions': - pass - - @abc.abstractmethod - def constrain_and_bind_subtype_variables( - self, - supertype: 'Type', - rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple['IndividualType', 'IndividualType']], - ) -> 'Substitutions': - pass + raise NotImplementedError # QUESTION: Should I remove this? Should I not distinguish between subtype # and supertype variables in the other two constraint methods? I should @@ -110,7 +97,7 @@ def constrain_and_bind_subtype_variables( # Easy'? def constrain(self, supertype: 'Type') -> None: if not self.is_subtype_of(supertype): - raise TypeError( + raise concat.typecheck.TypeError( '{} is not a subtype of {}'.format(self, supertype) ) @@ -118,30 +105,146 @@ def instantiate(self) -> 'Type': return self @abc.abstractmethod - def resolve_forward_references(self) -> 'Type': - pass + def resolve_forward_references(self) -> Self: + self._forward_references_resolved = True + return self @abc.abstractproperty def kind(self) -> 'Kind': pass + def set_internal_name(self, name: str) -> None: + self._internal_name = name -class IndividualType(Type, abc.ABC): - def to_for_all(self) -> Type: - return ForAll([], self) + def __str__(self) -> str: + if self._internal_name is not None: + return self._internal_name + return super().__str__() - def is_subtype_of(self, supertype: Type) -> bool: - if isinstance(supertype, _OptionalType): - if ( - self == none_type - or not supertype.type_arguments - or isinstance(supertype.type_arguments[0], IndividualType) - and self.is_subtype_of(supertype.type_arguments[0]) - ): - return True - return False - return super().is_subtype_of(supertype) +class GenericType(Type): + def __init__( + self, + type_parameters: Sequence['_Variable'], + body: Type, + is_variadic: bool = False, + ) -> None: + super().__init__() + assert type_parameters + self._type_parameters = type_parameters + if body.kind != IndividualKind(): + raise concat.typecheck.TypeError( + f'Cannot be polymorphic over non-individual type {body}' + ) + self._body = body + self._instantiations: Dict[Tuple[Type, ...], Type] = {} + self.is_variadic = is_variadic + + def __str__(self) -> str: + if self._internal_name is not None: + return self._internal_name + if self.is_variadic: + params = str(self._type_parameters[0]) + '...' + else: + params = ' '.join(map(str, self._type_parameters)) + + return f'forall {params}. {self._body}' + + def __repr__(self) -> str: + return f'{type(self).__qualname__}({self._type_parameters!r}, {self._body!r}, is_variadic={self.is_variadic!r})' + + def __getitem__(self, type_arguments: 'TypeArguments') -> 'Type': + from concat.typecheck import Substitutions + + type_arguments = tuple(type_arguments) + if type_arguments in self._instantiations: + return self._instantiations[type_arguments] + expected_kinds = [var.kind for var in self._type_parameters] + actual_kinds = [ty.kind for ty in type_arguments] + if expected_kinds != actual_kinds: + raise concat.typecheck.TypeError( + f'A type argument to {self} has the wrong kind, type arguments: {type_arguments}, expected kinds: {expected_kinds}' + ) + sub = Substitutions(zip(self._type_parameters, type_arguments)) + instance = sub(self._body) + self._instantiations[type_arguments] = instance + if self._internal_name is not None: + instance_internal_name = self._internal_name + instance_internal_name += ( + '[' + ', '.join(map(str, type_arguments)) + ']' + ) + instance.set_internal_name(instance_internal_name) + return instance + + @property + def kind(self) -> 'Kind': + kinds = [var.kind for var in self._type_parameters] + return GenericTypeKind(kinds) + + def resolve_forward_references(self) -> 'GenericType': + self._body = self._body.resolve_forward_references() + return self + + def instantiate(self) -> Type: + fresh_vars: Sequence[_Variable] = [ + type(var)() for var in self._type_parameters + ] + return self[fresh_vars] + + def constrain_and_bind_variables( + self, + supertype: 'Type', + rigid_variables: AbstractSet['_Variable'], + subtyping_assumptions: List[Tuple['IndividualType', 'IndividualType']], + ) -> 'Substitutions': + from concat.typecheck import Substitutions + + if self is supertype or _contains_assumption( + subtyping_assumptions, self, supertype + ): + return Substitutions([]) + if self.kind != supertype.kind: + raise concat.typecheck.TypeError( + f'{self} has kind {self.kind} but {supertype} has kind {supertype.kind}' + ) + if not isinstance(supertype, GenericType): + raise NotImplementedError(supertype) + shared_vars = [type(var)() for var in self._type_parameters] + self_instance = self[shared_vars] + supertype_instance = supertype[shared_vars] + rigid_variables = ( + rigid_variables + | set(self._type_parameters) + | set(supertype._type_parameters) + ) + return self_instance.constrain_and_bind_variables( + supertype_instance, rigid_variables, subtyping_assumptions + ) + + def apply_substitution(self, sub: 'Substitutions') -> 'GenericType': + from concat.typecheck import Substitutions + + sub = Substitutions( + { + var: ty + for var, ty in sub.items() + if var not in self._type_parameters + } + ) + ty = GenericType(self._type_parameters, sub(self._body)) + return ty + + @property + def attributes(self) -> NoReturn: + raise concat.typecheck.TypeError( + 'Generic types do not have attributes; maybe you forgot type arguments?' + ) + + def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: + return self._body.free_type_variables() - set(self._type_parameters) + + +class IndividualType(Type, abc.ABC): def instantiate(self) -> 'IndividualType': return cast(IndividualType, super().instantiate()) @@ -155,6 +258,10 @@ def apply_substitution( def kind(self) -> 'Kind': return IndividualKind() + @property + def attributes(self) -> Mapping[str, Type]: + return {} + class _Variable(Type, abc.ABC): """Objects that represent type variables. @@ -191,7 +298,7 @@ class IndividualVariable(_Variable, IndividualType): def __init__(self) -> None: super().__init__() - def constrain_and_bind_supertype_variables( + def constrain_and_bind_variables( self, supertype: Type, rigid_variables: AbstractSet['_Variable'], @@ -199,58 +306,38 @@ def constrain_and_bind_supertype_variables( ) -> 'Substitutions': from concat.typecheck import Substitutions - if self is supertype or supertype == get_object_type(): + if self is supertype: return Substitutions() - if not isinstance(supertype, IndividualType): - raise TypeError( + if supertype.kind != IndividualKind(): + raise concat.typecheck.TypeError( '{} must be an individual type: expected {}'.format( supertype, self ) ) - if self in rigid_variables: - raise TypeError( - '{} is considered fixed here and cannot become a subtype of {}'.format( - self, supertype - ) - ) + mapping: Mapping[_Variable, Type] if ( isinstance(supertype, IndividualVariable) and supertype not in rigid_variables ): - return Substitutions({supertype: self}) - # Let's not support bounded quantification or inferring the types of - # named functions. Thus, the subtype constraint should fail here. - raise TypeError( - '{} is an individual type variable and cannot be a subtype of {}'.format( - self, supertype - ) - ) - - def constrain_and_bind_subtype_variables( - self, - supertype: Type, - rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple[IndividualType, IndividualType]], - ) -> 'Substitutions': - from concat.typecheck import Substitutions - - if self is supertype: - return Substitutions() - if supertype == get_object_type(): - return Substitutions({self: get_object_type()}) - if not isinstance(supertype, IndividualType): - raise TypeError( - '{} must be an individual type: expected {}'.format( - supertype, self + mapping = {supertype: self} + return Substitutions(mapping) + if isinstance(supertype, _OptionalType): + try: + return self.constrain_and_bind_variables( + supertype.type_arguments[0], + rigid_variables, + subtyping_assumptions, ) - ) - if self in rigid_variables: - raise TypeError( - '{} is considered fixed here and cannot become a subtype of {}'.format( - self, supertype + except concat.typecheck.TypeError: + return self.constrain_and_bind_variables( + none_type, rigid_variables, subtyping_assumptions ) + if self in rigid_variables: + raise concat.typecheck.TypeError( + f'{self} is considered fixed here and cannot become a subtype of {supertype}' ) - return Substitutions({self: supertype}) + mapping = {self: supertype} + return Substitutions(mapping) # __hash__ by object identity is used since that's the only way for two # type variables to be ==. @@ -270,7 +357,7 @@ def apply_substitution( @property def attributes(self) -> NoReturn: - raise TypeError( + raise concat.typecheck.TypeError( '{} is an individual type variable, so its attributes are unknown'.format( self ) @@ -294,7 +381,7 @@ def __str__(self) -> str: def __hash__(self) -> int: return hash(id(self)) - def constrain_and_bind_supertype_variables( + def constrain_and_bind_variables( self, supertype: Type, rigid_variables: AbstractSet['_Variable'], @@ -303,57 +390,34 @@ def constrain_and_bind_supertype_variables( from concat.typecheck import Substitutions if not isinstance(supertype, (SequenceVariable, TypeSequence)): - raise TypeError( - '{} must be a sequence type, not {}'.format(self, supertype) - ) - if self in rigid_variables: - raise Exception('todo') - # occurs check - if self is not supertype and self in supertype.free_type_variables(): - raise TypeError( - '{} cannot be a subtype of {} because it appears in {}'.format( - self, supertype, supertype - ) - ) - if isinstance(supertype, SequenceVariable): - return Substitutions({supertype: self}) - return Substitutions() - - def constrain_and_bind_subtype_variables( - self, - supertype: Type, - rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple['IndividualType', 'IndividualType']], - ) -> 'Substitutions': - from concat.typecheck import Substitutions - - if not isinstance(supertype, (SequenceVariable, TypeSequence)): - raise TypeError( + raise concat.typecheck.TypeError( '{} must be a sequence type, not {}'.format(self, supertype) ) if self in rigid_variables: - raise TypeError( + raise concat.typecheck.TypeError( '{} is fixed here and cannot become a subtype of another type'.format( self ) ) # occurs check if self is not supertype and self in supertype.free_type_variables(): - raise TypeError( + raise concat.typecheck.TypeError( '{} cannot be a subtype of {} because it appears in {}'.format( self, supertype, supertype ) ) - return Substitutions({self: supertype}) + if isinstance(supertype, SequenceVariable): + return Substitutions([(supertype, self)]) + return Substitutions([(self, supertype)]) def get_type_of_attribute(self, name: str) -> NoReturn: - raise TypeError( + raise concat.typecheck.TypeError( 'the sequence type {} does not hold attributes'.format(self) ) @property def attributes(self) -> NoReturn: - raise TypeError( + raise concat.typecheck.TypeError( 'the sequence type {} does not hold attributes'.format(self) ) @@ -384,47 +448,14 @@ def as_sequence(self) -> Sequence['StackItemType']: def apply_substitution(self, sub) -> 'TypeSequence': subbed_types: List[StackItemType] = [] for type in self: - subbed_type: Union[StackItemType, Sequence[StackItemType]] = sub( - type - ) + subbed_type: Union[StackItemType, TypeSequence] = sub(type) if isinstance(subbed_type, TypeSequence): subbed_types += [*subbed_type] else: subbed_types.append(subbed_type) return TypeSequence(subbed_types) - def is_subtype_of(self, supertype: Type) -> bool: - if ( - isinstance(supertype, SequenceVariable) - and not self._individual_types - and self._rest is supertype - ): - return True - elif isinstance(supertype, TypeSequence): - if self._is_empty() and supertype._is_empty(): - return True - elif not self._individual_types: - if ( - self._rest - and supertype._rest - and not supertype._individual_types - ): - return self._rest is supertype._rest - else: - return False - elif self._individual_types and supertype._individual_types: - if ( - not self._individual_types[-1] - <= supertype._individual_types[-1] - ): - return False - return self[:-1] <= supertype[:-1] - else: - return False - else: - return False - - def constrain_and_bind_supertype_variables( + def constrain_and_bind_variables( self, supertype: Type, rigid_variables: AbstractSet['_Variable'], @@ -432,200 +463,100 @@ def constrain_and_bind_supertype_variables( ) -> 'Substitutions': """Check that self is a subtype of supertype. - Free type variables that appear in the supertype type sequence are set - to be equal to their counterparts in the subtype sequence so that type + Free type variables that appear in either type sequence are set to be + equal to their counterparts in the other sequence so that type information can be propagated into calls of named functions. """ from concat.typecheck import Substitutions + if _contains_assumption(subtyping_assumptions, self, supertype): + return Substitutions() + if isinstance(supertype, SequenceVariable): supertype = TypeSequence([supertype]) if isinstance(supertype, TypeSequence): - if self._is_empty() and supertype._is_empty(): - return Substitutions() - elif ( - self._is_empty() - and supertype._rest - and not supertype._individual_types - and supertype._rest not in rigid_variables - ): - return Substitutions({supertype._rest: self}) - elif ( - supertype._is_empty() - and self._rest - and not self._individual_types - ): - raise concat.typecheck.StackMismatchError(self, supertype) - elif not self._individual_types: - if ( - self._is_empty() - and supertype._is_empty() - or self._rest is supertype._rest - ): + if self._is_empty(): + # [] <: [] + if supertype._is_empty(): return Substitutions() - if ( - self._rest - and supertype._rest - and not supertype._individual_types - and supertype._rest not in rigid_variables - ): - return Substitutions({supertype._rest: self._rest}) + # [] <: *a, *a is not rigid + # --> *a = [] elif ( self._is_empty() and supertype._rest and not supertype._individual_types and supertype._rest not in rigid_variables ): - return Substitutions({supertype._rest: self}) + return Substitutions([(supertype._rest, self)]) + # [] <: *a? `t0 `t... + # error else: raise concat.typecheck.StackMismatchError(self, supertype) - elif ( - not supertype._individual_types - and supertype._rest - and supertype._rest not in self.free_type_variables() - and supertype._rest not in rigid_variables - ): - return Substitutions({supertype._rest: self}) - elif self._individual_types and supertype._individual_types: - sub = self._individual_types[ - -1 - ].constrain_and_bind_supertype_variables( - supertype._individual_types[-1], - rigid_variables, - subtyping_assumptions, - ) - # constrain individual variables in the second sequence type to - # be *equal* to the corresponding type in the first sequence - # type. - is_variable = isinstance( - supertype._individual_types[-1], IndividualVariable - ) - if ( - is_variable - and supertype._individual_types[-1] not in rigid_variables - ): - sub = Substitutions( - { - supertype._individual_types[ - -1 - ]: self._individual_types[-1] - } - )(sub) - try: - sub = sub( - self[:-1] - ).constrain_and_bind_supertype_variables( - sub(supertype[:-1]), - rigid_variables, - subtyping_assumptions, - )( - sub - ) - return sub - except concat.typecheck.StackMismatchError: - raise concat.typecheck.StackMismatchError(self, supertype) - else: - # TODO: Add info about occurs check and rigid variables. - raise concat.typecheck.StackMismatchError(self, supertype) - else: - raise TypeError( - '{} must be a sequence type, not {}'.format(self, supertype) - ) - - def constrain_and_bind_subtype_variables( - self, - supertype: Type, - rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple['IndividualType', 'IndividualType']], - ) -> 'Substitutions': - from concat.typecheck import Substitutions - - if isinstance(supertype, SequenceVariable): - supertype = TypeSequence([supertype]) - - if isinstance(supertype, TypeSequence): - if self._is_empty() and supertype._is_empty(): - return Substitutions() - elif ( - self._is_empty() - and supertype._rest - and not supertype._individual_types - and supertype._rest not in rigid_variables - ): - raise concat.typecheck.StackMismatchError(self, supertype) - elif ( - supertype._is_empty() - and self._rest - and not self._individual_types - and self._rest not in rigid_variables - ): - return Substitutions({self._rest: supertype}) elif not self._individual_types: + # *a <: [], *a is not rigid + # --> *a = [] + if supertype._is_empty() and self._rest not in rigid_variables: + assert self._rest is not None + return Substitutions([(self._rest, supertype)]) + # *a <: *a if ( - self._is_empty() - and supertype._is_empty() - or self._rest is supertype._rest + self._rest is supertype._rest + and not supertype._individual_types ): return Substitutions() + # *a <: *b? `t..., *a is not rigid, *a is not free in RHS + # --> *a = RHS if ( self._rest and self._rest not in rigid_variables and self._rest not in supertype.free_type_variables() ): - return Substitutions({self._rest: supertype}) - elif ( - self._is_empty() - and supertype._rest - and not supertype._individual_types - ): - # QUESTION: Should this be allowed? I'm being defensive here. - raise concat.typecheck.StackMismatchError(self, supertype) + return Substitutions([(self._rest, supertype)]) else: raise concat.typecheck.StackMismatchError(self, supertype) - elif ( - not supertype._individual_types - and supertype._rest - and supertype._rest not in self.free_type_variables() - and supertype._rest not in rigid_variables - ): - raise concat.typecheck.StackMismatchError(self, supertype) - elif self._individual_types and supertype._individual_types: - sub = self._individual_types[ - -1 - ].constrain_and_bind_subtype_variables( - supertype._individual_types[-1], - rigid_variables, - subtyping_assumptions, - ) - is_variable = isinstance( - self._individual_types[-1], IndividualVariable - ) - if ( - is_variable - and self._individual_types[-1] not in rigid_variables + else: + # *a? `t... `t_n <: [] + # error + if supertype._is_empty(): + raise concat.typecheck.StackMismatchError(self, supertype) + # *a? `t... `t_n <: *b, *b is not rigid, *b is not free in LHS + # --> *b = LHS + elif ( + not supertype._individual_types + and supertype._rest + and supertype._rest not in self.free_type_variables() + and supertype._rest not in rigid_variables ): - sub = Substitutions( - { - self._individual_types[ - -1 - ]: supertype._individual_types[-1] - } - )(sub) - try: - sub = sub(self[:-1]).constrain_and_bind_subtype_variables( - sub(supertype[:-1]), + return Substitutions([(supertype._rest, self)]) + # `t_n <: `s_m *a? `t... <: *b? `s... + # --- + # *a? `t... `t_n <: *b? `s... `s_m + elif supertype._individual_types: + sub = self._individual_types[ + -1 + ].constrain_and_bind_variables( + supertype._individual_types[-1], rigid_variables, subtyping_assumptions, - )(sub) - return sub - except concat.typecheck.StackMismatchError: + ) + try: + sub = sub(self[:-1]).constrain_and_bind_variables( + sub(supertype[:-1]), + rigid_variables, + subtyping_assumptions, + )(sub) + return sub + except concat.typecheck.StackMismatchError: + # TODO: Add info about occurs check and rigid variables. + raise concat.typecheck.StackMismatchError( + self, supertype + ) + else: raise concat.typecheck.StackMismatchError(self, supertype) - else: - raise concat.typecheck.StackMismatchError(self, supertype) else: - raise TypeError( - '{} must be a sequence type, not {}'.format(self, supertype) + raise concat.typecheck.TypeError( + f'{self} is a sequence type, not {supertype}' ) def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: @@ -636,7 +567,7 @@ def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: @property def attributes(self) -> NoReturn: - raise TypeError( + raise concat.typecheck.TypeError( 'the sequence type {} does not hold attributes'.format(self) ) @@ -684,12 +615,6 @@ def kind(self) -> 'Kind': return SequenceKind() -# TODO: Actually get rid of ForAll uses. This is a temporary measure since I -# don't want to do that work right now. -def ForAll(type_parameters: Sequence['_Variable'], type: Type) -> Type: - return ObjectType(IndividualVariable(), type.attributes, type_parameters,) - - # TODO: Rename to StackEffect at all use sites. class _Function(IndividualType): def __init__( @@ -716,73 +641,13 @@ def generalized_wrt(self, gamma: 'Environment') -> Type: parameters = list( self.free_type_variables() - gamma.free_type_variables() ) - return ObjectType( - IndividualVariable(), {'__call__': self,}, parameters, - ) + return GenericType(parameters, self) def __hash__(self) -> int: # FIXME: Alpha equivalence return hash((self.input, self.output)) - def is_subtype_of( - self, - supertype: Type, - _sub: Optional['concat.typecheck.Substitutions'] = None, - ) -> bool: - if super().is_subtype_of(supertype): - return True - if isinstance(supertype, _Function): - if len(tuple(self.input)) != len(tuple(supertype.input)) or len( - tuple(self.output) - ) != len(tuple(supertype.output)): - return False - # Sequence variables are handled through renaming. - if _sub is None: - _sub = concat.typecheck.Substitutions() - input_rename_result = self._rename_sequence_variable( - tuple(self.input), tuple(supertype.input), _sub - ) - output_rename_result = self._rename_sequence_variable( - tuple(supertype.output), tuple(self.output), _sub - ) - if not (input_rename_result and output_rename_result): - return False - # TODO: What about individual type variables. We should be careful - # with renaming those, too. - # input types are contravariant - for type_from_self, type_from_supertype in zip( - self.input, supertype.input - ): - type_from_self, type_from_supertype = ( - cast(StackItemType, _sub(type_from_self)), - cast(StackItemType, _sub(type_from_supertype)), - ) - if isinstance(type_from_supertype, _Function): - if not type_from_supertype.is_subtype_of( - type_from_self, _sub - ): - return False - elif not type_from_supertype.is_subtype_of(type_from_self): - return False - # output types are covariant - for type_from_self, type_from_supertype in zip( - self.output, supertype.output - ): - type_from_self, type_from_supertype = ( - cast(StackItemType, _sub(type_from_self)), - cast(StackItemType, _sub(type_from_supertype)), - ) - if isinstance(type_from_self, _Function): - if not type_from_self.is_subtype_of( - type_from_supertype, _sub - ): - return False - elif not type_from_self.is_subtype_of(type_from_supertype): - return False - return True - return False - - def constrain_and_bind_supertype_variables( + def constrain_and_bind_variables( self, supertype: Type, rigid_variables: AbstractSet['_Variable'], @@ -790,41 +655,33 @@ def constrain_and_bind_supertype_variables( ) -> 'Substitutions': from concat.typecheck import Substitutions + if ( + self is supertype + or _contains_assumption(subtyping_assumptions, self, supertype) + or supertype is get_object_type() + ): + return Substitutions() + if ( isinstance(supertype, IndividualVariable) and supertype not in rigid_variables ): - return Substitutions({supertype: self}) - if not isinstance(supertype, StackEffect): - raise TypeError( - '{} is not a subtype of {}'.format(self, supertype) + return Substitutions([(supertype, self)]) + if isinstance(supertype, _OptionalType): + return self.constrain_and_bind_variables( + supertype.type_arguments[0], + rigid_variables, + subtyping_assumptions, ) - # Remember that the input should be contravariant! - # QUESTION: Constrain the supertype variables here during contravariance check? - sub = supertype.input.constrain_and_bind_subtype_variables( - self.input, rigid_variables, subtyping_assumptions - ) - sub = sub(self.output).constrain_and_bind_supertype_variables( - sub(supertype.output), rigid_variables, subtyping_assumptions - )(sub) - return sub - - def constrain_and_bind_subtype_variables( - self, - supertype: Type, - rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple[IndividualType, IndividualType]], - ) -> 'Substitutions': if not isinstance(supertype, StackEffect): - raise TypeError( + raise concat.typecheck.TypeError( '{} is not a subtype of {}'.format(self, supertype) ) # Remember that the input should be contravariant! - # QUESTION: Constrain the supertype variables here during contravariance check? - sub = supertype.input.constrain_and_bind_supertype_variables( + sub = supertype.input.constrain_and_bind_variables( self.input, rigid_variables, subtyping_assumptions ) - sub = sub(self.output).constrain_and_bind_subtype_variables( + sub = sub(self.output).constrain_and_bind_variables( sub(supertype.output), rigid_variables, subtyping_assumptions )(sub) return sub @@ -893,42 +750,7 @@ class QuotationType(_Function): def __init__(self, fun_type: _Function) -> None: super().__init__(fun_type.input, fun_type.output) - def is_subtype_of( - self, - supertype: Type, - _sub: Optional['concat.typecheck.Substitutions'] = None, - ) -> bool: - if super().is_subtype_of(supertype, _sub): - return True - if supertype == iterable_type: - return True - return False - - def constrain_and_bind_supertype_variables( - self, - supertype: Type, - rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple[IndividualType, IndividualType]], - ) -> 'Substitutions': - if ( - isinstance(supertype, ObjectType) - and supertype.head == iterable_type - ): - # FIXME: Don't present new variables every time. - # FIXME: Account for the types of the elements of the quotation. - in_var = IndividualVariable() - out_var = IndividualVariable() - quotation_iterable_type = iterable_type[ - StackEffect(TypeSequence([in_var]), TypeSequence([out_var])), - ] - return quotation_iterable_type.constrain_and_bind_supertype_variables( - supertype, rigid_variables, subtyping_assumptions - ) - return super().constrain_and_bind_supertype_variables( - supertype, rigid_variables, subtyping_assumptions - ) - - def constrain_and_bind_subtype_variables( + def constrain_and_bind_variables( self, supertype: Type, rigid_variables: AbstractSet['_Variable'], @@ -945,10 +767,10 @@ def constrain_and_bind_subtype_variables( quotation_iterable_type = iterable_type[ StackEffect(TypeSequence([in_var]), TypeSequence([out_var])), ] - return quotation_iterable_type.constrain_and_bind_subtype_variables( + return quotation_iterable_type.constrain_and_bind_variables( supertype, rigid_variables, subtyping_assumptions ) - return super().constrain_and_bind_subtype_variables( + return super().constrain_and_bind_variables( supertype, rigid_variables, subtyping_assumptions ) @@ -970,12 +792,17 @@ def free_type_variables_of_mapping( return ftv -def init_primitives(): - pass +TypeArguments = Sequence[Type] +_T = TypeVar('_T') -TypeArguments = Sequence[Union[StackItemType, TypeSequence]] -_T = TypeVar('_T') +def _contains_assumption( + assumptions: Sequence[Tuple[Type, Type]], subtype: Type, supertype: Type +) -> bool: + for sub, sup in assumptions: + if sub is subtype and sup is supertype: + return True + return False class ObjectType(IndividualType): @@ -987,14 +814,10 @@ class ObjectType(IndividualType): def __init__( self, self_type: IndividualVariable, - # Attributes can be universally quantified since ObjectType allows it. - attributes: Mapping[str, IndividualType], - type_parameters: Sequence[_Variable] = (), + attributes: Mapping[str, Type], nominal_supertypes: Sequence[IndividualType] = (), nominal: bool = False, - _type_arguments: TypeArguments = (), _head: Optional['ObjectType'] = None, - **_other_kwargs, ) -> None: assert isinstance(self_type, IndividualVariable) super().__init__() @@ -1005,25 +828,17 @@ def __init__( self._attributes = attributes - self._type_parameters = type_parameters self._nominal_supertypes = nominal_supertypes self._nominal = nominal - self._type_arguments: TypeArguments = _type_arguments - self._head = _head or self self._internal_name: Optional[str] = None self._internal_name = self._head._internal_name - self._other_kwargs = _other_kwargs.copy() - if '_type_arguments' in self._other_kwargs: - del self._other_kwargs['_type_arguments'] - if 'nominal' in self._other_kwargs: - del self._other_kwargs['nominal'] - self.is_variadic = bool(self._other_kwargs.get('is_variadic')) - - self._instantiations: Dict[TypeArguments, ObjectType] = {} + @property + def nominal(self) -> bool: + return self._nominal def resolve_forward_references(self) -> 'ObjectType': self._attributes = { @@ -1033,52 +848,23 @@ def resolve_forward_references(self) -> 'ObjectType': self._nominal_supertypes = [ t.resolve_forward_references() for t in self._nominal_supertypes ] - self._type_arguments = [ - t.resolve_forward_references() for t in self._type_arguments - ] return self @property def kind(self) -> 'Kind': - if len(self._type_parameters) == 0: - return IndividualKind() - return GenericTypeKind([var.kind for var in self._type_parameters]) + return IndividualKind() def apply_substitution( - self, - sub: 'concat.typecheck.Substitutions', - _should_quantify_over_type_parameters=True, + self, sub: 'concat.typecheck.Substitutions', ) -> 'ObjectType': from concat.typecheck import Substitutions - if _should_quantify_over_type_parameters: - sub = Substitutions( - { - a: i - for a, i in sub.items() - # Don't include self_type in substitution, it is bound. - if a not in self._type_parameters - and a is not self._self_type - } - ) - # if no free type vars will be substituted, just return self - if not any( - free_var in sub for free_var in self.free_type_variables() - ): - return self - else: - sub = Substitutions( - {a: i for a, i in sub.items() if a is not self._self_type} - ) - # if no free type vars will be substituted, just return self - if not any( - free_var in sub - for free_var in { - *self.free_type_variables(), - *self._type_parameters, - } - ): - return self + sub = Substitutions( + {a: i for a, i in sub.items() if a is not self._self_type} + ) + # if no free type vars will be substituted, just return self + if not any(free_var in sub for free_var in self.free_type_variables()): + return self attributes = cast( Dict[str, IndividualType], {attr: sub(t) for attr, t in self._attributes.items()}, @@ -1086,77 +872,20 @@ def apply_substitution( nominal_supertypes = [ sub(supertype) for supertype in self._nominal_supertypes ] - type_arguments = [ - cast(Union[StackItemType, TypeSequence], sub(type_argument)) - for type_argument in self._type_arguments - ] subbed_type = type(self)( self._self_type, attributes, - type_parameters=self._type_parameters, nominal_supertypes=nominal_supertypes, nominal=self._nominal, - _type_arguments=type_arguments, # head is only used to keep track of where a type came from, so # there's no need to substitute it _head=self._head, - **self._other_kwargs, ) if self._internal_name is not None: subbed_type.set_internal_name(self._internal_name) return subbed_type - def is_subtype_of(self, supertype: 'Type') -> bool: - from concat.typecheck import Substitutions - - if supertype in self._nominal_supertypes or self is supertype: - return True - if isinstance(supertype, (_Function, PythonFunctionType)): - if '__call__' not in self._attributes: - return False - return self._attributes['__call__'] <= supertype - if not isinstance(supertype, ObjectType): - return super().is_subtype_of(supertype) - if self._arity != supertype._arity: - return False - if self._arity == 0 and supertype is get_object_type(): - return True - if supertype._nominal and self._head is not supertype._head: - return False - # instantiate these types in a way such that alpha equivalence is not - # an issue - if self._arity > 0: - parameter_pairs = zip( - self.type_parameters, supertype.type_parameters - ) - if not all(a.kind == b.kind for a, b in parameter_pairs): - return False - self = self.instantiate() - supertype = supertype.instantiate() - argument_pairs = dict( - zip(self.type_arguments, supertype.type_arguments) - ) - assert all( - a.kind == b.kind for a, b in argument_pairs.items() - ), str(repr(argument_pairs)) - parameter_sub = Substitutions( - {**argument_pairs, self.self_type: supertype.self_type,} - ) - self = parameter_sub(self) - - for attr, attr_type in supertype._attributes.items(): - if attr not in self._attributes: - return False - sub = concat.typecheck.Substitutions( - {self._self_type: supertype._self_type} - ) - if not ( - cast(IndividualType, sub(self._attributes[attr])) <= attr_type - ): - return False - return True - - def constrain_and_bind_supertype_variables( + def constrain_and_bind_variables( self, supertype: Type, rigid_variables: AbstractSet['_Variable'], @@ -1164,14 +893,18 @@ def constrain_and_bind_supertype_variables( ) -> 'Substitutions': from concat.typecheck import Substitutions - if (self, supertype) in subtyping_assumptions: + if _contains_assumption(subtyping_assumptions, self, supertype): return Substitutions() + # obj <: `t, `t is not rigid + # --> `t = obj if ( isinstance(supertype, IndividualVariable) and supertype not in rigid_variables ): - return Substitutions({supertype: self}) + return Substitutions([(supertype, self)]) + # obj <: *s? `t... + # error elif isinstance(supertype, (SequenceVariable, TypeSequence)): raise concat.typecheck.TypeError( '{} is an individual type, but {} is a sequence type'.format( @@ -1179,224 +912,103 @@ def constrain_and_bind_supertype_variables( ) ) - if isinstance(supertype, StackEffect): - subtyping_assumptions.append((self, supertype)) + if self.kind != supertype.kind: + raise concat.typecheck.TypeError( + f'{self} has kind {self.kind}, but {supertype} has kind {supertype.kind}' + ) - # We know instantiated_self is not a type constructor here, so - # there's no need to worry about variable binding + if isinstance(supertype, (StackEffect, PythonFunctionType)): return self.get_type_of_attribute( '__call__' - ).constrain_and_bind_supertype_variables( - supertype, rigid_variables, subtyping_assumptions - ) - if not isinstance(supertype, ObjectType): - raise NotImplementedError(supertype) - if self._arity < supertype._arity: - raise concat.typecheck.TypeError( - '{} is not as polymorphic as {}'.format(self, supertype) + ).constrain_and_bind_variables( + supertype, + rigid_variables, + subtyping_assumptions + [(self, supertype)], ) - # every object type is a subtype of object_type - if supertype == get_object_type(): - return Substitutions() - # Don't forget that there's nominal subtyping too. - if supertype._nominal: - if ( - supertype not in self._nominal_supertypes - and supertype != self - and self._head != supertype._head - ): - raise concat.typecheck.TypeError( - '{} is not a subtype of {}'.format(self, supertype) - ) - - subtyping_assumptions.append((self, supertype)) - - # constraining to an optional type - if ( - supertype._head == optional_type - and supertype._arity == 0 - and self._arity == 0 - ): + if isinstance(supertype, _OptionalType): try: - return self.constrain_and_bind_supertype_variables( - none_type, rigid_variables, subtyping_assumptions + return self.constrain_and_bind_variables( + none_type, + rigid_variables, + subtyping_assumptions + [(self, supertype)], ) except concat.typecheck.TypeError: - return self.constrain_and_bind_supertype_variables( - supertype._type_arguments[0], + return self.constrain_and_bind_variables( + supertype.type_arguments[0], rigid_variables, - subtyping_assumptions, - ) - - # don't constrain the type arguments, constrain those based on - # the attributes - sub = Substitutions() - # We must not bind any type parameters in self or supertype! To support - # higher-rank polymorphism, let's instantiate both types. At this - # point, self should be at least as polymorphic as supertype. - assert self._arity >= supertype._arity - instantiated_self = self.instantiate() - supertype = supertype.instantiate() - for name in supertype._attributes: - # FIXME: Really types of attributes should not be higher-kinded - type = instantiated_self.get_type_of_attribute(name).instantiate() - sub = sub(type).constrain_and_bind_supertype_variables( - sub(supertype.get_type_of_attribute(name).instantiate()), - rigid_variables, - subtyping_assumptions, - )(sub) - return sub - - def constrain_and_bind_subtype_variables( - self, - supertype: Type, - rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple[IndividualType, IndividualType]], - ) -> 'Substitutions': - from concat.typecheck import Substitutions - - if (self, supertype) in subtyping_assumptions: - return Substitutions() - - if isinstance(supertype, IndividualVariable): - raise TypeError( - '{} is unknown here and cannot be a supertype of {}'.format( - supertype, self - ) - ) - elif isinstance(supertype, (SequenceVariable, TypeSequence)): - raise TypeError( - '{} is an individual type, but {} is a sequence type'.format( - self, supertype + subtyping_assumptions + [(self, supertype)], ) - ) - - # To support higher-rank polymorphism, polymorphic types are subtypes - # of their instances. - - if isinstance(supertype, StackEffect): - subtyping_assumptions.append((self, supertype)) - - instantiated_self = self.instantiate() - # We know self is not a type constructor here, so there's no need - # to worry about variable binding - return instantiated_self.get_type_of_attribute( - '__call__' - ).constrain_and_bind_subtype_variables( - supertype, rigid_variables, subtyping_assumptions - ) if not isinstance(supertype, ObjectType): raise NotImplementedError(supertype) - if self._arity < supertype._arity: - raise concat.typecheck.TypeError( - '{} is not as polymorphic as {}'.format(self, supertype) - ) # every object type is a subtype of object_type - if supertype == get_object_type(): + if supertype is get_object_type(): return Substitutions() # Don't forget that there's nominal subtyping too. if supertype._nominal: - if ( - supertype not in self._nominal_supertypes - and supertype != self - and self._head != supertype._head - ): - raise TypeError( + if supertype in self._nominal_supertypes: + return Substitutions() + if supertype != self and self._head != supertype._head: + raise concat.typecheck.TypeError( '{} is not a subtype of {}'.format(self, supertype) ) + # BUG subtyping_assumptions.append((self, supertype)) # constraining to an optional type - if ( - supertype._head == optional_type - and supertype._arity == 0 - and self._arity == 0 - ): + if isinstance(supertype, _OptionalType): try: - return self.constrain_and_bind_subtype_variables( + return self.constrain_and_bind_variables( none_type, rigid_variables, subtyping_assumptions ) - except TypeError: - return self.constrain_and_bind_subtype_variables( - supertype._type_arguments[0], + except concat.typecheck.TypeError: + return self.constrain_and_bind_variables( + supertype.type_arguments[0], rigid_variables, subtyping_assumptions, ) - # don't constrain the type arguments, constrain those based on # the attributes sub = Substitutions() - # We must not bind any type parameters in self or supertype! To support - # higher-rank polymorphism, let's instantiate both types. At this - # point, self should be at least as polymorphic as supertype. - assert self._arity >= supertype._arity - instantiated_self = self.instantiate() - supertype = supertype.instantiate() for name in supertype._attributes: - type = instantiated_self.get_type_of_attribute(name) - # FIXME: Really types of attributes should not be higher-kinded - type = type.instantiate() - sub = type.constrain_and_bind_subtype_variables( - supertype.get_type_of_attribute(name).instantiate(), + type = self.get_type_of_attribute(name) + sub = sub(type).constrain_and_bind_variables( + sub(supertype.get_type_of_attribute(name)), rigid_variables, subtyping_assumptions, )(sub) return sub - def get_type_of_attribute(self, attribute: str) -> IndividualType: + def get_type_of_attribute(self, attribute: str) -> Type: if attribute not in self._attributes: raise concat.typecheck.AttributeError(self, attribute) - self_sub = concat.typecheck.Substitutions({self._self_type: self}) + self_sub = concat.typecheck.Substitutions([(self._self_type, self)]) return self_sub(self._attributes[attribute]) def __repr__(self) -> str: - return '{}({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})'.format( - type(self).__qualname__, - self._self_type, - self._attributes, - self._type_parameters, - self._nominal_supertypes, - self._nominal, - self._type_arguments, - None if self._head is self else self._head, - ) + head = None if self._head is self else self._head + return f'{type(self).__qualname__}(self_type={self._self_type!r}, attributes={self._attributes!r}, nominal_supertypes={self._nominal_supertypes!r}, nominal={self._nominal!r}, _head={head!r})' def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: ftv = free_type_variables_of_mapping(self.attributes) - for arg in self.type_arguments: - ftv |= arg.free_type_variables() # QUESTION: Include supertypes? - ftv -= {self.self_type, *self.type_parameters} + ftv -= {self.self_type} return ftv def __str__(self) -> str: if self._internal_name is not None: - if len(self._type_arguments) > 0: - return ( - self._internal_name - + '[' - + ', '.join(map(str, self._type_arguments)) - + ']' - ) return self._internal_name - assert self._internal_name is None - return '{}({}, {}, {}, {}, {}, {}, {})'.format( + return '{}({}, {}, {}, {}, {})'.format( type(self).__qualname__, self._self_type, _mapping_to_str(self._attributes), - _iterable_to_str(self._type_parameters), _iterable_to_str(self._nominal_supertypes), self._nominal, - _iterable_to_str(self._type_arguments), None if self._head is self else self._head, ) - def set_internal_name(self, name: str) -> None: - self._internal_name = name - _hash_variable = None def __hash__(self) -> int: @@ -1404,144 +1016,109 @@ def __hash__(self) -> int: if ObjectType._hash_variable is None: ObjectType._hash_variable = IndividualVariable() - sub = Substitutions({self._self_type: ObjectType._hash_variable}) - # Avoid sub(self) since the lru cache on that will hash self + sub = Substitutions([(self._self_type, ObjectType._hash_variable)]) type_to_hash = sub(self) return hash( ( tuple(type_to_hash._attributes.items()), - tuple(type_to_hash._type_parameters), tuple(type_to_hash._nominal_supertypes), type_to_hash._nominal, - # FIXME: I get 'not hashable' errors about this. - # tuple(type_to_hash._type_arguments), None if type_to_hash._head == self else type_to_hash._head, ) ) - def __getitem__(self, type_arguments: TypeArguments,) -> 'ObjectType': + @property + def attributes(self) -> Mapping[str, Type]: from concat.typecheck import Substitutions - if not isinstance(self.kind, GenericTypeKind): - raise concat.typecheck.TypeError(f'{self} is not a generic type') - - if self._arity != len(type_arguments): - raise concat.typecheck.TypeError( - 'type constructor {} given {} arguments, expected {} arguments'.format( - self, len(type_arguments), self._arity - ) - ) - - expected_kinds = tuple(self.kind.parameter_kinds) - given_kinds = tuple(ty.kind for ty in type_arguments) - if expected_kinds != given_kinds: - raise concat.typecheck.TypeError( - f'wrong kinds of arguments given to type: expected {expected_kinds}, given {given_kinds}' - ) - - type_arguments = tuple(type_arguments) - if type_arguments in self._instantiations: - return self._instantiations[type_arguments] - - sub = Substitutions(zip(self._type_parameters, type_arguments)) - result = self.apply_substitution( - sub, _should_quantify_over_type_parameters=False - ) - # HACK: We remove the parameters and add arguments through mutation. - result._type_parameters = () - result._type_arguments = type_arguments - - self._instantiations[type_arguments] = result - - return result - - def instantiate(self: _T) -> _T: - # Avoid overwriting the type arguments if type is already instantiated. - if self._arity == 0: - return self - fresh_variables = [type(a)() for a in self._type_parameters] - return self[fresh_variables] - - @property - def attributes(self) -> Dict[str, IndividualType]: - return self._attributes + sub = Substitutions([(self._self_type, self)]) + return {name: sub(ty) for name, ty in self._attributes.items()} @property def self_type(self) -> IndividualVariable: return self._self_type - @property - def type_arguments(self) -> TypeArguments: - return self._type_arguments - @property def head(self) -> 'ObjectType': return self._head - @property - def type_parameters(self) -> Sequence[_Variable]: - return self._type_parameters - @property def nominal_supertypes(self) -> Sequence[IndividualType]: return self._nominal_supertypes - @property - def _arity(self) -> int: - return len(self._type_parameters) - +# QUESTION: Should this exist, or should I use ObjectType class ClassType(ObjectType): """The representation of types of classes, like in "Design and Evaluation of Gradual Typing for Python" (Vitousek et al. 2014).""" - def is_subtype_of(self, supertype: Type) -> bool: + def constrain_and_bind_variables( + self, supertype, rigid_variables, subtyping_assumptions + ) -> 'Substitutions': if ( not supertype.has_attribute('__call__') or '__init__' not in self._attributes ): - return super().is_subtype_of(supertype) - bound_init = self._attributes['__init__'].bind() - return bound_init <= supertype + return super().constrain_and_bind_variables( + supertype, rigid_variables, subtyping_assumptions + ) + init = self.get_type_of_attribute('__init__') + while not isinstance(init, (StackEffect, PythonFunctionType)): + init = init.get_type_of_attribute('__call__') + bound_init = init.bind() + return bound_init.constrain_and_bind_variables( + supertype.get_type_of_attribute('__call__'), + rigid_variables, + subtyping_assumptions + [(self, supertype)], + ) -class PythonFunctionType(ObjectType): +class PythonFunctionType(IndividualType): def __init__( self, - self_type: IndividualVariable, - *args, _overloads: Sequence[ Tuple[Sequence[StackItemType], IndividualType] ] = (), - type_parameters=(), - **kwargs, + type_parameters: Sequence[_Variable] = (), + _type_arguments: Sequence[Type] = (), ) -> None: - self._kwargs = kwargs.copy() - # HACK: I shouldn't have to manipulate arguments like this - if 'type_parameters' in self._kwargs: - del self._kwargs['type_parameters'] - super().__init__( - self_type, - *args, - **self._kwargs, - _overloads=_overloads, - type_parameters=type_parameters, - ) - assert ( + super().__init__() + self._arity = len(type_parameters) + self._type_parameters = type_parameters + self._type_arguments = _type_arguments + if not ( self._arity == 0 and len(self._type_arguments) == 2 or self._arity == 2 and len(self._type_arguments) == 0 - ) + ): + raise concat.typecheck.TypeError( + f'Ill-formed Python function type with arguments {self._type_arguments}' + ) if self._arity == 0: assert isinstance(self.input, collections.abc.Sequence) assert self._type_arguments[1].kind == IndividualKind() - self._args = list(args) self._overloads = _overloads - if '_head' in self._kwargs: - del self._kwargs['_head'] - self._head: PythonFunctionType + self._hash: Optional[int] = None + + def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: + if self._arity == 0: + ftv = InsertionOrderedSet[_Variable]([]) + for ty in self.input: + ftv |= ty.free_type_variables() + ftv |= self.output.free_type_variables() + return ftv + else: + return InsertionOrderedSet([]) + + @property + def kind(self) -> 'Kind': + if self._arity == 0: + return IndividualKind() + return GenericTypeKind([SequenceKind(), IndividualKind()]) def resolve_forward_references(self) -> 'PythonFunctionType': + if self._forward_references_resolved: + return self super().resolve_forward_references() overloads = [] for args, ret in overloads: @@ -1552,8 +1129,37 @@ def resolve_forward_references(self) -> 'PythonFunctionType': ) ) self._overloads = overloads + self._type_arguments = list( + t.resolve_forward_references() for t in self._type_arguments + ) return self + def __eq__(self, other: object) -> bool: + if not isinstance(other, PythonFunctionType): + return False + if self.kind != other.kind: + return False + if isinstance(self.kind, GenericTypeKind): + return True + return ( + tuple(self.input) == tuple(other.input) + and self.output == other.output + ) + + def __hash__(self) -> int: + if self._hash is None: + self._hash = self._compute_hash() + return self._hash + + def _compute_hash(self) -> int: + if isinstance(self.kind, GenericTypeKind): + return 1 + return hash((tuple(self.input), self.output)) + + def __repr__(self) -> str: + # QUESTION: Is it worth using type(self)? + return f'{type(self).__qualname__}(_overloads={self._overloads!r}, type_parameters={self._type_parameters!r}, _type_arguments={self._type_arguments})' + def __str__(self) -> str: if not self._type_arguments: return 'py_function_type' @@ -1562,14 +1168,15 @@ def __str__(self) -> str: ) def get_type_of_attribute(self, attribute: str) -> IndividualType: - from concat.typecheck import Substitutions - - sub = Substitutions({self._self_type: self}) if attribute == '__call__': return self else: return super().get_type_of_attribute(attribute) + @property + def attributes(self) -> Mapping[str, Type]: + return {**super().attributes, '__call__': self} + def __getitem__( self, arguments: Tuple[TypeSequence, IndividualType] ) -> 'PythonFunctionType': @@ -1590,30 +1197,21 @@ def __getitem__( f'Second argument to {self} must be an individual type for the return type' ) return PythonFunctionType( - self._self_type, - *self._args, - **{ - **self._kwargs, - '_type_arguments': (input, output), - 'type_parameters': (), - }, - _overloads=[], - _head=self, + _type_arguments=(input, output), type_parameters=(), _overloads=[], ) def apply_substitution( self, sub: 'concat.typecheck.Substitutions' ) -> 'PythonFunctionType': if self._arity == 0: - type = py_function_type[ - sub(TypeSequence(self.input)), sub(self.output) + inp = sub(TypeSequence(self.input)) + out = sub(self.output) + overloads: Sequence[Tuple[TypeSequence, IndividualType]] = [ + (sub(TypeSequence(i)), sub(o)) for i, o in self._overloads ] - for overload in self._overloads: - # This is one of the few places where a type should be mutated. - type._add_overload( - [sub(i) for i in overload[0]], sub(overload[1]) - ) - return type + return PythonFunctionType( + _type_arguments=(inp, out), _overloads=overloads + ) return self @property @@ -1635,9 +1233,7 @@ def select_overload( ) -> Tuple['PythonFunctionType', 'Substitutions']: for overload in [(self.input, self.output), *self._overloads]: try: - sub = TypeSequence( - input_types - ).constrain_and_bind_supertype_variables( + sub = TypeSequence(input_types).constrain_and_bind_variables( TypeSequence(overload[0]), set(), [] ) except TypeError: @@ -1646,7 +1242,7 @@ def select_overload( sub(py_function_type[TypeSequence(overload[0]), overload[1]]), sub, ) - raise TypeError( + raise concat.typecheck.TypeError( 'no overload of {} matches types {}'.format(self, input_types) ) @@ -1654,41 +1250,21 @@ def with_overload( self, input: Sequence[StackItemType], output: IndividualType ) -> 'PythonFunctionType': return PythonFunctionType( - self._self_type, - *self._args, - **self._kwargs, + _type_arguments=self._type_arguments, _overloads=[*self._overloads, (input, output)], - _head=py_function_type, ) - def _add_overload( - self, input: Sequence[StackItemType], output: IndividualType - ) -> None: - self._overloads.append((input, output)) - def bind(self) -> 'PythonFunctionType': assert self._arity == 0 inputs = self.input[1:] output = self.output - return self._head[TypeSequence(inputs), output] - - def is_subtype_of(self, supertype: Type) -> bool: - if super().is_subtype_of(supertype): - return True - if isinstance(supertype, PythonFunctionType): - # NOTE: make sure types are of same kind (arity) - if len(self._type_parameters) != len(supertype._type_parameters): - return False - if len(self._type_parameters) == 2: - # both are py_function_type - return True - return ( - supertype._type_arguments[0] <= self._type_arguments[0] - and self._type_arguments[1] <= supertype._type_arguments[1] - ) - return False + overloads = [(i[1:], o) for i, o in self._overloads] + return PythonFunctionType( + _type_arguments=[TypeSequence(inputs), output], + _overloads=overloads, + ) - def constrain_and_bind_supertype_variables( + def constrain_and_bind_variables( self, supertype: Type, rigid_variables: AbstractSet['_Variable'], @@ -1696,113 +1272,72 @@ def constrain_and_bind_supertype_variables( ) -> 'Substitutions': from concat.typecheck import Substitutions - sub = super().constrain_and_bind_supertype_variables( - supertype, rigid_variables, subtyping_assumptions - ) - - if ( - isinstance(supertype, PythonFunctionType) - and supertype._arity <= self._arity + if self is supertype or _contains_assumption( + subtyping_assumptions, self, supertype ): - instantiated_self = self.instantiate() - supertype = supertype.instantiate() - - # ObjectType constrains the attributes, not the type arguments - # directly, so we'll doo that here. This isn't problematic because - # we know the variance of the arguments here. + return Substitutions() + if self.kind != supertype.kind: + raise concat.typecheck.TypeError( + f'{self} has kind {self.kind} but {supertype} has kind {supertype.kind}' + ) + if self.kind == IndividualKind(): + if ( + isinstance(supertype, IndividualVariable) + and supertype not in rigid_variables + ): + return Substitutions([(supertype, self)]) + if isinstance(supertype, _OptionalType): + return self.constrain_and_bind_variables( + supertype.type_arguments[0], + rigid_variables, + subtyping_assumptions, + ) + if isinstance(supertype, ObjectType) and not supertype.nominal: + sub = Substitutions() + for attr in supertype.attributes: + self_attr_type = sub(self.get_type_of_attribute(attr)) + supertype_attr_type = sub( + supertype.get_type_of_attribute(attr) + ) + sub = self_attr_type.constrain_and_bind_variables( + supertype_attr_type, + rigid_variables, + subtyping_assumptions, + ) + return sub + if isinstance(supertype, PythonFunctionType): + if isinstance(self.kind, GenericTypeKind): + return Substitutions() # No need to extend the rigid variables, we know both types have no # parameters at this point. # Support overloading the subtype. for overload in [ - (instantiated_self.input, instantiated_self.output), - *instantiated_self._overloads, + (self.input, self.output), + *self._overloads, ]: try: subtyping_assumptions_copy = subtyping_assumptions[:] self_input_types = TypeSequence(overload[0]) supertype_input_types = TypeSequence(supertype.input) - sub = supertype_input_types.constrain_and_bind_subtype_variables( + sub = supertype_input_types.constrain_and_bind_variables( self_input_types, rigid_variables, subtyping_assumptions_copy, - )( - sub ) - sub = sub( - instantiated_self.output - ).constrain_and_bind_supertype_variables( + sub = sub(self.output).constrain_and_bind_variables( sub(supertype.output), rigid_variables, subtyping_assumptions_copy, - )( - sub - ) - except TypeError: - continue - finally: - subtyping_assumptions[:] = subtyping_assumptions_copy + )(sub) return sub - - raise TypeError( - 'no overload of {} is a subtype of {}'.format(self, supertype) - ) - - def constrain_and_bind_subtype_variables( - self, - supertype: Type, - rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple[IndividualType, IndividualType]], - ) -> 'Substitutions': - from concat.typecheck import Substitutions - - sub = super().constrain_and_bind_subtype_variables( - supertype, rigid_variables, subtyping_assumptions - ) - - if ( - isinstance(supertype, PythonFunctionType) - and supertype._arity <= self._arity - ): - instantiated_self = self.instantiate() - supertype = supertype.instantiate() - - # ObjectType constrains the attributes, not the type arguments - # directly, so we'll doo that here. This isn't problematic because - # we know the variance of the arguments here. - - for overload in [ - (instantiated_self.input, instantiated_self.output), - *instantiated_self._overloads, - ]: - try: - subtyping_assumptions_copy = subtyping_assumptions[:] - self_input_types = TypeSequence(overload[0]) - supertype_input_types = TypeSequence(supertype.input) - sub = supertype_input_types.constrain_and_bind_supertype_variables( - self_input_types, - rigid_variables, - subtyping_assumptions_copy, - )( - sub - ) - sub = sub( - instantiated_self.output - ).constrain_and_bind_subtype_variables( - sub(supertype.output), - rigid_variables, - subtyping_assumptions_copy, - )( - sub - ) - except TypeError: + except concat.typecheck.TypeError: continue finally: subtyping_assumptions[:] = subtyping_assumptions_copy - return sub - raise TypeError( + raise concat.typecheck.TypeError( 'no overload of {} is a subtype of {}'.format(self, supertype) ) @@ -1830,7 +1365,9 @@ def __getitem__(self, args: Sequence[Type]) -> 'PythonFunctionType': @property def attributes(self) -> Mapping[str, 'Type']: - raise TypeError('py_overloaded does not have attributes') + raise concat.typecheck.TypeError( + 'py_overloaded does not have attributes' + ) def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: return InsertionOrderedSet([]) @@ -1845,15 +1382,7 @@ def instantiate(self) -> PythonFunctionType: py_function_type.instantiate(), ] - def constrain_and_bind_supertype_variables( - self, - supertype: 'Type', - rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple['IndividualType', 'IndividualType']], - ) -> 'Substitutions': - raise concat.typecheck.TypeError('py_overloaded is a generic type') - - def constrain_and_bind_subtype_variables( + def constrain_and_bind_variables( self, supertype: 'Type', rigid_variables: AbstractSet['_Variable'], @@ -1895,26 +1424,76 @@ def __repr__(self) -> str: return '{}()'.format(type(self).__qualname__) -class _OptionalType(ObjectType): - def __init__(self, _type_arguments=[]) -> None: - x = IndividualVariable() - type_var = IndividualVariable() - if len(_type_arguments) > 0: - super().__init__(x, {}, [], _type_arguments=_type_arguments) - else: - super().__init__(x, {}, [type_var]) +class _OptionalType(IndividualType): + def __init__(self, type_argument: IndividualType) -> None: + super().__init__() + while isinstance(type_argument, _OptionalType): + type_argument = type_argument._type_argument + self._type_argument: IndividualType = type_argument - def __getitem__( - self, type_arguments: Sequence[StackItemType] - ) -> '_OptionalType': - assert len(type_arguments) == 1 - return _OptionalType(type_arguments) + def __repr__(self) -> str: + return f'{type(self).__qualname__}({self._type_argument!r})' + + def __str__(self) -> str: + return f'optional_type[{self._type_argument}]' + + def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: + return self._type_argument.free_type_variables() + + def __eq__(self, other: object) -> bool: + if not isinstance(other, _OptionalType): + return False + return self._type_argument == other._type_argument + + def __hash__(self) -> int: + return hash(self._type_argument) + + def constrain_and_bind_variables( + self, supertype, rigid_variables, subtyping_assumptions + ) -> 'Substitutions': + from concat.typecheck import Substitutions + + if ( + self is supertype + or _contains_assumption(subtyping_assumptions, self, supertype) + or supertype is get_object_type() + ): + return Substitutions() + # A special case for better resuls (I think) + if isinstance(supertype, _OptionalType): + return self._type_argument.constrain_and_bind_variables( + supertype._type_argument, + rigid_variables, + subtyping_assumptions, + ) + if self.kind != supertype.kind: + raise concat.typecheck.TypeError( + f'{self} is an individual type, but {supertype} has kind {supertype.kind}' + ) + # FIXME: optional[none] should simplify to none + if self._type_argument is none_type and supertype is none_type: + return Substitutions() + + sub = none_type.constrain_and_bind_variables( + supertype, rigid_variables, subtyping_assumptions + ) + sub = sub(self._type_argument).constrain_and_bind_variables( + sub(supertype), rigid_variables, subtyping_assumptions + ) + return sub + + def resolve_forward_references(self) -> '_OptionalType': + self._type_argument = self._type_argument.resolve_forward_references() + return self def apply_substitution( self, sub: 'concat.typecheck.Substitutions' ) -> '_OptionalType': - # FIXME: self._type_arguments might not be a valid stack type. - return _OptionalType(tuple(sub(TypeSequence(self._type_arguments)))) + return _OptionalType(sub(self._type_argument)) + + @property + def type_arguments(self) -> Sequence[IndividualType]: + return [self._type_argument] class Kind(abc.ABC): @@ -2023,7 +1602,7 @@ def __getitem__(self, args: TypeArguments) -> IndividualType: self._resolution_env, _type_arguments=args, ) - raise TypeError(f'{self} is not a generic type') + raise concat.typecheck.TypeError(f'{self} is not a generic type') def resolve_forward_references(self) -> Type: if self._resolved_type is None: @@ -2047,25 +1626,7 @@ def attributes(self) -> Mapping[str, Type]: 'Cannot access attributes of type before they are defined' ) - def constrain_and_bind_subtype_variables( - self, - supertype: Type, - rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple[IndividualType, IndividualType]], - ) -> 'Substitutions': - if self is supertype: - return concat.typecheck.Substitutions() - - if self._resolved_type is not None: - return self._resolved_type.constrain_and_bind_subtype_variables( - supertype, rigid_variables, subtyping_assumptions - ) - - raise concat.typecheck.TypeError( - 'Supertypes of type are not known before its definition' - ) - - def constrain_and_bind_supertype_variables( + def constrain_and_bind_variables( self, supertype: Type, rigid_variables: AbstractSet['_Variable'], @@ -2075,7 +1636,7 @@ def constrain_and_bind_supertype_variables( return concat.typecheck.Substitutions() if self._resolved_type is not None: - return self._resolved_type.constrain_and_bind_supertype_variables( + return self._resolved_type.constrain_and_bind_variables( supertype, rigid_variables, subtyping_assumptions ) @@ -2156,42 +1717,50 @@ def set_str_type(ty: Type) -> None: _arg_type_var = SequenceVariable() _return_type_var = IndividualVariable() py_function_type = PythonFunctionType( - _x, {}, type_parameters=[_arg_type_var, _return_type_var] + type_parameters=[_arg_type_var, _return_type_var] ) py_function_type.set_internal_name('py_function_type') _invert_result_var = IndividualVariable() -invertible_type = ObjectType( - _x, - {'__invert__': py_function_type[TypeSequence([]), _invert_result_var]}, +invertible_type = GenericType( [_invert_result_var], + ObjectType( + _x, + {'__invert__': py_function_type[TypeSequence([]), _invert_result_var]}, + ), ) _sub_operand_type = IndividualVariable() _sub_result_type = IndividualVariable() # FIXME: Add reverse_substractable_type for __rsub__ -subtractable_type = ObjectType( - _x, - { - '__sub__': py_function_type[ - TypeSequence([_sub_operand_type]), _sub_result_type - ] - }, +subtractable_type = GenericType( [_sub_operand_type, _sub_result_type], + ObjectType( + _x, + { + '__sub__': py_function_type[ + TypeSequence([_sub_operand_type]), _sub_result_type + ] + }, + ), ) +subtractable_type.set_internal_name('subtractable_type') +_add_other_operand_type = IndividualVariable() _add_result_type = IndividualVariable() -addable_type = ObjectType( - _x, - { - '__add__': py_function_type[ - # FIXME: object should be the parameter type - TypeSequence([_x]), - _add_result_type, - ] - }, - [_add_result_type], +addable_type = GenericType( + [_add_other_operand_type, _add_result_type], + ObjectType( + _x, + { + '__add__': py_function_type[ + # QUESTION: Should methods include self? + TypeSequence([_add_other_operand_type]), + _add_result_type, + ] + }, + ), ) addable_type.set_internal_name('addable_type') @@ -2201,28 +1770,33 @@ def set_str_type(ty: Type) -> None: # QUESTION: Allow comparison methods to return any object? _other_type = IndividualVariable() -geq_comparable_type = ObjectType( - _x, - {'__ge__': py_function_type[TypeSequence([_other_type]), bool_type]}, +geq_comparable_type = GenericType( [_other_type], + ObjectType( + _x, + {'__ge__': py_function_type[TypeSequence([_other_type]), bool_type]}, + ), ) geq_comparable_type.set_internal_name('geq_comparable_type') -leq_comparable_type = ObjectType( - _x, - {'__le__': py_function_type[TypeSequence([_other_type]), bool_type]}, +leq_comparable_type = GenericType( [_other_type], + ObjectType( + _x, + {'__le__': py_function_type[TypeSequence([_other_type]), bool_type]}, + ), ) leq_comparable_type.set_internal_name('leq_comparable_type') -lt_comparable_type = ObjectType( - _x, - {'__lt__': py_function_type[TypeSequence([_other_type]), bool_type]}, +lt_comparable_type = GenericType( [_other_type], + ObjectType( + _x, + {'__lt__': py_function_type[TypeSequence([_other_type]), bool_type]}, + ), ) lt_comparable_type.set_internal_name('lt_comparable_type') -# FIXME: The parameter type should be object. _int_add_type = py_function_type[TypeSequence([_x]), _x] int_type = ObjectType( @@ -2244,24 +1818,30 @@ def set_str_type(ty: Type) -> None: _result_type = IndividualVariable() -iterator_type = ObjectType( - _x, - { - '__iter__': py_function_type[TypeSequence([]), _x], - '__next__': py_function_type[TypeSequence([none_type,]), _result_type], - }, +iterator_type = GenericType( [_result_type], + ObjectType( + _x, + { + '__iter__': py_function_type[TypeSequence([]), _x], + '__next__': py_function_type[ + TypeSequence([none_type,]), _result_type + ], + }, + ), ) iterator_type.set_internal_name('iterator_type') -iterable_type = ObjectType( - _x, - { - '__iter__': py_function_type[ - TypeSequence([]), iterator_type[_result_type,] - ] - }, +iterable_type = GenericType( [_result_type], + ObjectType( + _x, + { + '__iter__': py_function_type[ + TypeSequence([]), iterator_type[_result_type,] + ] + }, + ), ) iterable_type.set_internal_name('iterable_type') @@ -2276,7 +1856,10 @@ def set_str_type(ty: Type) -> None: ) context_manager_type.set_internal_name('context_manager_type') -optional_type = _OptionalType() +_optional_type_var = IndividualVariable() +optional_type = GenericType( + [_optional_type_var], _OptionalType(_optional_type_var) +) optional_type.set_internal_name('optional_type') _key_type_var = IndividualVariable() @@ -2294,16 +1877,14 @@ def set_str_type(ty: Type) -> None: dict_type.set_internal_name('dict_type') file_type = ObjectType( - _x, - { + self_type=_x, + attributes={ 'seek': py_function_type[TypeSequence([int_type]), int_type], 'read': py_function_type, '__enter__': py_function_type, '__exit__': py_function_type, }, - [], # context_manager_type is a structural supertype - [iterable_type], nominal=True, ) file_type.set_internal_name('file_type') @@ -2322,13 +1903,15 @@ def set_str_type(ty: Type) -> None: not_implemented_type = ObjectType(_x, {}) _element_types_var = SequenceVariable() -tuple_type = ObjectType( - _x, - {'__getitem__': py_function_type}, +tuple_type = GenericType( [_element_types_var], - nominal=True, + ObjectType( + _x, + {'__getitem__': py_function_type}, + nominal=True, + # iterable_type is a structural supertype + ), is_variadic=True, - # iterable_type is a structural supertype ) tuple_type.set_internal_name('tuple_type') From a732a09a5da337771fbbacfe73c673a1566960b9 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Fri, 31 May 2024 17:45:00 -0700 Subject: [PATCH 17/61] Break infinite recursions in subtyping and free variables --- concat/typecheck/types.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 11ecf6b..37618bc 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -73,6 +73,10 @@ def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: def free_type_variables(self) -> InsertionOrderedSet['_Variable']: if self._free_type_variables_cached is None: + # Break circular references. Recusring into the same type won't add + # new FTVs, so we can pretend there are none we finish finding the + # others. + self._free_type_variables_cached = InsertionOrderedSet([]) self._free_type_variables_cached = self._free_type_variables() return self._free_type_variables_cached @@ -893,7 +897,9 @@ def constrain_and_bind_variables( ) -> 'Substitutions': from concat.typecheck import Substitutions - if _contains_assumption(subtyping_assumptions, self, supertype): + if self is supertype or _contains_assumption( + subtyping_assumptions, self, supertype + ): return Substitutions() # obj <: `t, `t is not rigid @@ -947,7 +953,7 @@ def constrain_and_bind_variables( if supertype._nominal: if supertype in self._nominal_supertypes: return Substitutions() - if supertype != self and self._head != supertype._head: + if self._head is not supertype._head: raise concat.typecheck.TypeError( '{} is not a subtype of {}'.format(self, supertype) ) @@ -955,18 +961,6 @@ def constrain_and_bind_variables( # BUG subtyping_assumptions.append((self, supertype)) - # constraining to an optional type - if isinstance(supertype, _OptionalType): - try: - return self.constrain_and_bind_variables( - none_type, rigid_variables, subtyping_assumptions - ) - except concat.typecheck.TypeError: - return self.constrain_and_bind_variables( - supertype.type_arguments[0], - rigid_variables, - subtyping_assumptions, - ) # don't constrain the type arguments, constrain those based on # the attributes sub = Substitutions() @@ -1120,7 +1114,7 @@ def resolve_forward_references(self) -> 'PythonFunctionType': if self._forward_references_resolved: return self super().resolve_forward_references() - overloads = [] + overloads: List[Tuple[Sequence[StackItemType], IndividualType]] = [] for args, ret in overloads: overloads.append( ( @@ -1167,7 +1161,7 @@ def __str__(self) -> str: _iterable_to_str(self.input), self.output ) - def get_type_of_attribute(self, attribute: str) -> IndividualType: + def get_type_of_attribute(self, attribute: str) -> Type: if attribute == '__call__': return self else: From 8d392064e9b425b6d52c03e9a6da392bdf4dac22 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Fri, 31 May 2024 20:37:53 -0700 Subject: [PATCH 18/61] Trade <= with is_subtype_of --- concat/tests/test_typecheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concat/tests/test_typecheck.py b/concat/tests/test_typecheck.py index 971e496..646bbb6 100644 --- a/concat/tests/test_typecheck.py +++ b/concat/tests/test_typecheck.py @@ -307,7 +307,7 @@ def test_reflexive_equality(self, type): class TestSubtyping(unittest.TestCase): def test_int_not_subtype_of_float(self) -> None: """Differ from Reticulated Python: !(int <= float).""" - self.assertFalse(int_type <= float_type) + self.assertFalse(int_type.is_subtype_of(float_type)) @given(from_type(IndividualType), from_type(IndividualType)) @settings(suppress_health_check=(HealthCheck.filter_too_much,)) From 67ed6726602ffe25c03e366bdb595d5032c2dd5b Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Fri, 31 May 2024 21:02:33 -0700 Subject: [PATCH 19/61] Make notes about type-level lang design --- concat/type-level-lang-precedence.md | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 concat/type-level-lang-precedence.md diff --git a/concat/type-level-lang-precedence.md b/concat/type-level-lang-precedence.md new file mode 100644 index 0000000..67d46f5 --- /dev/null +++ b/concat/type-level-lang-precedence.md @@ -0,0 +1,37 @@ +The type constructor of a type application is restricted to being a name. + +```python +{ + seed: py_function[(int), none], + shuffle: forall `t. py_function[(list[`t]), none] +} +``` + +``` +Object + seed: + type application + name - py_function + args + sequence + item 1: + name - _ + name - int + name - none + shuffle: + forall + args + individual var - t + type + type application + name - py_function + args + sequence + item 1: + name - _ + type application + name - list + args + individual var - t + name - none +``` From 4b5a06c4f54f2f63c3efcc1464b7aa3e105adaf6 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Fri, 31 May 2024 23:16:46 -0700 Subject: [PATCH 20/61] Fix subtyping for object (top) and no_return (bottom) --- concat/tests/strategies.py | 5 + concat/typecheck/types.py | 390 ++++++++++++++++++++----------------- 2 files changed, 221 insertions(+), 174 deletions(-) diff --git a/concat/tests/strategies.py b/concat/tests/strategies.py index 932c06d..8a57402 100644 --- a/concat/tests/strategies.py +++ b/concat/tests/strategies.py @@ -7,6 +7,7 @@ StackEffect, TypeSequence, none_type, + no_return_type, optional_type, py_function_type, ) @@ -15,6 +16,7 @@ builds, dictionaries, from_type, + just, lists, none, recursive, @@ -70,6 +72,9 @@ def _mark_individual_type_strategy( _individual_type_strategy = recursive( _mark_individual_type_strategy( from_type(IndividualVariable), IndividualVariable + ) + | _mark_individual_type_strategy( + just(no_return_type), type(no_return_type) ), lambda children: _mark_individual_type_strategy( builds( diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 37618bc..42cd963 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -17,7 +17,7 @@ cast, overload, ) -from typing_extensions import Literal, Self +from typing_extensions import Self import abc import collections.abc @@ -53,6 +53,10 @@ def __eq__(self, other: object) -> bool: # QUESTION: Define == separately from is_subtype_of? return self.is_subtype_of(other) and other.is_subtype_of(self) + @abc.abstractmethod + def __hash__(self) -> int: + pass + def get_type_of_attribute(self, name: str) -> 'Type': raise AttributeError(self, name) @@ -126,138 +130,10 @@ def __str__(self) -> str: return super().__str__() -class GenericType(Type): - def __init__( - self, - type_parameters: Sequence['_Variable'], - body: Type, - is_variadic: bool = False, - ) -> None: - super().__init__() - assert type_parameters - self._type_parameters = type_parameters - if body.kind != IndividualKind(): - raise concat.typecheck.TypeError( - f'Cannot be polymorphic over non-individual type {body}' - ) - self._body = body - self._instantiations: Dict[Tuple[Type, ...], Type] = {} - self.is_variadic = is_variadic - - def __str__(self) -> str: - if self._internal_name is not None: - return self._internal_name - if self.is_variadic: - params = str(self._type_parameters[0]) + '...' - else: - params = ' '.join(map(str, self._type_parameters)) - - return f'forall {params}. {self._body}' - - def __repr__(self) -> str: - return f'{type(self).__qualname__}({self._type_parameters!r}, {self._body!r}, is_variadic={self.is_variadic!r})' - - def __getitem__(self, type_arguments: 'TypeArguments') -> 'Type': - from concat.typecheck import Substitutions - - type_arguments = tuple(type_arguments) - if type_arguments in self._instantiations: - return self._instantiations[type_arguments] - expected_kinds = [var.kind for var in self._type_parameters] - actual_kinds = [ty.kind for ty in type_arguments] - if expected_kinds != actual_kinds: - raise concat.typecheck.TypeError( - f'A type argument to {self} has the wrong kind, type arguments: {type_arguments}, expected kinds: {expected_kinds}' - ) - sub = Substitutions(zip(self._type_parameters, type_arguments)) - instance = sub(self._body) - self._instantiations[type_arguments] = instance - if self._internal_name is not None: - instance_internal_name = self._internal_name - instance_internal_name += ( - '[' + ', '.join(map(str, type_arguments)) + ']' - ) - instance.set_internal_name(instance_internal_name) - return instance - - @property - def kind(self) -> 'Kind': - kinds = [var.kind for var in self._type_parameters] - return GenericTypeKind(kinds) - - def resolve_forward_references(self) -> 'GenericType': - self._body = self._body.resolve_forward_references() - return self - - def instantiate(self) -> Type: - fresh_vars: Sequence[_Variable] = [ - type(var)() for var in self._type_parameters - ] - return self[fresh_vars] - - def constrain_and_bind_variables( - self, - supertype: 'Type', - rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple['IndividualType', 'IndividualType']], - ) -> 'Substitutions': - from concat.typecheck import Substitutions - - if self is supertype or _contains_assumption( - subtyping_assumptions, self, supertype - ): - return Substitutions([]) - if self.kind != supertype.kind: - raise concat.typecheck.TypeError( - f'{self} has kind {self.kind} but {supertype} has kind {supertype.kind}' - ) - if not isinstance(supertype, GenericType): - raise NotImplementedError(supertype) - shared_vars = [type(var)() for var in self._type_parameters] - self_instance = self[shared_vars] - supertype_instance = supertype[shared_vars] - rigid_variables = ( - rigid_variables - | set(self._type_parameters) - | set(supertype._type_parameters) - ) - return self_instance.constrain_and_bind_variables( - supertype_instance, rigid_variables, subtyping_assumptions - ) - - def apply_substitution(self, sub: 'Substitutions') -> 'GenericType': - from concat.typecheck import Substitutions - - sub = Substitutions( - { - var: ty - for var, ty in sub.items() - if var not in self._type_parameters - } - ) - ty = GenericType(self._type_parameters, sub(self._body)) - return ty - - @property - def attributes(self) -> NoReturn: - raise concat.typecheck.TypeError( - 'Generic types do not have attributes; maybe you forgot type arguments?' - ) - - def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: - return self._body.free_type_variables() - set(self._type_parameters) - - -class IndividualType(Type, abc.ABC): +class IndividualType(Type): def instantiate(self) -> 'IndividualType': return cast(IndividualType, super().instantiate()) - @abc.abstractmethod - def apply_substitution( - self, sub: 'concat.typecheck.Substitutions', - ) -> 'IndividualType': - pass - @property def kind(self) -> 'Kind': return IndividualKind() @@ -276,7 +152,7 @@ class _Variable(Type, abc.ABC): def apply_substitution( self, sub: 'concat.typecheck.Substitutions' - ) -> Union[IndividualType, '_Variable', 'TypeSequence']: + ) -> Union['IndividualType', '_Variable', 'TypeSequence']: if self in sub: result = sub[self] assert self.kind == result.kind, f'{self!r} --> {result!r}' @@ -310,7 +186,7 @@ def constrain_and_bind_variables( ) -> 'Substitutions': from concat.typecheck import Substitutions - if self is supertype: + if self is supertype or supertype is get_object_type(): return Substitutions() if supertype.kind != IndividualKind(): raise concat.typecheck.TypeError( @@ -397,6 +273,11 @@ def constrain_and_bind_variables( raise concat.typecheck.TypeError( '{} must be a sequence type, not {}'.format(self, supertype) ) + if ( + isinstance(supertype, SequenceVariable) + and supertype not in rigid_variables + ): + return Substitutions([(supertype, self)]) if self in rigid_variables: raise concat.typecheck.TypeError( '{} is fixed here and cannot become a subtype of another type'.format( @@ -410,8 +291,6 @@ def constrain_and_bind_variables( self, supertype, supertype ) ) - if isinstance(supertype, SequenceVariable): - return Substitutions([(supertype, self)]) return Substitutions([(self, supertype)]) def get_type_of_attribute(self, name: str) -> NoReturn: @@ -433,6 +312,142 @@ def kind(self) -> 'Kind': return SequenceKind() +class GenericType(Type): + def __init__( + self, + type_parameters: Sequence['_Variable'], + body: Type, + is_variadic: bool = False, + ) -> None: + super().__init__() + assert type_parameters + self._type_parameters = type_parameters + if body.kind != IndividualKind(): + raise concat.typecheck.TypeError( + f'Cannot be polymorphic over non-individual type {body}' + ) + self._body = body + self._instantiations: Dict[Tuple[Type, ...], Type] = {} + self.is_variadic = is_variadic + + def __str__(self) -> str: + if self._internal_name is not None: + return self._internal_name + if self.is_variadic: + params = str(self._type_parameters[0]) + '...' + else: + params = ' '.join(map(str, self._type_parameters)) + + return f'forall {params}. {self._body}' + + def __repr__(self) -> str: + return f'{type(self).__qualname__}({self._type_parameters!r}, {self._body!r}, is_variadic={self.is_variadic!r})' + + def __getitem__(self, type_arguments: 'TypeArguments') -> 'Type': + from concat.typecheck import Substitutions + + type_arguments = tuple(type_arguments) + if type_arguments in self._instantiations: + return self._instantiations[type_arguments] + expected_kinds = [var.kind for var in self._type_parameters] + actual_kinds = [ty.kind for ty in type_arguments] + if expected_kinds != actual_kinds: + raise concat.typecheck.TypeError( + f'A type argument to {self} has the wrong kind, type arguments: {type_arguments}, expected kinds: {expected_kinds}' + ) + sub = Substitutions(zip(self._type_parameters, type_arguments)) + instance = sub(self._body) + self._instantiations[type_arguments] = instance + if self._internal_name is not None: + instance_internal_name = self._internal_name + instance_internal_name += ( + '[' + ', '.join(map(str, type_arguments)) + ']' + ) + instance.set_internal_name(instance_internal_name) + return instance + + @property + def kind(self) -> 'Kind': + kinds = [var.kind for var in self._type_parameters] + return GenericTypeKind(kinds) + + def resolve_forward_references(self) -> 'GenericType': + self._body = self._body.resolve_forward_references() + return self + + def instantiate(self) -> Type: + fresh_vars: Sequence[_Variable] = [ + type(var)() for var in self._type_parameters + ] + return self[fresh_vars] + + def constrain_and_bind_variables( + self, + supertype: 'Type', + rigid_variables: AbstractSet['_Variable'], + subtyping_assumptions: List[Tuple['IndividualType', 'IndividualType']], + ) -> 'Substitutions': + from concat.typecheck import Substitutions + + if self is supertype or _contains_assumption( + subtyping_assumptions, self, supertype + ): + return Substitutions([]) + if self.kind != supertype.kind: + raise concat.typecheck.TypeError( + f'{self} has kind {self.kind} but {supertype} has kind {supertype.kind}' + ) + if not isinstance(supertype, GenericType): + raise NotImplementedError(supertype) + shared_vars = [type(var)() for var in self._type_parameters] + self_instance = self[shared_vars] + supertype_instance = supertype[shared_vars] + rigid_variables = ( + rigid_variables + | set(self._type_parameters) + | set(supertype._type_parameters) + ) + return self_instance.constrain_and_bind_variables( + supertype_instance, rigid_variables, subtyping_assumptions + ) + + def __hash__(self) -> int: + # QUESTION: Is it better for perf to compute de Bruijn indices instead + # of using substitution? + vars_for_hash: List[_Variable] = [] + for p in self._type_parameters: + if p.kind == IndividualKind(): + vars_for_hash.append(self._individual_var_for_hash) + else: + vars_for_hash.append(self._sequence_var_for_hash) + return hash(self[vars_for_hash]) + + _individual_var_for_hash = IndividualVariable() + _sequence_var_for_hash = SequenceVariable() + + def apply_substitution(self, sub: 'Substitutions') -> 'GenericType': + from concat.typecheck import Substitutions + + sub = Substitutions( + { + var: ty + for var, ty in sub.items() + if var not in self._type_parameters + } + ) + ty = GenericType(self._type_parameters, sub(self._body)) + return ty + + @property + def attributes(self) -> NoReturn: + raise concat.typecheck.TypeError( + 'Generic types do not have attributes; maybe you forgot type arguments?' + ) + + def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: + return self._body.free_type_variables() - set(self._type_parameters) + + class TypeSequence(Type, Iterable['StackItemType']): def __init__(self, sequence: Sequence['StackItemType']) -> None: super().__init__() @@ -819,7 +834,7 @@ def __init__( self, self_type: IndividualVariable, attributes: Mapping[str, Type], - nominal_supertypes: Sequence[IndividualType] = (), + nominal_supertypes: Sequence[Type] = (), nominal: bool = False, _head: Optional['ObjectType'] = None, ) -> None: @@ -832,7 +847,13 @@ def __init__( self._attributes = attributes + for t in nominal_supertypes: + if t.kind != IndividualKind(): + raise concat.typecheck.TypeError( + f'{t} must be an individual type, but has kind {t.kind}' + ) self._nominal_supertypes = nominal_supertypes + self._nominal = nominal self._head = _head or self @@ -944,6 +965,10 @@ def constrain_and_bind_variables( rigid_variables, subtyping_assumptions + [(self, supertype)], ) + if isinstance(supertype, _NoReturnType): + raise concat.typecheck.TypeError( + f'No other type, in this case, {self}, is a subtype of {supertype}' + ) if not isinstance(supertype, ObjectType): raise NotImplementedError(supertype) # every object type is a subtype of object_type @@ -1037,11 +1062,11 @@ def head(self) -> 'ObjectType': return self._head @property - def nominal_supertypes(self) -> Sequence[IndividualType]: + def nominal_supertypes(self) -> Sequence[Type]: return self._nominal_supertypes -# QUESTION: Should this exist, or should I use ObjectType +# QUESTION: Should this exist, or should I use ObjectType? class ClassType(ObjectType): """The representation of types of classes, like in "Design and Evaluation of Gradual Typing for Python" (Vitousek et al. 2014).""" @@ -1069,9 +1094,7 @@ def constrain_and_bind_variables( class PythonFunctionType(IndividualType): def __init__( self, - _overloads: Sequence[ - Tuple[Sequence[StackItemType], IndividualType] - ] = (), + _overloads: Sequence[Tuple[Type, Type]] = (), type_parameters: Sequence[_Variable] = (), _type_arguments: Sequence[Type] = (), ) -> None: @@ -1089,16 +1112,30 @@ def __init__( f'Ill-formed Python function type with arguments {self._type_arguments}' ) if self._arity == 0: - assert isinstance(self.input, collections.abc.Sequence) - assert self._type_arguments[1].kind == IndividualKind() - self._overloads = _overloads + i, o = _type_arguments + if i.kind != SequenceKind(): + raise concat.typecheck.TypeError( + f'{i} must be a sequence type, but has kind {i.kind}' + ) + if o.kind != IndividualKind(): + raise concat.typecheck.TypeError( + f'{o} must be an individual type, but has kind {o.kind}' + ) + for i, o in _overloads: + if i.kind != SequenceKind(): + raise concat.typecheck.TypeError( + f'{i} must be a sequence type, but has kind {i.kind}' + ) + if o.kind != IndividualKind(): + raise concat.typecheck.TypeError( + f'{o} must be an individual type, but has kind {o.kind}' + ) + self._overloads = _overloads self._hash: Optional[int] = None def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: if self._arity == 0: - ftv = InsertionOrderedSet[_Variable]([]) - for ty in self.input: - ftv |= ty.free_type_variables() + ftv = self.input.free_type_variables() ftv |= self.output.free_type_variables() return ftv else: @@ -1114,11 +1151,11 @@ def resolve_forward_references(self) -> 'PythonFunctionType': if self._forward_references_resolved: return self super().resolve_forward_references() - overloads: List[Tuple[Sequence[StackItemType], IndividualType]] = [] + overloads: List[Tuple[Type, Type]] = [] for args, ret in overloads: overloads.append( ( - [arg.resolve_forward_references() for arg in args], + args.resolve_forward_references(), ret.resolve_forward_references(), ) ) @@ -1135,10 +1172,7 @@ def __eq__(self, other: object) -> bool: return False if isinstance(self.kind, GenericTypeKind): return True - return ( - tuple(self.input) == tuple(other.input) - and self.output == other.output - ) + return self.input == other.input and self.output == other.output def __hash__(self) -> int: if self._hash is None: @@ -1172,7 +1206,7 @@ def attributes(self) -> Mapping[str, Type]: return {**super().attributes, '__call__': self} def __getitem__( - self, arguments: Tuple[TypeSequence, IndividualType] + self, arguments: Tuple[Type, Type] ) -> 'PythonFunctionType': if self._arity != 2: raise concat.typecheck.TypeError(f'{self} is not a generic type') @@ -1200,8 +1234,8 @@ def apply_substitution( if self._arity == 0: inp = sub(TypeSequence(self.input)) out = sub(self.output) - overloads: Sequence[Tuple[TypeSequence, IndividualType]] = [ - (sub(TypeSequence(i)), sub(o)) for i, o in self._overloads + overloads: Sequence[Tuple[Type, Type]] = [ + (sub(i), sub(o)) for i, o in self._overloads ] return PythonFunctionType( _type_arguments=(inp, out), _overloads=overloads @@ -1209,17 +1243,13 @@ def apply_substitution( return self @property - def input(self) -> Sequence[StackItemType]: + def input(self) -> Type: assert self._arity == 0 - if isinstance(self._type_arguments[0], SequenceVariable): - return (self._type_arguments[0],) - assert not isinstance(self._type_arguments[0], IndividualType) - return tuple(self._type_arguments[0]) + return self._type_arguments[0] @property - def output(self) -> IndividualType: + def output(self) -> Type: assert self._arity == 0 - assert self._type_arguments[1].kind == IndividualKind() return self._type_arguments[1] def select_overload( @@ -1240,9 +1270,7 @@ def select_overload( 'no overload of {} matches types {}'.format(self, input_types) ) - def with_overload( - self, input: Sequence[StackItemType], output: IndividualType - ) -> 'PythonFunctionType': + def with_overload(self, input: Type, output: Type) -> 'PythonFunctionType': return PythonFunctionType( _type_arguments=self._type_arguments, _overloads=[*self._overloads, (input, output)], @@ -1275,6 +1303,8 @@ def constrain_and_bind_variables( f'{self} has kind {self.kind} but {supertype} has kind {supertype.kind}' ) if self.kind == IndividualKind(): + if supertype is get_object_type(): + return Substitutions() if ( isinstance(supertype, IndividualVariable) and supertype not in rigid_variables @@ -1347,13 +1377,16 @@ def __getitem__(self, args: Sequence[Type]) -> 'PythonFunctionType': raise concat.typecheck.TypeError( 'py_overloaded must be applied to at least one argument' ) - for arg in args: + fun_type = args[0] + if not isinstance(fun_type, PythonFunctionType): + raise concat.typecheck.TypeError( + 'Arguments to py_overloaded must be Python function types' + ) + for arg in args[1:]: if not isinstance(arg, PythonFunctionType): raise concat.typecheck.TypeError( 'Arguments to py_overloaded must be Python function types' ) - fun_type = args[0] - for arg in args[1:]: fun_type = fun_type.with_overload(arg.input, arg.output) return fun_type @@ -1401,13 +1434,13 @@ def __eq__(self, other: object) -> bool: py_overloaded_type = _PythonOverloadedType() -class _NoReturnType(ObjectType): - def __init__(self) -> None: - x = IndividualVariable() - super().__init__(x, {}) +class _NoReturnType(IndividualType): + def constrain_and_bind_variables( + self, supertype, rigid_variables, subtyping_assumptions + ) -> 'Substitutions': + from concat.typecheck import Substitutions - def is_subtype_of(self, _: Type) -> Literal[True]: - return True + return Substitutions() def apply_substitution( self, sub: 'concat.typecheck.Substitutions' @@ -1417,6 +1450,15 @@ def apply_substitution( def __repr__(self) -> str: return '{}()'.format(type(self).__qualname__) + def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: + return InsertionOrderedSet([]) + + def resolve_forward_references(self) -> Self: + return self + + def __hash__(self) -> int: + return 0 + class _OptionalType(IndividualType): def __init__(self, type_argument: IndividualType) -> None: From eefc36ad403e971ee8198d94fcf929300e0dc655 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Mon, 3 Jun 2024 21:24:47 -0700 Subject: [PATCH 21/61] Try to get test_add_operator_inference to pass again * Fix type of map * Separate fixpoints from ObjectType * Don't immediately resolve all forward references * Use the integer type defined in stubs * Introduce ids for substitution caching * Move type error clssses to new module --- concat/stdlib/pyinterop/__init__.py | 31 +- concat/tests/strategies.py | 4 +- concat/tests/test_typecheck.py | 54 ++- concat/tests/typecheck/test_types.py | 106 ++++- concat/typecheck/__init__.py | 241 ++++------ concat/typecheck/errors.py | 79 ++++ concat/typecheck/preamble.cati | 15 + concat/typecheck/preamble_types.py | 2 - concat/typecheck/py_builtins.cati | 6 +- concat/typecheck/types.py | 649 ++++++++++++++++----------- mypy.ini | 1 + tox.ini | 1 + 12 files changed, 717 insertions(+), 472 deletions(-) create mode 100644 concat/typecheck/errors.py diff --git a/concat/stdlib/pyinterop/__init__.py b/concat/stdlib/pyinterop/__init__.py index c06c189..9cad93c 100644 --- a/concat/stdlib/pyinterop/__init__.py +++ b/concat/stdlib/pyinterop/__init__.py @@ -50,24 +50,21 @@ TypeSequence([_stack_type_var, _y]), ), ), - 'map': ObjectType( - _x, - { - '__call__': StackEffect( - TypeSequence( - [ - _rest_var, - StackEffect( - TypeSequence([_rest_var, _y]), - TypeSequence([_rest_var, _z]), - ), - iterable_type[_y,], - ] - ), - TypeSequence([_rest_var, iterable_type[_z,]]), - ) - }, + 'map': GenericType( [_rest_var, _y, _z], + StackEffect( + TypeSequence( + [ + _rest_var, + StackEffect( + TypeSequence([_rest_var, _y]), + TypeSequence([_rest_var, _z]), + ), + iterable_type[_y,], + ] + ), + TypeSequence([_rest_var, iterable_type[_z,]]), + ), ), 'to_dict': GenericType( [_stack_type_var, _x, _y], diff --git a/concat/tests/strategies.py b/concat/tests/strategies.py index 8a57402..d8b4e90 100644 --- a/concat/tests/strategies.py +++ b/concat/tests/strategies.py @@ -5,6 +5,7 @@ PythonFunctionType, SequenceVariable, StackEffect, + StuckTypeApplication, TypeSequence, none_type, no_return_type, @@ -104,7 +105,8 @@ def _mark_individual_type_strategy( ) for subclass in _individual_type_subclasses: - assert subclass in _individual_type_strategies, subclass + if subclass not in [StuckTypeApplication]: + assert subclass in _individual_type_strategies, subclass register_type_strategy(IndividualType, _individual_type_strategy) register_type_strategy( diff --git a/concat/tests/test_typecheck.py b/concat/tests/test_typecheck.py index 646bbb6..5860885 100644 --- a/concat/tests/test_typecheck.py +++ b/concat/tests/test_typecheck.py @@ -15,7 +15,7 @@ ellipsis_type, float_type, get_object_type, - int_type, + get_int_type, no_return_type, none_type, not_implemented_type, @@ -40,7 +40,6 @@ default_env = concat.typecheck.load_builtins_and_preamble() -default_env.resolve_forward_references() def lex_string(string: str) -> List[concat.lex.Token]: @@ -70,32 +69,34 @@ def test_attribute_word(self, attr_word) -> None: initial_stack=TypeSequence( [ ObjectType( - IndividualVariable(), { attr_word.value: StackEffect( - TypeSequence([]), TypeSequence([int_type]) + TypeSequence([]), + TypeSequence([get_int_type()]), ), }, ), ] ), ) - self.assertEqual(list(type.output), [int_type]) + self.assertEqual(list(type.output), [get_int_type()]) - @given(integers(min_value=0), integers(min_value=0)) - def test_add_operator_inference(self, a: int, b: int) -> None: - try_prog = '{!r} {!r} +\n'.format(a, b) + def test_add_operator_inference(self) -> None: + try_prog = '0 0 +\n' tree = parse(try_prog) sub, type, _ = concat.typecheck.infer( concat.typecheck.Environment({'+': default_env['+']}), tree.children, is_top_level=True, ) - note(repr(type)) - note(str(sub)) - note(repr(default_env['+'])) + print( + 'explain!:', + type + == StackEffect(TypeSequence([]), TypeSequence([get_int_type()])), + ) + self.assertEqual( - type, StackEffect(TypeSequence([]), TypeSequence([int_type])) + type, StackEffect(TypeSequence([]), TypeSequence([get_int_type()])) ) def test_if_then_inference(self) -> None: @@ -121,7 +122,7 @@ def test_call_inference(self) -> None: is_top_level=True, ) self.assertEqual( - type, StackEffect(TypeSequence([]), TypeSequence([int_type])) + type, StackEffect(TypeSequence([]), TypeSequence([get_int_type()])) ) @given(sampled_from(['None', '...', 'NotImplemented'])) @@ -183,7 +184,7 @@ def test_cast_word(self) -> None: is_top_level=True, ) self.assertEqual( - type, StackEffect(TypeSequence([]), TypeSequence([int_type])) + type, StackEffect(TypeSequence([]), TypeSequence([get_int_type()])) ) @@ -296,9 +297,8 @@ def test_attribute_error_location(self) -> None: class TestTypeEquality(unittest.TestCase): @given(from_type(ConcatType)) - @example(type=int_type.self_type) - @example(type=int_type.get_type_of_attribute('__add__')) - @example(type=int_type) + @example(type=get_int_type().get_type_of_attribute('__add__')) + @example(type=get_int_type()) @settings(suppress_health_check=(HealthCheck.filter_too_much,)) def test_reflexive_equality(self, type): self.assertEqual(type, type) @@ -307,7 +307,7 @@ def test_reflexive_equality(self, type): class TestSubtyping(unittest.TestCase): def test_int_not_subtype_of_float(self) -> None: """Differ from Reticulated Python: !(int <= float).""" - self.assertFalse(int_type.is_subtype_of(float_type)) + self.assertFalse(get_int_type().is_subtype_of(float_type)) @given(from_type(IndividualType), from_type(IndividualType)) @settings(suppress_health_check=(HealthCheck.filter_too_much,)) @@ -342,9 +342,8 @@ def test_object_is_top_type(self, type) -> None: def test_object_structural_subtyping( self, attributes, other_attributes ) -> None: - x1, x2 = IndividualVariable(), IndividualVariable() - object1 = ObjectType(x1, {**other_attributes, **attributes}) - object2 = ObjectType(x2, attributes) + object1 = ObjectType({**other_attributes, **attributes}) + object2 = ObjectType(attributes) self.assertTrue(object1.is_subtype_of(object2)) @given(__attributes_generator, __attributes_generator) @@ -352,16 +351,14 @@ def test_object_structural_subtyping( def test_class_structural_subtyping( self, attributes, other_attributes ) -> None: - x1, x2 = IndividualVariable(), IndividualVariable() - object1 = ClassType(x1, {**other_attributes, **attributes}) - object2 = ClassType(x2, attributes) + object1 = ClassType({**other_attributes, **attributes}) + object2 = ClassType(attributes) self.assertTrue(object1.is_subtype_of(object2)) @given(from_type(StackEffect)) @settings(suppress_health_check=(HealthCheck.filter_too_much,)) def test_object_subtype_of_stack_effect(self, effect) -> None: - x = IndividualVariable() - object = ObjectType(x, {'__call__': effect}) + object = ObjectType({'__call__': effect}) self.assertTrue(object.is_subtype_of(effect)) @given(from_type(IndividualType), from_type(IndividualType)) @@ -372,9 +369,8 @@ def test_object_subtype_of_stack_effect(self, effect) -> None: ) ) def test_object_subtype_of_py_function(self, type1, type2) -> None: - x = IndividualVariable() py_function = py_function_type[TypeSequence([type1]), type2] - object = ObjectType(x, {'__call__': py_function}) + object = ObjectType({'__call__': py_function}) self.assertTrue(object.is_subtype_of(py_function)) @given(from_type(StackEffect)) @@ -398,7 +394,7 @@ def test_class_subtype_of_py_function(self, type1, type2) -> None: x = IndividualVariable() py_function = py_function_type[TypeSequence([type1]), type2] unbound_py_function = py_function_type[TypeSequence([x, type1]), type2] - cls = ClassType(x, {'__init__': unbound_py_function}) + cls = ClassType({'__init__': unbound_py_function}) self.assertTrue(cls.is_subtype_of(py_function)) @given(from_type(IndividualType)) diff --git a/concat/tests/typecheck/test_types.py b/concat/tests/typecheck/test_types.py index c3e67fd..2ef109c 100644 --- a/concat/tests/typecheck/test_types.py +++ b/concat/tests/typecheck/test_types.py @@ -1,16 +1,25 @@ from concat.typecheck import ( + Environment, + Substitutions, TypeError as ConcatTypeError, load_builtins_and_preamble, ) from concat.typecheck.types import ( + BoundVariable, + Fix, + ForwardTypeReference, + IndividualKind, IndividualVariable, ObjectType, SequenceVariable, StackEffect, TypeSequence, addable_type, - int_type, + get_int_type, + get_object_type, + optional_type, py_function_type, + tuple_type, ) import unittest @@ -21,75 +30,75 @@ class TestIndividualVariableConstrain(unittest.TestCase): def test_individual_variable_subtype(self) -> None: v = IndividualVariable() - ty = int_type + ty = get_int_type() sub = v.constrain_and_bind_variables(ty, set(), []) self.assertEqual(ty, sub(v)) def test_individual_variable_supertype(self) -> None: v = IndividualVariable() - ty = int_type + ty = get_int_type() sub = ty.constrain_and_bind_variables(v, set(), []) self.assertEqual(ty, sub(v)) def test_attribute_subtype(self) -> None: v = IndividualVariable() - attr_ty = ObjectType(IndividualVariable(), {'__add__': v}) - ty = int_type + attr_ty = ObjectType({'__add__': v}) + ty = get_int_type() with self.assertRaises(ConcatTypeError): attr_ty.constrain_and_bind_variables(ty, set(), []) def test_attribute_supertype(self) -> None: v = IndividualVariable() - attr_ty = ObjectType(IndividualVariable(), {'__add__': v}) - ty = int_type + attr_ty = ObjectType({'__add__': v}) + ty = get_int_type() sub = ty.constrain_and_bind_variables(attr_ty, set(), []) self.assertEqual(ty.get_type_of_attribute('__add__'), sub(v)) def test_py_function_return_subtype(self) -> None: v = IndividualVariable() - py_fun_ty = py_function_type[TypeSequence([int_type]), v] - ty = int_type.get_type_of_attribute('__add__') + py_fun_ty = py_function_type[TypeSequence([get_int_type()]), v] + ty = get_int_type().get_type_of_attribute('__add__') sub = py_fun_ty.constrain_and_bind_variables(ty, set(), []) - self.assertEqual(int_type, sub(v)) + self.assertEqual(get_int_type(), sub(v)) def test_py_function_return_supertype(self) -> None: v = IndividualVariable() - py_fun_ty = py_function_type[TypeSequence([int_type]), v] - ty = int_type.get_type_of_attribute('__add__') + py_fun_ty = py_function_type[TypeSequence([get_int_type()]), v] + ty = get_int_type().get_type_of_attribute('__add__') sub = ty.constrain_and_bind_variables(py_fun_ty, set(), []) - self.assertEqual(int_type, sub(v)) + self.assertEqual(get_int_type(), sub(v)) def test_type_sequence_subtype(self) -> None: v = IndividualVariable() seq_ty = TypeSequence([v]) - ty = TypeSequence([int_type]) + ty = TypeSequence([get_int_type()]) sub = seq_ty.constrain_and_bind_variables(ty, set(), []) - self.assertEqual(int_type, sub(v)) + self.assertEqual(get_int_type(), sub(v)) def test_type_sequence_supertype(self) -> None: v = IndividualVariable() seq_ty = TypeSequence([v]) - ty = TypeSequence([int_type]) + ty = TypeSequence([get_int_type()]) sub = ty.constrain_and_bind_variables(seq_ty, set(), []) - self.assertEqual(int_type, sub(v)) + self.assertEqual(get_int_type(), sub(v)) def test_int_addable(self) -> None: v = IndividualVariable() - sub = int_type.constrain_and_bind_variables( + sub = get_int_type().constrain_and_bind_variables( addable_type[v, v], set(), [] ) - self.assertEqual(int_type, sub(v)) + self.assertEqual(get_int_type(), sub(v)) def test_int__add__addable__add__(self) -> None: v = IndividualVariable() - int_add = int_type.get_type_of_attribute('__add__') + int_add = get_int_type().get_type_of_attribute('__add__') addable_add = addable_type[v, v].get_type_of_attribute('__add__') sub = int_add.constrain_and_bind_variables(addable_add, set(), []) print(v) print(int_add) print(addable_add) print(sub) - self.assertEqual(int_type, sub(v)) + self.assertEqual(get_int_type(), sub(v)) class TestSequenceVariableConstrain(unittest.TestCase): @@ -106,3 +115,58 @@ def test_stack_effect_input_supertype(self) -> None: ty = StackEffect(TypeSequence([]), TypeSequence([])) sub = ty.constrain_and_bind_variables(effect_ty, set(), []) self.assertEqual(TypeSequence([]), sub(v)) + + +class TestFix(unittest.TestCase): + fix_var = BoundVariable(IndividualKind()) + linked_list = Fix( + fix_var, + optional_type[ + tuple_type[TypeSequence([get_object_type(), fix_var]),], + ], + ) + + def test_unroll_supertype(self) -> None: + self.assertEqual( + Substitutions(), + self.linked_list.constrain_and_bind_variables( + self.linked_list.unroll(), set(), [] + ), + ) + + def test_unroll_subtype(self) -> None: + self.assertEqual( + Substitutions(), + self.linked_list.unroll().constrain_and_bind_variables( + self.linked_list, set(), [] + ), + ) + + def test_unroll_equal(self) -> None: + self.assertEqual(self.linked_list.unroll(), self.linked_list) + self.assertEqual(self.linked_list, self.linked_list.unroll()) + + +class TestForwardReferences(unittest.TestCase): + env = Environment({'ty': get_object_type()}) + ty = ForwardTypeReference(IndividualKind(), 'ty', env) + + def test_resolve_supertype(self) -> None: + self.assertEqual( + Substitutions(), + self.ty.constrain_and_bind_variables( + self.ty.resolve_forward_references(), set(), [] + ), + ) + + def test_resolve_subtype(self) -> None: + self.assertEqual( + Substitutions(), + self.ty.resolve_forward_references().constrain_and_bind_variables( + self.ty, set(), [] + ), + ) + + def test_resolve_equal(self) -> None: + self.assertEqual(self.ty.resolve_forward_references(), self.ty) + self.assertEqual(self.ty, self.ty.resolve_forward_references()) diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index 0756574..b86f06b 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -5,7 +5,12 @@ """ -import collections.abc +from concat.typecheck.errors import ( + NameError, + StaticAnalysisError, + TypeError, + UnhandledNodeTypeError, +) from typing import ( Callable, Dict, @@ -26,6 +31,30 @@ from typing_extensions import Protocol +if TYPE_CHECKING: + import concat.astutils + from concat.orderedset import InsertionOrderedSet + from concat.typecheck.types import Type, _Variable + + +class Environment(Dict[str, 'Type']): + _next_id = -1 + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.id = Environment._next_id + Environment._next_id -= 1 + + def copy(self) -> 'Environment': + return Environment(super().copy()) + + def apply_substitution(self, sub: 'Substitutions') -> 'Environment': + return Environment({name: sub(t) for name, t in self.items()}) + + def free_type_variables(self) -> 'InsertionOrderedSet[_Variable]': + return free_type_variables_of_mapping(self) + + _Result = TypeVar('_Result', covariant=True) @@ -34,7 +63,7 @@ def apply_substitution(self, sub: 'Substitutions') -> _Result: pass -class Substitutions(collections.abc.Mapping, Mapping['_Variable', 'Type']): +class Substitutions(Mapping['_Variable', 'Type']): def __init__( self, sub: Union[ @@ -47,6 +76,7 @@ def __init__( raise TypeError( f'{variable} is being substituted by {ty}, which has the wrong kind ({variable.kind} vs {ty.kind})' ) + self._cache: Dict[int, Type] = {} def __getitem__(self, var: '_Variable') -> 'Type': return self._sub[var] @@ -57,12 +87,27 @@ def __iter__(self) -> Iterator['_Variable']: def __len__(self) -> int: return len(self._sub) + def __bool__(self) -> bool: + return bool(self._sub) + def __call__(self, arg: _Substitutable[_Result]) -> _Result: + from concat.typecheck.types import Type + + result: _Result # Previously I tried caching results by the id of the argument. But # since the id is the memory address of the object in CPython, another # object might have the same id later. I think this was leading to # nondeterministic Concat type errors from the type checker. - return arg.apply_substitution(self) + if isinstance(arg, Type): + if arg._type_id not in self._cache: + self._cache[arg._type_id] = arg.apply_substitution(self) + result = self._cache[arg._type_id] + if isinstance(arg, Environment): + if arg.id not in self._cache: + self._cache[arg.id] = arg.apply_substitution(self) + result = self._cache[arg.id] + result = arg.apply_substitution(self) + return result def _dom(self) -> Set['_Variable']: return {*self} @@ -89,14 +134,16 @@ def __hash__(self) -> int: from concat.typecheck.types import ( + BoundVariable, + Fix, ForwardTypeReference, GenericType, GenericTypeKind, IndividualKind, IndividualType, IndividualVariable, + Kind, ObjectType, - PythonFunctionType, QuotationType, SequenceKind, SequenceVariable, @@ -104,28 +151,19 @@ def __hash__(self) -> int: StackItemType, Type, TypeSequence, - bool_type, - context_manager_type, - ellipsis_type, free_type_variables_of_mapping, + get_int_type, get_list_type, get_object_type, - int_type, - invertible_type, - iterable_type, + get_str_type, module_type, no_return_type, - none_type, - not_implemented_type, py_function_type, - slice_type, - get_str_type, subscriptable_type, subtractable_type, tuple_type, ) import abc -import builtins from concat.error_reporting import create_parsing_failure_message from concat.lex import Token import functools @@ -138,97 +176,6 @@ def __hash__(self) -> int: import concat.parse -if TYPE_CHECKING: - import concat.astutils - from concat.orderedset import InsertionOrderedSet - from concat.typecheck.types import _Variable - - -class StaticAnalysisError(Exception): - def __init__(self, message: str) -> None: - self.message = message - self.location: Optional['concat.astutils.Location'] = None - self.path: Optional[pathlib.Path] = None - - def set_location_if_missing( - self, location: 'concat.astutils.Location' - ) -> None: - if not self.location: - self.location = location - - def set_path_if_missing(self, path: pathlib.Path) -> None: - if self.path is None: - self.path = path - - def __str__(self) -> str: - return '{} at {}'.format(self.message, self.location) - - -class TypeError(StaticAnalysisError, builtins.TypeError): - pass - - -class NameError(StaticAnalysisError, builtins.NameError): - def __init__( - self, - name: Union[concat.parse.NameWordNode, str], - location: Optional[concat.astutils.Location] = None, - ) -> None: - if isinstance(name, concat.parse.NameWordNode): - location = name.location - name = name.value - super().__init__(f'name {name!r} not previously defined') - self._name = name - self.location = location - - def __str__(self) -> str: - location_info = '' - if self.location: - location_info = ' (error at {}:{})'.format(*self.location) - return self.message + location_info - - -class AttributeError(TypeError, builtins.AttributeError): - def __init__(self, type: 'Type', attribute: str) -> None: - super().__init__( - 'object of type {} does not have attribute {}'.format( - type, attribute - ) - ) - self._type = type - self._attribute = attribute - - -class StackMismatchError(TypeError): - def __init__( - self, actual: 'TypeSequence', expected: 'TypeSequence' - ) -> None: - super().__init__( - 'The stack here is {}, but sequence type {} was expected'.format( - actual, expected - ) - ) - - -class UnhandledNodeTypeError(builtins.NotImplementedError): - pass - - -class Environment(Dict[str, Type]): - def copy(self) -> 'Environment': - return Environment(super().copy()) - - def apply_substitution(self, sub: 'Substitutions') -> 'Environment': - return Environment({name: sub(t) for name, t in self.items()}) - - def free_type_variables(self) -> 'InsertionOrderedSet[_Variable]': - return free_type_variables_of_mapping(self) - - def resolve_forward_references(self) -> None: - for name, t in self.items(): - self[name] = t.resolve_forward_references() - - def load_builtins_and_preamble() -> Environment: env = _check_stub( pathlib.Path(__file__).with_name('py_builtins.cati'), is_builtins=True, @@ -283,7 +230,6 @@ def check( source_dir, check_bodies=_should_check_bodies, ) - # res[2].resolve_forward_references() return res[2] @@ -313,13 +259,17 @@ def infer( for node in e: if isinstance(node, concat.parse.ClassdefStatementNode): type_name = node.class_name - kind = IndividualKind() + kind: Kind = IndividualKind() type_parameters = [] + parameter_kinds: Sequence[Kind] if node.is_variadic: parameter_kinds = [SequenceKind()] else: for param in node.type_parameters: - type_parameters.append(param.to_type(gamma)[0]) + if isinstance(param, TypeNode): + type_parameters.append(param.to_type(gamma)[0]) + continue + raise UnhandledNodeTypeError(param) if type_parameters: parameter_kinds = [ variable.kind for variable in type_parameters @@ -332,15 +282,21 @@ def infer( S, (i, o) = current_subs, current_effect if isinstance(node, concat.parse.PragmaNode): - if node.pragma == 'concat.typecheck.builtin_object': + namespace = 'concat.typecheck.' + if node.pragma.startswith(namespace): + pragma = node.pragma[len(namespace) :] + if pragma == 'builtin_object': name = node.args[0] concat.typecheck.types.set_object_type(gamma[name]) - if node.pragma == 'concat.typecheck.builtin_list': + if pragma == 'builtin_list': name = node.args[0] concat.typecheck.types.set_list_type(gamma[name]) - if node.pragma == 'concat.typecheck.builtin_str': + if pragma == 'builtin_str': name = node.args[0] concat.typecheck.types.set_str_type(gamma[name]) + if pragma == 'builtin_int': + name = node.args[0] + concat.typecheck.types.set_int_type(gamma[name]) elif isinstance(node, concat.parse.PushWordNode): S1, (i1, o1) = S, (i, o) child = node.children[0] @@ -495,7 +451,10 @@ def infer( if module_spec is not None: path = module_spec.submodule_search_locations break + assert module_spec is not None module_path = module_spec.origin + if module_path is None: + raise TypeError(f'Cannot find path of module {node.value}') # For now, assume the module's written in Python. stub_path = pathlib.Path(module_path).with_suffix('.cati') stub_env = _check_stub(stub_path) @@ -570,7 +529,7 @@ def infer( # type check decorators _, final_type_stack, _ = infer( gamma, - node.decorators, + list(node.decorators), is_top_level=False, extensions=extensions, initial_stack=TypeSequence([effect]), @@ -595,7 +554,7 @@ def infer( elif isinstance(node, concat.parse.NumberWordNode): if isinstance(node.value, int): current_effect = StackEffect( - i, TypeSequence([*o, int_type]) + i, TypeSequence([*o, get_int_type()]) ) else: raise UnhandledNodeTypeError @@ -619,8 +578,6 @@ def infer( constraint_subs = o1.constrain_and_bind_variables( type_of_name.input, set(), [] ) - print(repr(o1)) - print(constraint_subs) current_subs = constraint_subs(current_subs) current_effect = current_subs( StackEffect(i1, type_of_name.output) @@ -689,15 +646,22 @@ def infer( elif not check_bodies and isinstance( node, concat.parse.ClassdefStatementNode ): - type_parameters = [] - temp_gamma = gamma + type_parameters: List[_Variable] = [] + temp_gamma = gamma.copy() if node.is_variadic: type_parameters.append(SequenceVariable()) else: for param_node in node.type_parameters: + if not isinstance(param_node, TypeNode): + raise UnhandledNodeTypeError(param_node) param, temp_gamma = param_node.to_type(temp_gamma) type_parameters.append(param) + kind: Kind = IndividualKind() + if type_parameters: + kind = GenericTypeKind([v.kind for v in type_parameters]) + self_type = BoundVariable(kind) + temp_gamma[node.class_name] = self_type _, _, body_attrs = infer( temp_gamma, node.children, @@ -714,18 +678,16 @@ def infer( if name not in temp_gamma } ) - ty = ObjectType( - self_type=IndividualVariable(), - attributes=body_attrs, - nominal_supertypes=(), - nominal=True, + ty: Type = ObjectType( + attributes=body_attrs, nominal_supertypes=(), nominal=True, ) if type_parameters: ty = GenericType( type_parameters, ty, is_variadic=node.is_variadic ) + ty = Fix(self_type, ty) + ty.set_internal_name(node.class_name) gamma[node.class_name] = ty - gamma[node.class_name].set_internal_name(node.class_name) # elif isinstance(node, concat.parse.TypeAliasStatementNode): # gamma[node.name], _ = node.type_node.to_type(gamma) else: @@ -749,7 +711,6 @@ def _check_stub_resolved_path( except IOError as e: raise TypeError(f'Failed to read type stubs at {path}') from e tokens = concat.lex.tokenize(source) - # print(tokens) env = Environment() from concat.transpile import parse @@ -1120,7 +1081,8 @@ def __init__( location: concat.astutils.Location, end_location: concat.astutils.Location, ) -> None: - super().__init__(location, end_location) + super().__init__(location) + self.end_location = end_location self._attribute_type_pairs = attribute_type_pairs def to_type(self, env: Environment) -> Tuple[ObjectType, Environment]: @@ -1129,11 +1091,9 @@ def to_type(self, env: Environment) -> Tuple[ObjectType, Environment]: for attribute, type_node in self._attribute_type_pairs: ty, temp_env = type_node.to_type(temp_env) attribute_type_mapping[attribute.value] = ty + # FIXME: Support recursive types in syntax return ( - ObjectType( - self_type=IndividualVariable(), # FIXME: Support recursive types in syntax - attributes=attribute_type_mapping, - ), + ObjectType(attributes=attribute_type_mapping,), env, ) @@ -1337,17 +1297,13 @@ def _generate_type_of_innermost_module( for name in dir(module): attribute_type = get_object_type() if isinstance(getattr(module, name), int): - attribute_type = int_type + attribute_type = get_int_type() elif callable(getattr(module, name)): attribute_type = py_function_type module_attributes[name] = attribute_type - module_t = ObjectType( - IndividualVariable(), - module_attributes, - nominal_supertypes=[module_type], - ) + module_t = ObjectType(module_attributes, nominal_supertypes=[module_type],) return StackEffect( - TypeSequence([_seq_var]), TypeSequence([_seq_var, module_type]) + TypeSequence([_seq_var]), TypeSequence([_seq_var, module_t]) ) @@ -1358,7 +1314,6 @@ def _generate_module_type( _full_name = '.'.join(components) if len(components) > 1: module_t = ObjectType( - IndividualVariable(), { components[1]: _generate_module_type( components[1:], _full_name, source_dir @@ -1367,18 +1322,14 @@ def _generate_module_type( nominal_supertypes=[module_type], ) effect = StackEffect( - TypeSequence([_seq_var]), TypeSequence([_seq_var, module_type]) - ) - return ObjectType( - IndividualVariable(), {'__call__': effect,}, [_seq_var] + TypeSequence([_seq_var]), TypeSequence([_seq_var, module_t]) ) + return ObjectType({'__call__': effect,}, [_seq_var]) else: innermost_type = _generate_type_of_innermost_module( _full_name, source_dir ) - return ObjectType( - IndividualVariable(), {'__call__': innermost_type,}, [_seq_var] - ) + return ObjectType({'__call__': innermost_type,}, [_seq_var]) def _ensure_type( diff --git a/concat/typecheck/errors.py b/concat/typecheck/errors.py new file mode 100644 index 0000000..d597655 --- /dev/null +++ b/concat/typecheck/errors.py @@ -0,0 +1,79 @@ +import builtins +import pathlib +from typing import Optional, Union, TYPE_CHECKING + + +if TYPE_CHECKING: + import concat.astutils + import concat.parse + from concat.typecheck.types import Type, TypeSequence + + +class StaticAnalysisError(Exception): + def __init__(self, message: str) -> None: + self.message = message + self.location: Optional['concat.astutils.Location'] = None + self.path: Optional[pathlib.Path] = None + + def set_location_if_missing( + self, location: 'concat.astutils.Location' + ) -> None: + if not self.location: + self.location = location + + def set_path_if_missing(self, path: pathlib.Path) -> None: + if self.path is None: + self.path = path + + def __str__(self) -> str: + return '{} at {}'.format(self.message, self.location) + + +class TypeError(StaticAnalysisError, builtins.TypeError): + pass + + +class NameError(StaticAnalysisError, builtins.NameError): + def __init__( + self, + name: Union['concat.parse.NameWordNode', str], + location: Optional['concat.astutils.Location'] = None, + ) -> None: + if isinstance(name, concat.parse.NameWordNode): + location = name.location + name = name.value + super().__init__(f'name {name!r} not previously defined') + self._name = name + self.location = location + + def __str__(self) -> str: + location_info = '' + if self.location: + location_info = ' (error at {}:{})'.format(*self.location) + return self.message + location_info + + +class AttributeError(TypeError, builtins.AttributeError): + def __init__(self, type: 'Type', attribute: str) -> None: + super().__init__( + 'object of type {} does not have attribute {}'.format( + type, attribute + ) + ) + self._type = type + self._attribute = attribute + + +class StackMismatchError(TypeError): + def __init__( + self, actual: 'TypeSequence', expected: 'TypeSequence' + ) -> None: + super().__init__( + 'The stack here is {}, but sequence type {} was expected'.format( + actual, expected + ) + ) + + +class UnhandledNodeTypeError(builtins.NotImplementedError): + pass diff --git a/concat/typecheck/preamble.cati b/concat/typecheck/preamble.cati index 1c45a4d..887c7e9 100644 --- a/concat/typecheck/preamble.cati +++ b/concat/typecheck/preamble.cati @@ -1,3 +1,18 @@ +# FIXME: Exists only at compile time. Should probably be io.FileIO or something +# in typing +class file: + def seek(--) @cast (py_function[(int), int]): + () + + def read(--) @cast (py_function[*any, `any2]): + () + + def __enter__(--) @cast (py_function[*any, `any2]): + () + + def __exit__(--) @cast (py_function[*any, `any2]): + () + def to_list(*rest_var i:iterable[`a_var] -- *rest_var l:list[`a_var]): () diff --git a/concat/typecheck/preamble_types.py b/concat/typecheck/preamble_types.py index c8c26d0..520d54c 100644 --- a/concat/typecheck/preamble_types.py +++ b/concat/typecheck/preamble_types.py @@ -7,7 +7,6 @@ addable_type, context_manager_type, ellipsis_type, - file_type, geq_comparable_type, iterable_type, iterator_type, @@ -115,7 +114,6 @@ 'py_function': py_function_type, 'py_overloaded': py_overloaded_type, 'Optional': optional_type, - 'file': file_type, 'none': none_type, 'None': GenericType( [_stack_type_var], diff --git a/concat/typecheck/py_builtins.cati b/concat/typecheck/py_builtins.cati index e8f75f5..58c1e26 100644 --- a/concat/typecheck/py_builtins.cati +++ b/concat/typecheck/py_builtins.cati @@ -7,13 +7,13 @@ class bool: () class int: - def __add__(--) @cast (py_function[(object), int]): + def __add__(--) @cast (py_function[(int), int]): () def __invert__(--) @cast (py_function[(), int]): () - def __sub__(--) @cast (py_function[(object), str]): + def __sub__(--) @cast (py_function[(int), str]): () def __le__(--) @cast (py_function[(int), bool]): @@ -25,6 +25,8 @@ class int: def __ge__(--) @cast (py_function[(int), bool]): () +!@@concat.typecheck.builtin_int int + class float: () diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 42cd963..dd2487c 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -1,7 +1,14 @@ from concat.orderedset import InsertionOrderedSet import concat.typecheck +from concat.typecheck.errors import ( + AttributeError as ConcatAttributeError, + StackMismatchError, + StaticAnalysisError, + TypeError as ConcatTypeError, +) from typing import ( AbstractSet, + Any, Dict, Iterable, Iterator, @@ -19,28 +26,51 @@ ) from typing_extensions import Self import abc -import collections.abc if TYPE_CHECKING: from concat.typecheck import Environment, Substitutions +class SubtypeExplanation: + def __init__(self, data: Any) -> None: + self._data = data + + def __bool__(self) -> bool: + if isinstance(self._data, concat.typecheck.Substitutions): + return not self._data + return not isinstance(self._data, StaticAnalysisError) + + def __str__(self) -> str: + if isinstance(self._data, StaticAnalysisError): + e: Optional[BaseException] = self._data + string = '' + while e is not None: + string += '\n' + str(e) + e = e.__cause__ or e.__context__ + return string + return str(self._data) + + class Type(abc.ABC): + _next_type_id = 0 + def __init__(self) -> None: self._free_type_variables_cached: Optional[ InsertionOrderedSet[_Variable] ] = None self._internal_name: Optional[str] = None self._forward_references_resolved = False + self._type_id = Type._next_type_id + Type._next_type_id += 1 # QUESTION: Do I need this? - def is_subtype_of(self, supertype: 'Type') -> bool: + def is_subtype_of(self, supertype: 'Type') -> SubtypeExplanation: try: sub = self.constrain_and_bind_variables(supertype, set(), []) - except concat.typecheck.TypeError: - return False - return not sub + except ConcatTypeError as e: + return SubtypeExplanation(e) + return SubtypeExplanation(sub) # No <= implementation using subtyping, because variables overload that for # sort by identity. @@ -57,8 +87,9 @@ def __eq__(self, other: object) -> bool: def __hash__(self) -> int: pass + # TODO: Define in terms of .attributes def get_type_of_attribute(self, name: str) -> 'Type': - raise AttributeError(self, name) + raise ConcatAttributeError(self, name) def has_attribute(self, name: str) -> bool: try: @@ -95,7 +126,7 @@ def constrain_and_bind_variables( self, supertype: 'Type', rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple['IndividualType', 'IndividualType']], + subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': raise NotImplementedError @@ -105,7 +136,7 @@ def constrain_and_bind_variables( # Easy'? def constrain(self, supertype: 'Type') -> None: if not self.is_subtype_of(supertype): - raise concat.typecheck.TypeError( + raise ConcatTypeError( '{} is not a subtype of {}'.format(self, supertype) ) @@ -113,7 +144,7 @@ def instantiate(self) -> 'Type': return self @abc.abstractmethod - def resolve_forward_references(self) -> Self: + def resolve_forward_references(self) -> 'Type': self._forward_references_resolved = True return self @@ -129,6 +160,11 @@ def __str__(self) -> str: return self._internal_name return super().__str__() + def __getitem__(self, _: Any) -> Any: + raise ConcatTypeError( + f'{self} is neither a generic type nor a sequence type' + ) + class IndividualType(Type): def instantiate(self) -> 'IndividualType': @@ -143,6 +179,54 @@ def attributes(self) -> Mapping[str, Type]: return {} +class StuckTypeApplication(IndividualType): + def __init__(self, head: Type, args: 'TypeArguments') -> None: + super().__init__() + self._head = head + self._args = args + + def apply_substitution(self, sub: 'Substitutions') -> Type: + return sub(self._head)[[sub(t) for t in self._args]] + + def constrain_and_bind_variables( + self, supertype, rigid_variables, subtyping_assumptions + ) -> 'Substitutions': + from concat.typecheck import Substitutions + + if self is supertype or supertype is get_object_type(): + return Substitutions() + if isinstance(supertype, StuckTypeApplication): + # TODO: Variance + return self._head.constrain_and_bind_variables( + supertype._head, rigid_variables, subtyping_assumptions + ) + raise ConcatTypeError( + f'Cannot deduce that {self} is a subtype of {supertype} here' + ) + + def __str__(self) -> str: + if self._internal_name is not None: + return self._internal_name + return f'{self._head}{_iterable_to_str(self._args)}' + + def __repr__(self) -> str: + return f'StuckTypeApplication({self._head!r}, {self._args!r})' + + def __hash__(self) -> int: + return hash((self._head, tuple(self._args))) + + def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: + ftv = self._head.free_type_variables() + for arg in self._args: + ftv |= arg.free_type_variables() + return ftv + + def resolve_forward_references(self) -> Type: + head = self._head.resolve_forward_references() + args = [arg.resolve_forward_references() for arg in self._args] + return head[args] + + class _Variable(Type, abc.ABC): """Objects that represent type variables. @@ -171,25 +255,54 @@ def __gt__(self, other) -> bool: return id(self) > id(other) def __eq__(self, other) -> bool: - return id(self) == id(other) + return self is other + def resolve_forward_references(self) -> '_Variable': + return self -class IndividualVariable(_Variable, IndividualType): - def __init__(self) -> None: + # __hash__ by object identity is used since that's the only way for two + # type variables to be ==. + def __hash__(self) -> int: + return hash(id(self)) + + +class BoundVariable(_Variable): + def __init__(self, kind: 'Kind') -> None: super().__init__() + self._kind = kind + + @property + def kind(self) -> 'Kind': + return self._kind + + def constrain_and_bind_variables( + self, supertype, rigid_variables, subtyping_assumptions + ) -> 'Substitutions': + raise TypeError('Cannot constrain bound variables') + + def __getitem__(self, args: 'TypeArguments') -> Type: + assert isinstance(self.kind, GenericTypeKind) + assert list(self.kind.parameter_kinds) == [t.kind for t in args] + return StuckTypeApplication(self, args) + + @property + def attributes(self) -> Mapping[str, Type]: + raise TypeError('Cannot get attributes of bound variables') + +class IndividualVariable(_Variable, IndividualType): def constrain_and_bind_variables( self, supertype: Type, rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple[IndividualType, IndividualType]], + subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': from concat.typecheck import Substitutions if self is supertype or supertype is get_object_type(): return Substitutions() if supertype.kind != IndividualKind(): - raise concat.typecheck.TypeError( + raise ConcatTypeError( '{} must be an individual type: expected {}'.format( supertype, self ) @@ -208,22 +321,17 @@ def constrain_and_bind_variables( rigid_variables, subtyping_assumptions, ) - except concat.typecheck.TypeError: + except ConcatTypeError: return self.constrain_and_bind_variables( none_type, rigid_variables, subtyping_assumptions ) if self in rigid_variables: - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'{self} is considered fixed here and cannot become a subtype of {supertype}' ) mapping = {self: supertype} return Substitutions(mapping) - # __hash__ by object identity is used since that's the only way for two - # type variables to be ==. - def __hash__(self) -> int: - return hash(id(self)) - def __str__(self) -> str: return '`t_{}'.format(id(self)) @@ -237,15 +345,12 @@ def apply_substitution( @property def attributes(self) -> NoReturn: - raise concat.typecheck.TypeError( + raise ConcatTypeError( '{} is an individual type variable, so its attributes are unknown'.format( self ) ) - def resolve_forward_references(self) -> 'IndividualVariable': - return self - @property def kind(self) -> 'Kind': return IndividualKind() @@ -258,19 +363,16 @@ def __init__(self) -> None: def __str__(self) -> str: return '*t_{}'.format(id(self)) - def __hash__(self) -> int: - return hash(id(self)) - def constrain_and_bind_variables( self, supertype: Type, rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple['IndividualType', 'IndividualType']], + subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': from concat.typecheck import Substitutions if not isinstance(supertype, (SequenceVariable, TypeSequence)): - raise concat.typecheck.TypeError( + raise ConcatTypeError( '{} must be a sequence type, not {}'.format(self, supertype) ) if ( @@ -279,14 +381,14 @@ def constrain_and_bind_variables( ): return Substitutions([(supertype, self)]) if self in rigid_variables: - raise concat.typecheck.TypeError( + raise ConcatTypeError( '{} is fixed here and cannot become a subtype of another type'.format( self ) ) # occurs check if self is not supertype and self in supertype.free_type_variables(): - raise concat.typecheck.TypeError( + raise ConcatTypeError( '{} cannot be a subtype of {} because it appears in {}'.format( self, supertype, supertype ) @@ -294,19 +396,16 @@ def constrain_and_bind_variables( return Substitutions([(self, supertype)]) def get_type_of_attribute(self, name: str) -> NoReturn: - raise concat.typecheck.TypeError( + raise ConcatTypeError( 'the sequence type {} does not hold attributes'.format(self) ) @property def attributes(self) -> NoReturn: - raise concat.typecheck.TypeError( + raise ConcatTypeError( 'the sequence type {} does not hold attributes'.format(self) ) - def resolve_forward_references(self) -> 'SequenceVariable': - return self - @property def kind(self) -> 'Kind': return SequenceKind() @@ -323,7 +422,7 @@ def __init__( assert type_parameters self._type_parameters = type_parameters if body.kind != IndividualKind(): - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'Cannot be polymorphic over non-individual type {body}' ) self._body = body @@ -352,7 +451,7 @@ def __getitem__(self, type_arguments: 'TypeArguments') -> 'Type': expected_kinds = [var.kind for var in self._type_parameters] actual_kinds = [ty.kind for ty in type_arguments] if expected_kinds != actual_kinds: - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'A type argument to {self} has the wrong kind, type arguments: {type_arguments}, expected kinds: {expected_kinds}' ) sub = Substitutions(zip(self._type_parameters, type_arguments)) @@ -372,8 +471,8 @@ def kind(self) -> 'Kind': return GenericTypeKind(kinds) def resolve_forward_references(self) -> 'GenericType': - self._body = self._body.resolve_forward_references() - return self + body = self._body.resolve_forward_references() + return GenericType(self._type_parameters, body, self.is_variadic) def instantiate(self) -> Type: fresh_vars: Sequence[_Variable] = [ @@ -385,7 +484,7 @@ def constrain_and_bind_variables( self, supertype: 'Type', rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple['IndividualType', 'IndividualType']], + subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': from concat.typecheck import Substitutions @@ -394,7 +493,7 @@ def constrain_and_bind_variables( ): return Substitutions([]) if self.kind != supertype.kind: - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'{self} has kind {self.kind} but {supertype} has kind {supertype.kind}' ) if not isinstance(supertype, GenericType): @@ -440,7 +539,7 @@ def apply_substitution(self, sub: 'Substitutions') -> 'GenericType': @property def attributes(self) -> NoReturn: - raise concat.typecheck.TypeError( + raise ConcatTypeError( 'Generic types do not have attributes; maybe you forgot type arguments?' ) @@ -448,8 +547,8 @@ def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: return self._body.free_type_variables() - set(self._type_parameters) -class TypeSequence(Type, Iterable['StackItemType']): - def __init__(self, sequence: Sequence['StackItemType']) -> None: +class TypeSequence(Type, Iterable[Type]): + def __init__(self, sequence: Sequence[Type]) -> None: super().__init__() self._rest: Optional[SequenceVariable] if sequence and isinstance(sequence[0], SequenceVariable): @@ -459,12 +558,15 @@ def __init__(self, sequence: Sequence['StackItemType']) -> None: self._rest = None self._individual_types = sequence - def as_sequence(self) -> Sequence['StackItemType']: + def as_sequence(self) -> Sequence[Type]: if self._rest is not None: return [self._rest, *self._individual_types] return self._individual_types def apply_substitution(self, sub) -> 'TypeSequence': + if all(v not in self.free_type_variables() for v in sub): + return self + subbed_types: List[StackItemType] = [] for type in self: subbed_type: Union[StackItemType, TypeSequence] = sub(type) @@ -478,7 +580,7 @@ def constrain_and_bind_variables( self, supertype: Type, rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple['IndividualType', 'IndividualType']], + subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': """Check that self is a subtype of supertype. @@ -511,7 +613,7 @@ def constrain_and_bind_variables( # [] <: *a? `t0 `t... # error else: - raise concat.typecheck.StackMismatchError(self, supertype) + raise StackMismatchError(self, supertype) elif not self._individual_types: # *a <: [], *a is not rigid # --> *a = [] @@ -533,12 +635,12 @@ def constrain_and_bind_variables( ): return Substitutions([(self._rest, supertype)]) else: - raise concat.typecheck.StackMismatchError(self, supertype) + raise StackMismatchError(self, supertype) else: # *a? `t... `t_n <: [] # error if supertype._is_empty(): - raise concat.typecheck.StackMismatchError(self, supertype) + raise StackMismatchError(self, supertype) # *a? `t... `t_n <: *b, *b is not rigid, *b is not free in LHS # --> *b = LHS elif ( @@ -566,15 +668,14 @@ def constrain_and_bind_variables( subtyping_assumptions, )(sub) return sub - except concat.typecheck.StackMismatchError: - # TODO: Add info about occurs check and rigid variables. - raise concat.typecheck.StackMismatchError( - self, supertype - ) + except StackMismatchError: + # TODO: Add info about occurs check and rigid + # variables. + raise StackMismatchError(self, supertype) else: - raise concat.typecheck.StackMismatchError(self, supertype) + raise StackMismatchError(self, supertype) else: - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'{self} is a sequence type, not {supertype}' ) @@ -586,7 +687,7 @@ def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: @property def attributes(self) -> NoReturn: - raise concat.typecheck.TypeError( + raise ConcatTypeError( 'the sequence type {} does not hold attributes'.format(self) ) @@ -604,9 +705,7 @@ def __getitem__(self, key: int) -> 'StackItemType': def __getitem__(self, key: slice) -> 'TypeSequence': ... - def __getitem__( - self, key: Union[int, slice] - ) -> Union['StackItemType', 'TypeSequence']: + def __getitem__(self, key: Union[int, slice]) -> Type: if isinstance(key, int): return self.as_sequence()[key] return TypeSequence(self.as_sequence()[key]) @@ -617,17 +716,18 @@ def __str__(self) -> str: def __repr__(self) -> str: return 'TypeSequence([' + ', '.join(repr(t) for t in self) + '])' - def __iter__(self) -> Iterator['StackItemType']: + def __iter__(self) -> Iterator[Type]: return iter(self.as_sequence()) def __hash__(self) -> int: return hash(tuple(self.as_sequence())) def resolve_forward_references(self) -> 'TypeSequence': - self._individual_types = [ + individual_types = [ t.resolve_forward_references() for t in self._individual_types ] - return self + rest = [] if self._rest is None else [self._rest] + return TypeSequence(rest + individual_types) @property def kind(self) -> 'Kind': @@ -641,14 +741,10 @@ def __init__( ) -> None: for ty in input_types[1:]: if ty.kind != IndividualKind(): - raise concat.typecheck.TypeError( - f'{ty} must be an individual type' - ) + raise ConcatTypeError(f'{ty} must be an individual type') for ty in output_types[1:]: if ty.kind != IndividualKind(): - raise concat.typecheck.TypeError( - f'{ty} must be an individual type' - ) + raise ConcatTypeError(f'{ty} must be an individual type') super().__init__() self.input = input_types self.output = output_types @@ -670,7 +766,7 @@ def constrain_and_bind_variables( self, supertype: Type, rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple[IndividualType, IndividualType]], + subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': from concat.typecheck import Substitutions @@ -693,7 +789,7 @@ def constrain_and_bind_variables( subtyping_assumptions, ) if not isinstance(supertype, StackEffect): - raise concat.typecheck.TypeError( + raise ConcatTypeError( '{} is not a subtype of {}'.format(self, supertype) ) # Remember that the input should be contravariant! @@ -745,7 +841,7 @@ def __str__(self) -> str: def get_type_of_attribute(self, name: str) -> '_Function': if name == '__call__': return self - raise AttributeError(self, name) + raise ConcatAttributeError(self, name) @property def attributes(self) -> Mapping[str, 'StackEffect']: @@ -760,9 +856,9 @@ def bind(self) -> '_Function': return _Function(self.input[:-1], self.output) def resolve_forward_references(self) -> 'StackEffect': - self.input = self.input.resolve_forward_references() - self.output = self.output.resolve_forward_references() - return self + input = self.input.resolve_forward_references() + output = self.output.resolve_forward_references() + return StackEffect(input, output) class QuotationType(_Function): @@ -773,7 +869,7 @@ def constrain_and_bind_variables( self, supertype: Type, rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple[IndividualType, IndividualType]], + subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': if ( isinstance(supertype, ObjectType) @@ -832,24 +928,18 @@ class ObjectType(IndividualType): def __init__( self, - self_type: IndividualVariable, attributes: Mapping[str, Type], nominal_supertypes: Sequence[Type] = (), nominal: bool = False, _head: Optional['ObjectType'] = None, ) -> None: - assert isinstance(self_type, IndividualVariable) super().__init__() - # There should be no need to make the self_type variable unique because - # it is treated as a bound variable in apply_substitution. In other - # words, it is removed from any substitution received. - self._self_type = self_type self._attributes = attributes for t in nominal_supertypes: if t.kind != IndividualKind(): - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'{t} must be an individual type, but has kind {t.kind}' ) self._nominal_supertypes = nominal_supertypes @@ -866,14 +956,16 @@ def nominal(self) -> bool: return self._nominal def resolve_forward_references(self) -> 'ObjectType': - self._attributes = { + attributes = { attr: t.resolve_forward_references() for attr, t in self._attributes.items() } - self._nominal_supertypes = [ + nominal_supertypes = [ t.resolve_forward_references() for t in self._nominal_supertypes ] - return self + return ObjectType( + attributes, nominal_supertypes, self.nominal, self._head + ) @property def kind(self) -> 'Kind': @@ -884,12 +976,10 @@ def apply_substitution( ) -> 'ObjectType': from concat.typecheck import Substitutions - sub = Substitutions( - {a: i for a, i in sub.items() if a is not self._self_type} - ) # if no free type vars will be substituted, just return self if not any(free_var in sub for free_var in self.free_type_variables()): return self + attributes = cast( Dict[str, IndividualType], {attr: sub(t) for attr, t in self._attributes.items()}, @@ -898,7 +988,6 @@ def apply_substitution( sub(supertype) for supertype in self._nominal_supertypes ] subbed_type = type(self)( - self._self_type, attributes, nominal_supertypes=nominal_supertypes, nominal=self._nominal, @@ -914,7 +1003,7 @@ def constrain_and_bind_variables( self, supertype: Type, rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple[IndividualType, IndividualType]], + subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': from concat.typecheck import Substitutions @@ -933,14 +1022,14 @@ def constrain_and_bind_variables( # obj <: *s? `t... # error elif isinstance(supertype, (SequenceVariable, TypeSequence)): - raise concat.typecheck.TypeError( + raise ConcatTypeError( '{} is an individual type, but {} is a sequence type'.format( self, supertype ) ) if self.kind != supertype.kind: - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'{self} has kind {self.kind}, but {supertype} has kind {supertype.kind}' ) @@ -959,16 +1048,30 @@ def constrain_and_bind_variables( rigid_variables, subtyping_assumptions + [(self, supertype)], ) - except concat.typecheck.TypeError: + except ConcatTypeError: return self.constrain_and_bind_variables( supertype.type_arguments[0], rigid_variables, subtyping_assumptions + [(self, supertype)], ) if isinstance(supertype, _NoReturnType): - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'No other type, in this case, {self}, is a subtype of {supertype}' ) + if isinstance(supertype, Fix): + unrolled = supertype.unroll() + return self.constrain_and_bind_variables( + unrolled, + rigid_variables, + subtyping_assumptions + [(self, supertype)], + ) + if isinstance(supertype, ForwardTypeReference): + resolved = supertype.resolve_forward_references() + return self.constrain_and_bind_variables( + resolved, + rigid_variables, + subtyping_assumptions + [(self, supertype)], + ) if not isinstance(supertype, ObjectType): raise NotImplementedError(supertype) # every object type is a subtype of object_type @@ -979,7 +1082,7 @@ def constrain_and_bind_variables( if supertype in self._nominal_supertypes: return Substitutions() if self._head is not supertype._head: - raise concat.typecheck.TypeError( + raise ConcatTypeError( '{} is not a subtype of {}'.format(self, supertype) ) @@ -1000,62 +1103,42 @@ def constrain_and_bind_variables( def get_type_of_attribute(self, attribute: str) -> Type: if attribute not in self._attributes: - raise concat.typecheck.AttributeError(self, attribute) + raise ConcatAttributeError(self, attribute) - self_sub = concat.typecheck.Substitutions([(self._self_type, self)]) - - return self_sub(self._attributes[attribute]) + return self._attributes[attribute] def __repr__(self) -> str: head = None if self._head is self else self._head - return f'{type(self).__qualname__}(self_type={self._self_type!r}, attributes={self._attributes!r}, nominal_supertypes={self._nominal_supertypes!r}, nominal={self._nominal!r}, _head={head!r})' + return f'{type(self).__qualname__}(attributes={self._attributes!r}, nominal_supertypes={self._nominal_supertypes!r}, nominal={self._nominal!r}, _head={head!r})' def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: ftv = free_type_variables_of_mapping(self.attributes) # QUESTION: Include supertypes? - ftv -= {self.self_type} return ftv def __str__(self) -> str: if self._internal_name is not None: return self._internal_name - return '{}({}, {}, {}, {}, {})'.format( - type(self).__qualname__, - self._self_type, - _mapping_to_str(self._attributes), - _iterable_to_str(self._nominal_supertypes), - self._nominal, - None if self._head is self else self._head, - ) + return f'ObjectType({_mapping_to_str(self._attributes)}, {_iterable_to_str(self._nominal_supertypes)}, {self._nominal}, {None if self._head is self else self._head})' _hash_variable = None def __hash__(self) -> int: - from concat.typecheck import Substitutions - if ObjectType._hash_variable is None: ObjectType._hash_variable = IndividualVariable() - sub = Substitutions([(self._self_type, ObjectType._hash_variable)]) - type_to_hash = sub(self) + type_to_hash = self return hash( ( tuple(type_to_hash._attributes.items()), tuple(type_to_hash._nominal_supertypes), type_to_hash._nominal, - None if type_to_hash._head == self else type_to_hash._head, + None if type_to_hash._head is self else type_to_hash._head, ) ) @property def attributes(self) -> Mapping[str, Type]: - from concat.typecheck import Substitutions - - sub = Substitutions([(self._self_type, self)]) - return {name: sub(ty) for name, ty in self._attributes.items()} - - @property - def self_type(self) -> IndividualVariable: - return self._self_type + return self._attributes @property def head(self) -> 'ObjectType': @@ -1102,32 +1185,33 @@ def __init__( self._arity = len(type_parameters) self._type_parameters = type_parameters self._type_arguments = _type_arguments + self._overloads: Sequence[Tuple[Type, Type]] = [] if not ( self._arity == 0 and len(self._type_arguments) == 2 or self._arity == 2 and len(self._type_arguments) == 0 ): - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'Ill-formed Python function type with arguments {self._type_arguments}' ) if self._arity == 0: i, o = _type_arguments if i.kind != SequenceKind(): - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'{i} must be a sequence type, but has kind {i.kind}' ) if o.kind != IndividualKind(): - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'{o} must be an individual type, but has kind {o.kind}' ) for i, o in _overloads: if i.kind != SequenceKind(): - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'{i} must be a sequence type, but has kind {i.kind}' ) if o.kind != IndividualKind(): - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'{o} must be an individual type, but has kind {o.kind}' ) self._overloads = _overloads @@ -1148,9 +1232,8 @@ def kind(self) -> 'Kind': return GenericTypeKind([SequenceKind(), IndividualKind()]) def resolve_forward_references(self) -> 'PythonFunctionType': - if self._forward_references_resolved: + if self._arity == 2: return self - super().resolve_forward_references() overloads: List[Tuple[Type, Type]] = [] for args, ret in overloads: overloads.append( @@ -1159,11 +1242,14 @@ def resolve_forward_references(self) -> 'PythonFunctionType': ret.resolve_forward_references(), ) ) - self._overloads = overloads - self._type_arguments = list( + type_arguments = list( t.resolve_forward_references() for t in self._type_arguments ) - return self + return PythonFunctionType( + _overloads=overloads, + type_parameters=[], + _type_arguments=type_arguments, + ) def __eq__(self, other: object) -> bool: if not isinstance(other, PythonFunctionType): @@ -1182,7 +1268,7 @@ def __hash__(self) -> int: def _compute_hash(self) -> int: if isinstance(self.kind, GenericTypeKind): return 1 - return hash((tuple(self.input), self.output)) + return hash((self.input, self.output)) def __repr__(self) -> str: # QUESTION: Is it worth using type(self)? @@ -1191,9 +1277,7 @@ def __repr__(self) -> str: def __str__(self) -> str: if not self._type_arguments: return 'py_function_type' - return 'py_function_type[{}, {}]'.format( - _iterable_to_str(self.input), self.output - ) + return f'py_function_type[{self.input}, {self.output}]' def get_type_of_attribute(self, attribute: str) -> Type: if attribute == '__call__': @@ -1209,19 +1293,19 @@ def __getitem__( self, arguments: Tuple[Type, Type] ) -> 'PythonFunctionType': if self._arity != 2: - raise concat.typecheck.TypeError(f'{self} is not a generic type') + raise ConcatTypeError(f'{self} is not a generic type') if len(arguments) != 2: - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'{self} takes two arguments, got {len(arguments)}' ) input = arguments[0] output = arguments[1] if input.kind != SequenceKind(): - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'First argument to {self} must be a sequence type of function arguments' ) if output.kind != IndividualKind(): - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'Second argument to {self} must be an individual type for the return type' ) return PythonFunctionType( @@ -1232,7 +1316,7 @@ def apply_substitution( self, sub: 'concat.typecheck.Substitutions' ) -> 'PythonFunctionType': if self._arity == 0: - inp = sub(TypeSequence(self.input)) + inp = sub(self.input) out = sub(self.output) overloads: Sequence[Tuple[Type, Type]] = [ (sub(i), sub(o)) for i, o in self._overloads @@ -1258,15 +1342,15 @@ def select_overload( for overload in [(self.input, self.output), *self._overloads]: try: sub = TypeSequence(input_types).constrain_and_bind_variables( - TypeSequence(overload[0]), set(), [] + overload[0], set(), [] ) except TypeError: continue return ( - sub(py_function_type[TypeSequence(overload[0]), overload[1]]), + sub(py_function_type[overload]), sub, ) - raise concat.typecheck.TypeError( + raise ConcatTypeError( 'no overload of {} matches types {}'.format(self, input_types) ) @@ -1290,7 +1374,7 @@ def constrain_and_bind_variables( self, supertype: Type, rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple[IndividualType, IndividualType]], + subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': from concat.typecheck import Substitutions @@ -1299,7 +1383,7 @@ def constrain_and_bind_variables( ): return Substitutions() if self.kind != supertype.kind: - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'{self} has kind {self.kind} but {supertype} has kind {supertype.kind}' ) if self.kind == IndividualKind(): @@ -1343,8 +1427,8 @@ def constrain_and_bind_variables( ]: try: subtyping_assumptions_copy = subtyping_assumptions[:] - self_input_types = TypeSequence(overload[0]) - supertype_input_types = TypeSequence(supertype.input) + self_input_types = overload[0] + supertype_input_types = supertype.input sub = supertype_input_types.constrain_and_bind_variables( self_input_types, rigid_variables, @@ -1356,12 +1440,12 @@ def constrain_and_bind_variables( subtyping_assumptions_copy, )(sub) return sub - except concat.typecheck.TypeError: + except ConcatTypeError: continue finally: subtyping_assumptions[:] = subtyping_assumptions_copy - raise concat.typecheck.TypeError( + raise ConcatTypeError( 'no overload of {} is a subtype of {}'.format(self, supertype) ) @@ -1371,20 +1455,18 @@ def __init__(self) -> None: super().__init__() def __getitem__(self, args: Sequence[Type]) -> 'PythonFunctionType': - import concat.typecheck - if len(args) == 0: - raise concat.typecheck.TypeError( + raise ConcatTypeError( 'py_overloaded must be applied to at least one argument' ) fun_type = args[0] if not isinstance(fun_type, PythonFunctionType): - raise concat.typecheck.TypeError( + raise ConcatTypeError( 'Arguments to py_overloaded must be Python function types' ) for arg in args[1:]: if not isinstance(arg, PythonFunctionType): - raise concat.typecheck.TypeError( + raise ConcatTypeError( 'Arguments to py_overloaded must be Python function types' ) fun_type = fun_type.with_overload(arg.input, arg.output) @@ -1392,9 +1474,7 @@ def __getitem__(self, args: Sequence[Type]) -> 'PythonFunctionType': @property def attributes(self) -> Mapping[str, 'Type']: - raise concat.typecheck.TypeError( - 'py_overloaded does not have attributes' - ) + raise ConcatTypeError('py_overloaded does not have attributes') def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: return InsertionOrderedSet([]) @@ -1413,9 +1493,9 @@ def constrain_and_bind_variables( self, supertype: 'Type', rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple['IndividualType', 'IndividualType']], + subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': - raise concat.typecheck.TypeError('py_overloaded is a generic type') + raise ConcatTypeError('py_overloaded is a generic type') def resolve_forward_references(self) -> '_PythonOverloadedType': return self @@ -1461,11 +1541,15 @@ def __hash__(self) -> int: class _OptionalType(IndividualType): - def __init__(self, type_argument: IndividualType) -> None: + def __init__(self, type_argument: Type) -> None: super().__init__() + if type_argument.kind != IndividualKind(): + raise ConcatTypeError( + f'{type_argument} must be an individual type, but has kind {type_argument.kind}' + ) while isinstance(type_argument, _OptionalType): type_argument = type_argument._type_argument - self._type_argument: IndividualType = type_argument + self._type_argument: Type = type_argument def __repr__(self) -> str: return f'{type(self).__qualname__}({self._type_argument!r})' @@ -1477,9 +1561,9 @@ def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: return self._type_argument.free_type_variables() def __eq__(self, other: object) -> bool: - if not isinstance(other, _OptionalType): - return False - return self._type_argument == other._type_argument + if isinstance(other, _OptionalType): + return self._type_argument == other._type_argument + return super().__eq__(other) def __hash__(self) -> int: return hash(self._type_argument) @@ -1503,7 +1587,7 @@ def constrain_and_bind_variables( subtyping_assumptions, ) if self.kind != supertype.kind: - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'{self} is an individual type, but {supertype} has kind {supertype.kind}' ) # FIXME: optional[none] should simplify to none @@ -1519,8 +1603,8 @@ def constrain_and_bind_variables( return sub def resolve_forward_references(self) -> '_OptionalType': - self._type_argument = self._type_argument.resolve_forward_references() - return self + type_argument = self._type_argument.resolve_forward_references() + return _OptionalType(type_argument) def apply_substitution( self, sub: 'concat.typecheck.Substitutions' @@ -1528,7 +1612,7 @@ def apply_substitution( return _OptionalType(sub(self._type_argument)) @property - def type_arguments(self) -> Sequence[IndividualType]: + def type_arguments(self) -> Sequence[Type]: return [self._type_argument] @@ -1573,6 +1657,98 @@ def __hash__(self) -> int: return hash(tuple(self.parameter_kinds)) +class Fix(Type): + def __init__(self, var: _Variable, body: Type) -> None: + from concat.typecheck import Substitutions + + super().__init__() + assert var.kind == body.kind + self._var = var + self._body = body + self._unrolled_ty: Optional[Type] = None + self._cache: Dict[int, Type] = {} + + def __repr__(self) -> str: + return f'Fix({self._var!r}, {self._body!r})' + + def _apply(self, t: Type) -> Type: + from concat.typecheck import Substitutions + + if t._type_id not in self._cache: + sub = Substitutions([(self._var, t)]) + self._cache[t._type_id] = sub(self._body) + assert ( + self._var not in self._cache[t._type_id].free_type_variables() + ) + return self._cache[t._type_id] + + def unroll(self) -> Type: + if self._unrolled_ty is None: + self._unrolled_ty = self._apply(self) + return self._unrolled_ty + + def __hash__(self) -> int: + return hash( + (self._var, self._body) + ) # FIXME: Probably broken, alpha equivalence + + def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: + return self._body.free_type_variables() - {self._var} + + def apply_substitution(self, sub: 'Substitutions') -> Type: + from concat.typecheck import Substitutions + + if all(v not in self.free_type_variables() for v in sub): + return self + sub = Substitutions( + {v: t for v, t in sub.items() if v is not self._var} + ) + + return Fix(self._var, sub(self._body)) + + @property + def attributes(self) -> Mapping[str, Type]: + return self.unroll().attributes + + def get_type_of_attribute(self, name: str) -> Type: + return self.attributes[name] + + def constrain_and_bind_variables( + self, supertype, rigid_variables, subtyping_assumptions + ) -> 'Substitutions': + from concat.typecheck import Substitutions + + if supertype is get_object_type() or _contains_assumption( + subtyping_assumptions, self, supertype + ): + return Substitutions() + + if isinstance(supertype, Fix): + unrolled = supertype.unroll() + return self.unroll().constrain_and_bind_variables( + unrolled, + rigid_variables, + subtyping_assumptions + [(self, supertype)], + ) + + return self.unroll().constrain_and_bind_variables( + supertype, + rigid_variables, + subtyping_assumptions + [(self, supertype)], + ) + + @property + def kind(self) -> Kind: + return self._var.kind + + def resolve_forward_references(self) -> Type: + body = self._body.resolve_forward_references() + return Fix(self._var, body) + + def __getitem__(self, args: Any) -> Any: + return self.unroll()[args] + + class ForwardTypeReference(Type): def __init__( self, @@ -1607,29 +1783,18 @@ def __hash__(self) -> int: return hash(self._resolved_type) return hash(self._as_hashable_tuple()) - def __eq__(self, other: object) -> bool: - if super().__eq__(other): - return True - if not isinstance(other, Type): - return NotImplemented - if self._resolved_type is not None: - return self._resolved_type == other - if not isinstance(other, ForwardTypeReference): - return False - return self._as_hashable_tuple() == other._as_hashable_tuple() - - def __getitem__(self, args: TypeArguments) -> IndividualType: + def __getitem__(self, args: TypeArguments) -> Type: if self._resolved_type is not None: return self._resolved_type[args] if isinstance(self.kind, GenericTypeKind): if len(self.kind.parameter_kinds) != len(args): - raise concat.typecheck.TypeError( + raise ConcatTypeError( 'Wrong number of arguments to generic type' ) for kind, arg in zip(self.kind.parameter_kinds, args): if kind != arg.kind: - raise concat.typecheck.TypeError( + raise ConcatTypeError( f'Type argument has kind {arg.kind}, expected kind {kind}' ) return ForwardTypeReference( @@ -1638,7 +1803,7 @@ def __getitem__(self, args: TypeArguments) -> IndividualType: self._resolution_env, _type_arguments=args, ) - raise concat.typecheck.TypeError(f'{self} is not a generic type') + raise ConcatTypeError(f'{self} is not a generic type') def resolve_forward_references(self) -> Type: if self._resolved_type is None: @@ -1647,7 +1812,7 @@ def resolve_forward_references(self) -> Type: def apply_substitution( self, sub: 'concat.typecheck.Substitutions' - ) -> 'ForwardTypeReference': + ) -> Type: if self._resolved_type is not None: return sub(self._resolved_type) @@ -1658,7 +1823,7 @@ def attributes(self) -> Mapping[str, Type]: if self._resolved_type is not None: return self._resolved_type.attributes - raise concat.typecheck.TypeError( + raise ConcatTypeError( 'Cannot access attributes of type before they are defined' ) @@ -1666,18 +1831,17 @@ def constrain_and_bind_variables( self, supertype: Type, rigid_variables: AbstractSet['_Variable'], - subtyping_assumptions: List[Tuple[IndividualType, IndividualType]], + subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': - if self is supertype: + if self is supertype or _contains_assumption( + subtyping_assumptions, self, supertype + ): return concat.typecheck.Substitutions() - if self._resolved_type is not None: - return self._resolved_type.constrain_and_bind_variables( - supertype, rigid_variables, subtyping_assumptions - ) - - raise concat.typecheck.TypeError( - 'Supertypes of type are not known before its definition' + return self.resolve_forward_references().constrain_and_bind_variables( + supertype, + rigid_variables, + subtyping_assumptions + [(self, supertype)], ) def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: @@ -1705,9 +1869,9 @@ def _mapping_to_str(mapping: Mapping) -> str: # expose _Function as StackEffect StackEffect = _Function -_x = IndividualVariable() +_x = BoundVariable(kind=IndividualKind()) -float_type = ObjectType(_x, {}, nominal=True) +float_type = ObjectType({}, nominal=True) no_return_type = _NoReturnType() @@ -1750,6 +1914,19 @@ def set_str_type(ty: Type) -> None: _str_type = ty +_int_type: Optional[Type] = None + + +def get_int_type() -> Type: + assert _int_type is not None + return _int_type + + +def set_int_type(ty: Type) -> None: + global _int_type + _int_type = ty + + _arg_type_var = SequenceVariable() _return_type_var = IndividualVariable() py_function_type = PythonFunctionType( @@ -1761,7 +1938,6 @@ def set_str_type(ty: Type) -> None: invertible_type = GenericType( [_invert_result_var], ObjectType( - _x, {'__invert__': py_function_type[TypeSequence([]), _invert_result_var]}, ), ) @@ -1772,7 +1948,6 @@ def set_str_type(ty: Type) -> None: subtractable_type = GenericType( [_sub_operand_type, _sub_result_type], ObjectType( - _x, { '__sub__': py_function_type[ TypeSequence([_sub_operand_type]), _sub_result_type @@ -1788,7 +1963,6 @@ def set_str_type(ty: Type) -> None: addable_type = GenericType( [_add_other_operand_type, _add_result_type], ObjectType( - _x, { '__add__': py_function_type[ # QUESTION: Should methods include self? @@ -1800,7 +1974,7 @@ def set_str_type(ty: Type) -> None: ) addable_type.set_internal_name('addable_type') -bool_type = ObjectType(_x, {}, nominal=True) +bool_type = ObjectType({}, nominal=True) bool_type.set_internal_name('bool_type') # QUESTION: Allow comparison methods to return any object? @@ -1809,7 +1983,6 @@ def set_str_type(ty: Type) -> None: geq_comparable_type = GenericType( [_other_type], ObjectType( - _x, {'__ge__': py_function_type[TypeSequence([_other_type]), bool_type]}, ), ) @@ -1818,7 +1991,6 @@ def set_str_type(ty: Type) -> None: leq_comparable_type = GenericType( [_other_type], ObjectType( - _x, {'__le__': py_function_type[TypeSequence([_other_type]), bool_type]}, ), ) @@ -1827,43 +1999,28 @@ def set_str_type(ty: Type) -> None: lt_comparable_type = GenericType( [_other_type], ObjectType( - _x, {'__lt__': py_function_type[TypeSequence([_other_type]), bool_type]}, ), ) lt_comparable_type.set_internal_name('lt_comparable_type') -_int_add_type = py_function_type[TypeSequence([_x]), _x] - -int_type = ObjectType( - _x, - { - '__add__': _int_add_type, - '__invert__': py_function_type[TypeSequence([]), _x], - '__sub__': _int_add_type, - '__le__': py_function_type[TypeSequence([_x]), bool_type], - '__lt__': py_function_type[TypeSequence([_x]), bool_type], - '__ge__': py_function_type[TypeSequence([_x]), bool_type], - }, - nominal=True, -) -int_type.set_internal_name('int_type') - -none_type = ObjectType(_x, {}) +none_type = ObjectType({}, nominal=True) none_type.set_internal_name('none_type') _result_type = IndividualVariable() iterator_type = GenericType( [_result_type], - ObjectType( + Fix( _x, - { - '__iter__': py_function_type[TypeSequence([]), _x], - '__next__': py_function_type[ - TypeSequence([none_type,]), _result_type - ], - }, + ObjectType( + { + '__iter__': py_function_type[TypeSequence([]), _x], + '__next__': py_function_type[ + TypeSequence([none_type,]), _result_type + ], + }, + ), ), ) iterator_type.set_internal_name('iterator_type') @@ -1871,7 +2028,6 @@ def set_str_type(ty: Type) -> None: iterable_type = GenericType( [_result_type], ObjectType( - _x, { '__iter__': py_function_type[ TypeSequence([]), iterator_type[_result_type,] @@ -1882,7 +2038,6 @@ def set_str_type(ty: Type) -> None: iterable_type.set_internal_name('iterable_type') context_manager_type = ObjectType( - _x, { # TODO: Add argument and return types. I think I'll need a special # py_function representation for that. @@ -1901,7 +2056,6 @@ def set_str_type(ty: Type) -> None: _key_type_var = IndividualVariable() _value_type_var = IndividualVariable() dict_type = ObjectType( - _x, { '__iter__': py_function_type[ TypeSequence([]), iterator_type[_key_type_var,] @@ -1912,37 +2066,23 @@ def set_str_type(ty: Type) -> None: ) dict_type.set_internal_name('dict_type') -file_type = ObjectType( - self_type=_x, - attributes={ - 'seek': py_function_type[TypeSequence([int_type]), int_type], - 'read': py_function_type, - '__enter__': py_function_type, - '__exit__': py_function_type, - }, - # context_manager_type is a structural supertype - nominal=True, -) -file_type.set_internal_name('file_type') - _start_type_var, _stop_type_var, _step_type_var = ( IndividualVariable(), IndividualVariable(), IndividualVariable(), ) slice_type = ObjectType( - _x, {}, [_start_type_var, _stop_type_var, _step_type_var], nominal=True + {}, [_start_type_var, _stop_type_var, _step_type_var], nominal=True ) slice_type.set_internal_name('slice_type') -ellipsis_type = ObjectType(_x, {}) -not_implemented_type = ObjectType(_x, {}) +ellipsis_type = ObjectType({}, nominal=True) +not_implemented_type = ObjectType({}, nominal=True) _element_types_var = SequenceVariable() tuple_type = GenericType( [_element_types_var], ObjectType( - _x, {'__getitem__': py_function_type}, nominal=True, # iterable_type is a structural supertype @@ -1951,13 +2091,12 @@ def set_str_type(ty: Type) -> None: ) tuple_type.set_internal_name('tuple_type') -base_exception_type = ObjectType(_x, {}) -module_type = ObjectType(_x, {}) +base_exception_type = ObjectType({}, nominal=True) +module_type = ObjectType({}, nominal=True) _index_type_var = IndividualVariable() _result_type_var = IndividualVariable() subscriptable_type = ObjectType( - IndividualVariable(), { '__getitem__': py_function_type[ TypeSequence([_index_type_var]), _result_type_var @@ -1968,6 +2107,6 @@ def set_str_type(ty: Type) -> None: _answer_type_var = IndividualVariable() continuation_monad_type = ObjectType( - _x, {}, [_result_type_var, _answer_type_var], nominal=True + {}, [_result_type_var, _answer_type_var], nominal=True ) continuation_monad_type.set_internal_name('continuation_monad_type') diff --git a/mypy.ini b/mypy.ini index 5a93ed2..4af7784 100644 --- a/mypy.ini +++ b/mypy.ini @@ -2,3 +2,4 @@ check_untyped_defs = True disallow_any_unimported = True mypy_path = stubs/ +python_version = 3.7 diff --git a/tox.ini b/tox.ini index d9a42c5..d12cce6 100644 --- a/tox.ini +++ b/tox.ini @@ -24,3 +24,4 @@ setenv = [flake8] ignore = E741, F402 +max-complexity = 15 From ef29b5749f46b73f205ec71c9b9ffe746ca179c1 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Mon, 3 Jun 2024 21:45:59 -0700 Subject: [PATCH 22/61] Remove type hashing, except for variables --- concat/typecheck/types.py | 107 ++++++-------------------------------- 1 file changed, 16 insertions(+), 91 deletions(-) diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index dd2487c..d412b33 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -83,9 +83,10 @@ def __eq__(self, other: object) -> bool: # QUESTION: Define == separately from is_subtype_of? return self.is_subtype_of(other) and other.is_subtype_of(self) - @abc.abstractmethod - def __hash__(self) -> int: - pass + # NOTE: Avoid hashing types. I might I'm having correctness issues related + # to hashing that I'd rather avoid entirely. Maybe one day I'll introduce + # hash consing, but that would only reflect syntactic eequality, and I've + # been using hashing for type equality. # TODO: Define in terms of .attributes def get_type_of_attribute(self, name: str) -> 'Type': @@ -212,9 +213,6 @@ def __str__(self) -> str: def __repr__(self) -> str: return f'StuckTypeApplication({self._head!r}, {self._args!r})' - def __hash__(self) -> int: - return hash((self._head, tuple(self._args))) - def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: ftv = self._head.free_type_variables() for arg in self._args: @@ -260,9 +258,14 @@ def __eq__(self, other) -> bool: def resolve_forward_references(self) -> '_Variable': return self - # __hash__ by object identity is used since that's the only way for two - # type variables to be ==. + # NOTE: This hash impl is kept because sets of variables are fine and + # variables are simple. def __hash__(self) -> int: + """Hash a variable by its identity. + + __hash__ by object identity is used since that's the only way for two + type variables to be ==.""" + return hash(id(self)) @@ -426,7 +429,7 @@ def __init__( f'Cannot be polymorphic over non-individual type {body}' ) self._body = body - self._instantiations: Dict[Tuple[Type, ...], Type] = {} + self._instantiations: Dict[Tuple[int, ...], Type] = {} self.is_variadic = is_variadic def __str__(self) -> str: @@ -445,9 +448,9 @@ def __repr__(self) -> str: def __getitem__(self, type_arguments: 'TypeArguments') -> 'Type': from concat.typecheck import Substitutions - type_arguments = tuple(type_arguments) - if type_arguments in self._instantiations: - return self._instantiations[type_arguments] + type_argument_ids = tuple(t._type_id for t in type_arguments) + if type_argument_ids in self._instantiations: + return self._instantiations[type_argument_ids] expected_kinds = [var.kind for var in self._type_parameters] actual_kinds = [ty.kind for ty in type_arguments] if expected_kinds != actual_kinds: @@ -456,7 +459,7 @@ def __getitem__(self, type_arguments: 'TypeArguments') -> 'Type': ) sub = Substitutions(zip(self._type_parameters, type_arguments)) instance = sub(self._body) - self._instantiations[type_arguments] = instance + self._instantiations[type_argument_ids] = instance if self._internal_name is not None: instance_internal_name = self._internal_name instance_internal_name += ( @@ -510,20 +513,6 @@ def constrain_and_bind_variables( supertype_instance, rigid_variables, subtyping_assumptions ) - def __hash__(self) -> int: - # QUESTION: Is it better for perf to compute de Bruijn indices instead - # of using substitution? - vars_for_hash: List[_Variable] = [] - for p in self._type_parameters: - if p.kind == IndividualKind(): - vars_for_hash.append(self._individual_var_for_hash) - else: - vars_for_hash.append(self._sequence_var_for_hash) - return hash(self[vars_for_hash]) - - _individual_var_for_hash = IndividualVariable() - _sequence_var_for_hash = SequenceVariable() - def apply_substitution(self, sub: 'Substitutions') -> 'GenericType': from concat.typecheck import Substitutions @@ -719,9 +708,6 @@ def __repr__(self) -> str: def __iter__(self) -> Iterator[Type]: return iter(self.as_sequence()) - def __hash__(self) -> int: - return hash(tuple(self.as_sequence())) - def resolve_forward_references(self) -> 'TypeSequence': individual_types = [ t.resolve_forward_references() for t in self._individual_types @@ -758,10 +744,6 @@ def generalized_wrt(self, gamma: 'Environment') -> Type: ) return GenericType(parameters, self) - def __hash__(self) -> int: - # FIXME: Alpha equivalence - return hash((self.input, self.output)) - def constrain_and_bind_variables( self, supertype: Type, @@ -1121,21 +1103,6 @@ def __str__(self) -> str: return self._internal_name return f'ObjectType({_mapping_to_str(self._attributes)}, {_iterable_to_str(self._nominal_supertypes)}, {self._nominal}, {None if self._head is self else self._head})' - _hash_variable = None - - def __hash__(self) -> int: - if ObjectType._hash_variable is None: - ObjectType._hash_variable = IndividualVariable() - type_to_hash = self - return hash( - ( - tuple(type_to_hash._attributes.items()), - tuple(type_to_hash._nominal_supertypes), - type_to_hash._nominal, - None if type_to_hash._head is self else type_to_hash._head, - ) - ) - @property def attributes(self) -> Mapping[str, Type]: return self._attributes @@ -1260,16 +1227,6 @@ def __eq__(self, other: object) -> bool: return True return self.input == other.input and self.output == other.output - def __hash__(self) -> int: - if self._hash is None: - self._hash = self._compute_hash() - return self._hash - - def _compute_hash(self) -> int: - if isinstance(self.kind, GenericTypeKind): - return 1 - return hash((self.input, self.output)) - def __repr__(self) -> str: # QUESTION: Is it worth using type(self)? return f'{type(self).__qualname__}(_overloads={self._overloads!r}, type_parameters={self._type_parameters!r}, _type_arguments={self._type_arguments})' @@ -1504,9 +1461,6 @@ def resolve_forward_references(self) -> '_PythonOverloadedType': def kind(self) -> 'Kind': return GenericTypeKind([SequenceKind()]) - def __hash__(self) -> int: - return hash(type(self).__qualname__) - def __eq__(self, other: object) -> bool: return isinstance(other, type(self)) @@ -1536,9 +1490,6 @@ def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: def resolve_forward_references(self) -> Self: return self - def __hash__(self) -> int: - return 0 - class _OptionalType(IndividualType): def __init__(self, type_argument: Type) -> None: @@ -1565,9 +1516,6 @@ def __eq__(self, other: object) -> bool: return self._type_argument == other._type_argument return super().__eq__(other) - def __hash__(self) -> int: - return hash(self._type_argument) - def constrain_and_bind_variables( self, supertype, rigid_variables, subtyping_assumptions ) -> 'Substitutions': @@ -1621,26 +1569,16 @@ class Kind(abc.ABC): def __eq__(self, other: object) -> bool: pass - @abc.abstractmethod - def __hash__(self) -> int: - pass - class IndividualKind(Kind): def __eq__(self, other: object) -> bool: return isinstance(other, IndividualKind) - def __hash__(self) -> int: - return hash(type(self).__qualname__) - class SequenceKind(Kind): def __eq__(self, other: object) -> bool: return isinstance(other, SequenceKind) - def __hash__(self) -> int: - return hash(type(self).__qualname__) - class GenericTypeKind(Kind): def __init__(self, parameter_kinds: Sequence[Kind]) -> None: @@ -1653,9 +1591,6 @@ def __eq__(self, other: object) -> bool: and self.parameter_kinds == other.parameter_kinds ) - def __hash__(self) -> int: - return hash(tuple(self.parameter_kinds)) - class Fix(Type): def __init__(self, var: _Variable, body: Type) -> None: @@ -1687,11 +1622,6 @@ def unroll(self) -> Type: self._unrolled_ty = self._apply(self) return self._unrolled_ty - def __hash__(self) -> int: - return hash( - (self._var, self._body) - ) # FIXME: Probably broken, alpha equivalence - def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: return self._body.free_type_variables() - {self._var} @@ -1778,11 +1708,6 @@ def _as_hashable_tuple(self) -> tuple: tuple(self._type_arguments), ) - def __hash__(self) -> int: - if self._resolved_type is not None: - return hash(self._resolved_type) - return hash(self._as_hashable_tuple()) - def __getitem__(self, args: TypeArguments) -> Type: if self._resolved_type is not None: return self._resolved_type[args] From 2b2d811caab7a6760e8e43064d7f805eda3d2610 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Mon, 3 Jun 2024 23:35:51 -0700 Subject: [PATCH 23/61] Prevent sequence variables automatically introduced by type sequence AST from entering Python function type args --- concat/tests/test_typecheck.py | 9 +- concat/tests/typecheck/test_types.py | 13 ++ concat/typecheck/__init__.py | 15 ++- concat/typecheck/types.py | 172 +++++++++++++++++++++------ language-concat/lib/main.ts | 4 + language-concat/package.json | 5 + 6 files changed, 175 insertions(+), 43 deletions(-) diff --git a/concat/tests/test_typecheck.py b/concat/tests/test_typecheck.py index 5860885..4f7e253 100644 --- a/concat/tests/test_typecheck.py +++ b/concat/tests/test_typecheck.py @@ -4,7 +4,10 @@ import concat.parse from concat.typecheck import Environment from concat.typecheck.types import ( + BoundVariable, ClassType, + Fix, + IndividualKind, IndividualType, IndividualVariable, ObjectType, @@ -375,12 +378,12 @@ def test_object_subtype_of_py_function(self, type1, type2) -> None: @given(from_type(StackEffect)) def test_class_subtype_of_stack_effect(self, effect) -> None: - x = IndividualVariable() + x = BoundVariable(kind=IndividualKind()) # NOTE: self-last convention is modelled after Factor. unbound_effect = StackEffect( TypeSequence([*effect.input, x]), effect.output ) - cls = ClassType(x, {'__init__': unbound_effect}) + cls = Fix(x, ClassType({'__init__': unbound_effect})) self.assertTrue(cls.is_subtype_of(effect)) @given(from_type(IndividualType), from_type(IndividualType)) @@ -394,7 +397,7 @@ def test_class_subtype_of_py_function(self, type1, type2) -> None: x = IndividualVariable() py_function = py_function_type[TypeSequence([type1]), type2] unbound_py_function = py_function_type[TypeSequence([x, type1]), type2] - cls = ClassType({'__init__': unbound_py_function}) + cls = Fix(x, ClassType({'__init__': unbound_py_function})) self.assertTrue(cls.is_subtype_of(py_function)) @given(from_type(IndividualType)) diff --git a/concat/tests/typecheck/test_types.py b/concat/tests/typecheck/test_types.py index 2ef109c..59affa3 100644 --- a/concat/tests/typecheck/test_types.py +++ b/concat/tests/typecheck/test_types.py @@ -170,3 +170,16 @@ def test_resolve_subtype(self) -> None: def test_resolve_equal(self) -> None: self.assertEqual(self.ty.resolve_forward_references(), self.ty) self.assertEqual(self.ty, self.ty.resolve_forward_references()) + + +class TestTypeSequence(unittest.TestCase): + def test_constrain_empty(self) -> None: + self.assertEqual( + Substitutions(), + TypeSequence([]).constrain_and_bind_variables( + TypeSequence([]), set(), [] + ), + ) + + def test_empty_equal(self) -> None: + self.assertEqual(TypeSequence([]), TypeSequence([])) diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index b86f06b..ed20d2b 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -12,6 +12,7 @@ UnhandledNodeTypeError, ) from typing import ( + Any, Callable, Dict, Generator, @@ -77,6 +78,13 @@ def __init__( f'{variable} is being substituted by {ty}, which has the wrong kind ({variable.kind} vs {ty.kind})' ) self._cache: Dict[int, Type] = {} + # innermost first + self.subtyping_provenance: List[Any] = [] + + def add_subtyping_provenance( + self, subtyping_query: Tuple['Type', 'Type'] + ) -> None: + self.subtyping_provenance.append(subtyping_query) def __getitem__(self, var: '_Variable') -> 'Type': return self._sub[var] @@ -122,12 +130,16 @@ def __str__(self) -> str: ) def apply_substitution(self, sub: 'Substitutions') -> 'Substitutions': - return Substitutions( + new_sub = Substitutions( { **sub, **{a: sub(i) for a, i in self.items() if a not in sub._dom()}, } ) + new_sub.subtyping_provenance = [ + (self.subtyping_provenance, sub.subtyping_provenance) + ] + return new_sub def __hash__(self) -> int: return hash(tuple(self.items())) @@ -893,6 +905,7 @@ def to_type(self, env: Environment) -> Tuple[TypeSequence, Environment]: temp_env = env.copy() if self._sequence_variable is None: # implicit stack polymorphism + # FIXME: This should be handled in stack effect construction sequence.append(SequenceVariable()) elif self._sequence_variable.name not in temp_env: temp_env = temp_env.copy() diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index d412b33..7911cf0 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -49,6 +49,12 @@ def __str__(self) -> str: string += '\n' + str(e) e = e.__cause__ or e.__context__ return string + if isinstance(self._data, concat.typecheck.Substitutions): + string = str(self._data) + string += '\n' + '\n'.join( + (map(str, self._data.subtyping_provenance)) + ) + return string return str(self._data) @@ -288,6 +294,12 @@ def __getitem__(self, args: 'TypeArguments') -> Type: assert list(self.kind.parameter_kinds) == [t.kind for t in args] return StuckTypeApplication(self, args) + def __repr__(self) -> str: + return f'' + + def __str__(self) -> str: + return f't_{id(self)}' + @property def attributes(self) -> Mapping[str, Type]: raise TypeError('Cannot get attributes of bound variables') @@ -366,6 +378,9 @@ def __init__(self) -> None: def __str__(self) -> str: return '*t_{}'.format(id(self)) + def __repr__(self) -> str: + return f'' + def constrain_and_bind_variables( self, supertype: Type, @@ -382,7 +397,9 @@ def constrain_and_bind_variables( isinstance(supertype, SequenceVariable) and supertype not in rigid_variables ): - return Substitutions([(supertype, self)]) + sub = Substitutions([(supertype, self)]) + sub.add_subtyping_provenance((self, supertype)) + return sub if self in rigid_variables: raise ConcatTypeError( '{} is fixed here and cannot become a subtype of another type'.format( @@ -396,7 +413,8 @@ def constrain_and_bind_variables( self, supertype, supertype ) ) - return Substitutions([(self, supertype)]) + sub = Substitutions([(self, supertype)]) + sub.add_subtyping_provenance((self, supertype)) def get_type_of_attribute(self, name: str) -> NoReturn: raise ConcatTypeError( @@ -579,8 +597,12 @@ def constrain_and_bind_variables( """ from concat.typecheck import Substitutions - if _contains_assumption(subtyping_assumptions, self, supertype): - return Substitutions() + if self is supertype or _contains_assumption( + subtyping_assumptions, self, supertype + ): + sub = Substitutions() + sub.add_subtyping_provenance((self, supertype)) + return sub if isinstance(supertype, SequenceVariable): supertype = TypeSequence([supertype]) @@ -589,7 +611,9 @@ def constrain_and_bind_variables( if self._is_empty(): # [] <: [] if supertype._is_empty(): - return Substitutions() + sub = Substitutions() + sub.add_subtyping_provenance((self, supertype)) + return sub # [] <: *a, *a is not rigid # --> *a = [] elif ( @@ -598,7 +622,9 @@ def constrain_and_bind_variables( and not supertype._individual_types and supertype._rest not in rigid_variables ): - return Substitutions([(supertype._rest, self)]) + sub = Substitutions([(supertype._rest, self)]) + sub.add_subtyping_provenance((self, supertype)) + return sub # [] <: *a? `t0 `t... # error else: @@ -608,13 +634,17 @@ def constrain_and_bind_variables( # --> *a = [] if supertype._is_empty() and self._rest not in rigid_variables: assert self._rest is not None - return Substitutions([(self._rest, supertype)]) + sub = Substitutions([(self._rest, supertype)]) + sub.add_subtyping_provenance((self, supertype)) + return sub # *a <: *a if ( self._rest is supertype._rest and not supertype._individual_types ): - return Substitutions() + sub = Substitutions() + sub.add_subtyping_provenance((self, supertype)) + return sub # *a <: *b? `t..., *a is not rigid, *a is not free in RHS # --> *a = RHS if ( @@ -622,7 +652,9 @@ def constrain_and_bind_variables( and self._rest not in rigid_variables and self._rest not in supertype.free_type_variables() ): - return Substitutions([(self._rest, supertype)]) + sub = Substitutions([(self._rest, supertype)]) + sub.add_subtyping_provenance((self, supertype)) + return sub else: raise StackMismatchError(self, supertype) else: @@ -638,7 +670,9 @@ def constrain_and_bind_variables( and supertype._rest not in self.free_type_variables() and supertype._rest not in rigid_variables ): - return Substitutions([(supertype._rest, self)]) + sub = Substitutions([(supertype._rest, self)]) + sub.add_subtyping_provenance((self, supertype)) + return sub # `t_n <: `s_m *a? `t... <: *b? `s... # --- # *a? `t... `t_n <: *b? `s... `s_m @@ -656,6 +690,7 @@ def constrain_and_bind_variables( rigid_variables, subtyping_assumptions, )(sub) + # sub.add_subtyping_provenance((self, supertype)) return sub except StackMismatchError: # TODO: Add info about occurs check and rigid @@ -992,7 +1027,9 @@ def constrain_and_bind_variables( if self is supertype or _contains_assumption( subtyping_assumptions, self, supertype ): - return Substitutions() + sub = Substitutions() + sub.add_subtyping_provenance((self, supertype)) + return sub # obj <: `t, `t is not rigid # --> `t = obj @@ -1000,7 +1037,9 @@ def constrain_and_bind_variables( isinstance(supertype, IndividualVariable) and supertype not in rigid_variables ): - return Substitutions([(supertype, self)]) + sub = Substitutions([(supertype, self)]) + sub.add_subtyping_provenance((self, supertype)) + return sub # obj <: *s? `t... # error elif isinstance(supertype, (SequenceVariable, TypeSequence)): @@ -1016,60 +1055,73 @@ def constrain_and_bind_variables( ) if isinstance(supertype, (StackEffect, PythonFunctionType)): - return self.get_type_of_attribute( + sub = self.get_type_of_attribute( '__call__' ).constrain_and_bind_variables( supertype, rigid_variables, subtyping_assumptions + [(self, supertype)], ) + sub.add_subtyping_provenance((self, supertype)) + return sub if isinstance(supertype, _OptionalType): try: - return self.constrain_and_bind_variables( + sub = self.constrain_and_bind_variables( none_type, rigid_variables, subtyping_assumptions + [(self, supertype)], ) + sub.add_subtyping_provenance((self, supertype)) + return sub except ConcatTypeError: - return self.constrain_and_bind_variables( + sub = self.constrain_and_bind_variables( supertype.type_arguments[0], rigid_variables, subtyping_assumptions + [(self, supertype)], ) + sub.add_subtyping_provenance((self, supertype)) + return sub if isinstance(supertype, _NoReturnType): raise ConcatTypeError( f'No other type, in this case, {self}, is a subtype of {supertype}' ) if isinstance(supertype, Fix): unrolled = supertype.unroll() - return self.constrain_and_bind_variables( + sub = self.constrain_and_bind_variables( unrolled, rigid_variables, subtyping_assumptions + [(self, supertype)], ) + sub.add_subtyping_provenance((self, supertype)) + return sub if isinstance(supertype, ForwardTypeReference): resolved = supertype.resolve_forward_references() - return self.constrain_and_bind_variables( + sub = self.constrain_and_bind_variables( resolved, rigid_variables, subtyping_assumptions + [(self, supertype)], ) + sub.add_subtyping_provenance((self, supertype)) + return sub if not isinstance(supertype, ObjectType): raise NotImplementedError(supertype) # every object type is a subtype of object_type if supertype is get_object_type(): - return Substitutions() + sub = Substitutions() + sub.add_subtyping_provenance((self, supertype)) + return sub # Don't forget that there's nominal subtyping too. if supertype._nominal: if supertype in self._nominal_supertypes: - return Substitutions() + sub = Substitutions() + sub.add_subtyping_provenance((self, supertype)) + return sub if self._head is not supertype._head: raise ConcatTypeError( '{} is not a subtype of {}'.format(self, supertype) ) - # BUG - subtyping_assumptions.append((self, supertype)) + subtyping_assumptions = subtyping_assumptions + [(self, supertype)] # don't constrain the type arguments, constrain those based on # the attributes @@ -1081,6 +1133,7 @@ def constrain_and_bind_variables( rigid_variables, subtyping_assumptions, )(sub) + sub.add_subtyping_provenance((self, supertype)) return sub def get_type_of_attribute(self, attribute: str) -> Type: @@ -1127,18 +1180,22 @@ def constrain_and_bind_variables( not supertype.has_attribute('__call__') or '__init__' not in self._attributes ): - return super().constrain_and_bind_variables( + sub = super().constrain_and_bind_variables( supertype, rigid_variables, subtyping_assumptions ) + sub.add_subtyping_provenance((self, supertype)) + return sub init = self.get_type_of_attribute('__init__') while not isinstance(init, (StackEffect, PythonFunctionType)): init = init.get_type_of_attribute('__call__') bound_init = init.bind() - return bound_init.constrain_and_bind_variables( + sub = bound_init.constrain_and_bind_variables( supertype.get_type_of_attribute('__call__'), rigid_variables, subtyping_assumptions + [(self, supertype)], ) + sub.add_subtyping_provenance((self, supertype)) + return sub class PythonFunctionType(IndividualType): @@ -1151,16 +1208,16 @@ def __init__( super().__init__() self._arity = len(type_parameters) self._type_parameters = type_parameters - self._type_arguments = _type_arguments + self._type_arguments: Sequence[Type] = [] self._overloads: Sequence[Tuple[Type, Type]] = [] if not ( self._arity == 0 - and len(self._type_arguments) == 2 + and len(_type_arguments) == 2 or self._arity == 2 - and len(self._type_arguments) == 0 + and len(_type_arguments) == 0 ): raise ConcatTypeError( - f'Ill-formed Python function type with arguments {self._type_arguments}' + f'Ill-formed Python function type with arguments {_type_arguments}' ) if self._arity == 0: i, o = _type_arguments @@ -1168,21 +1225,37 @@ def __init__( raise ConcatTypeError( f'{i} must be a sequence type, but has kind {i.kind}' ) + # HACK: Sequence variables are introduced by the type sequence AST nodes + if ( + isinstance(i, TypeSequence) + and i + and i[0].kind == SequenceKind() + ): + i = TypeSequence(i.as_sequence()[1:]) + _type_arguments = i, o if o.kind != IndividualKind(): raise ConcatTypeError( f'{o} must be an individual type, but has kind {o.kind}' ) + _fixed_overloads: List[Tuple[Type, Type]] = [] for i, o in _overloads: if i.kind != SequenceKind(): raise ConcatTypeError( f'{i} must be a sequence type, but has kind {i.kind}' ) + if ( + isinstance(i, TypeSequence) + and i + and i[0].kind == SequenceKind() + ): + i = TypeSequence(i.as_sequence()[1:]) if o.kind != IndividualKind(): raise ConcatTypeError( f'{o} must be an individual type, but has kind {o.kind}' ) - self._overloads = _overloads - self._hash: Optional[int] = None + _fixed_overloads.append((i, o)) + self._overloads = _fixed_overloads + self._type_arguments = _type_arguments def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: if self._arity == 0: @@ -1338,25 +1411,33 @@ def constrain_and_bind_variables( if self is supertype or _contains_assumption( subtyping_assumptions, self, supertype ): - return Substitutions() + sub = Substitutions() + sub.add_subtyping_provenance((self, supertype)) + return sub if self.kind != supertype.kind: raise ConcatTypeError( f'{self} has kind {self.kind} but {supertype} has kind {supertype.kind}' ) if self.kind == IndividualKind(): if supertype is get_object_type(): - return Substitutions() + sub = Substitutions() + sub.add_subtyping_provenance((self, supertype)) + return sub if ( isinstance(supertype, IndividualVariable) and supertype not in rigid_variables ): - return Substitutions([(supertype, self)]) + sub = Substitutions([(supertype, self)]) + sub.add_subtyping_provenance((self, supertype)) + return sub if isinstance(supertype, _OptionalType): - return self.constrain_and_bind_variables( + sub = self.constrain_and_bind_variables( supertype.type_arguments[0], rigid_variables, subtyping_assumptions, ) + sub.add_subtyping_provenance((self, supertype)) + return sub if isinstance(supertype, ObjectType) and not supertype.nominal: sub = Substitutions() for attr in supertype.attributes: @@ -1369,10 +1450,13 @@ def constrain_and_bind_variables( rigid_variables, subtyping_assumptions, ) + sub.add_subtyping_provenance((self, supertype)) return sub if isinstance(supertype, PythonFunctionType): if isinstance(self.kind, GenericTypeKind): - return Substitutions() + sub = Substitutions() + sub.add_subtyping_provenance((self, supertype)) + return sub # No need to extend the rigid variables, we know both types have no # parameters at this point. @@ -1396,6 +1480,7 @@ def constrain_and_bind_variables( rigid_variables, subtyping_assumptions_copy, )(sub) + sub.add_subtyping_provenance((self, supertype)) return sub except ConcatTypeError: continue @@ -1594,8 +1679,6 @@ def __eq__(self, other: object) -> bool: class Fix(Type): def __init__(self, var: _Variable, body: Type) -> None: - from concat.typecheck import Substitutions - super().__init__() assert var.kind == body.kind self._var = var @@ -1606,6 +1689,11 @@ def __init__(self, var: _Variable, body: Type) -> None: def __repr__(self) -> str: return f'Fix({self._var!r}, {self._body!r})' + def __str__(self) -> str: + if self._internal_name is not None: + return self._internal_name + return f'Fix({self._var}, {self._body})' + def _apply(self, t: Type) -> Type: from concat.typecheck import Substitutions @@ -1651,21 +1739,27 @@ def constrain_and_bind_variables( if supertype is get_object_type() or _contains_assumption( subtyping_assumptions, self, supertype ): - return Substitutions() + sub = Substitutions() + sub.add_subtyping_provenance((self, supertype)) + return sub if isinstance(supertype, Fix): unrolled = supertype.unroll() - return self.unroll().constrain_and_bind_variables( + sub = self.unroll().constrain_and_bind_variables( unrolled, rigid_variables, subtyping_assumptions + [(self, supertype)], ) + sub.add_subtyping_provenance((self, supertype)) + return sub - return self.unroll().constrain_and_bind_variables( + sub = self.unroll().constrain_and_bind_variables( supertype, rigid_variables, subtyping_assumptions + [(self, supertype)], ) + sub.add_subtyping_provenance((self, supertype)) + return sub @property def kind(self) -> Kind: diff --git a/language-concat/lib/main.ts b/language-concat/lib/main.ts index 903afb4..5842e8d 100644 --- a/language-concat/lib/main.ts +++ b/language-concat/lib/main.ts @@ -32,4 +32,8 @@ module.exports = { consumeLinterV2: concatLanguageClient.consumeLinterV2.bind( concatLanguageClient ), + + consumeDatatip: concatLanguageClient.consumeDatatip.bind( + concatLanguageClient + ), }; diff --git a/language-concat/package.json b/language-concat/package.json index 5c1c432..4f17017 100644 --- a/language-concat/package.json +++ b/language-concat/package.json @@ -36,6 +36,11 @@ "versions": { "2.0.0": "consumeLinterV2" } + }, + "datatip": { + "versions": { + "0.1.0": "consumeDatatip" + } } } } From da1b127953c5709b31c3d2505a974ed9df22aea5 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Mon, 3 Jun 2024 23:52:37 -0700 Subject: [PATCH 24/61] Define get_type_of_attribute in terms of attributes --- concat/typecheck/types.py | 36 +++++++----------------------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 7911cf0..ef603fd 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -94,20 +94,22 @@ def __eq__(self, other: object) -> bool: # hash consing, but that would only reflect syntactic eequality, and I've # been using hashing for type equality. - # TODO: Define in terms of .attributes def get_type_of_attribute(self, name: str) -> 'Type': - raise ConcatAttributeError(self, name) + attributes = self.attributes + if name not in attributes: + raise ConcatAttributeError(self, name) + return attributes[name] def has_attribute(self, name: str) -> bool: try: self.get_type_of_attribute(name) return True - except AttributeError: + except ConcatAttributeError: return False @abc.abstractproperty def attributes(self) -> Mapping[str, 'Type']: - pass + return {} @abc.abstractmethod def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: @@ -415,11 +417,7 @@ def constrain_and_bind_variables( ) sub = Substitutions([(self, supertype)]) sub.add_subtyping_provenance((self, supertype)) - - def get_type_of_attribute(self, name: str) -> NoReturn: - raise ConcatTypeError( - 'the sequence type {} does not hold attributes'.format(self) - ) + return sub @property def attributes(self) -> NoReturn: @@ -855,11 +853,6 @@ def __str__(self) -> str: out_types = ' '.join(map(str, self.output)) return '({} -- {})'.format(in_types, out_types) - def get_type_of_attribute(self, name: str) -> '_Function': - if name == '__call__': - return self - raise ConcatAttributeError(self, name) - @property def attributes(self) -> Mapping[str, 'StackEffect']: return {'__call__': self} @@ -1136,12 +1129,6 @@ def constrain_and_bind_variables( sub.add_subtyping_provenance((self, supertype)) return sub - def get_type_of_attribute(self, attribute: str) -> Type: - if attribute not in self._attributes: - raise ConcatAttributeError(self, attribute) - - return self._attributes[attribute] - def __repr__(self) -> str: head = None if self._head is self else self._head return f'{type(self).__qualname__}(attributes={self._attributes!r}, nominal_supertypes={self._nominal_supertypes!r}, nominal={self._nominal!r}, _head={head!r})' @@ -1309,12 +1296,6 @@ def __str__(self) -> str: return 'py_function_type' return f'py_function_type[{self.input}, {self.output}]' - def get_type_of_attribute(self, attribute: str) -> Type: - if attribute == '__call__': - return self - else: - return super().get_type_of_attribute(attribute) - @property def attributes(self) -> Mapping[str, Type]: return {**super().attributes, '__call__': self} @@ -1728,9 +1709,6 @@ def apply_substitution(self, sub: 'Substitutions') -> Type: def attributes(self) -> Mapping[str, Type]: return self.unroll().attributes - def get_type_of_attribute(self, name: str) -> Type: - return self.attributes[name] - def constrain_and_bind_variables( self, supertype, rigid_variables, subtyping_assumptions ) -> 'Substitutions': From 4d8e98a68753bb717599b761c1bb2677f3d993b0 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Tue, 4 Jun 2024 00:10:03 -0700 Subject: [PATCH 25/61] Import concat.parse at runtime --- concat/typecheck/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concat/typecheck/errors.py b/concat/typecheck/errors.py index d597655..93fca27 100644 --- a/concat/typecheck/errors.py +++ b/concat/typecheck/errors.py @@ -1,11 +1,11 @@ import builtins +import concat.parse import pathlib from typing import Optional, Union, TYPE_CHECKING if TYPE_CHECKING: import concat.astutils - import concat.parse from concat.typecheck.types import Type, TypeSequence From 85adac8f7c729ef529d4892e0a82eb90fa70db70 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Wed, 5 Jun 2024 00:42:43 -0700 Subject: [PATCH 26/61] Give unrolled Fix the same id as the original type and use ids to compare with object type --- concat/tests/test_typecheck.py | 2 -- concat/typecheck/preamble.cati | 2 +- concat/typecheck/types.py | 12 ++++++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/concat/tests/test_typecheck.py b/concat/tests/test_typecheck.py index 4f7e253..50beb36 100644 --- a/concat/tests/test_typecheck.py +++ b/concat/tests/test_typecheck.py @@ -14,7 +14,6 @@ StackEffect, Type as ConcatType, TypeSequence, - addable_type, ellipsis_type, float_type, get_object_type, @@ -36,7 +35,6 @@ from hypothesis.strategies import ( dictionaries, from_type, - integers, sampled_from, text, ) diff --git a/concat/typecheck/preamble.cati b/concat/typecheck/preamble.cati index 887c7e9..beefbbc 100644 --- a/concat/typecheck/preamble.cati +++ b/concat/typecheck/preamble.cati @@ -16,7 +16,7 @@ class file: def to_list(*rest_var i:iterable[`a_var] -- *rest_var l:list[`a_var]): () -def py_call(*rest_var kwargs:iterable[object] args:iterable[object] f:py_function[(*seq_var), `a_var] -- *rest_var res:`a_var): +def py_call(*rest_var kwargs:iterable[object] args:iterable[object] f:py_function[*seq_var, `a_var] -- *rest_var res:`a_var): () def nip(*rest_var a:object b:`a_var -- *rest_var b:`a_var): diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index ef603fd..603adf4 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -1099,7 +1099,7 @@ def constrain_and_bind_variables( if not isinstance(supertype, ObjectType): raise NotImplementedError(supertype) # every object type is a subtype of object_type - if supertype is get_object_type(): + if supertype._type_id == get_object_type()._type_id: sub = Substitutions() sub.add_subtyping_provenance((self, supertype)) return sub @@ -1443,6 +1443,7 @@ def constrain_and_bind_variables( # parameters at this point. # Support overloading the subtype. + exceptions = [] for overload in [ (self.input, self.output), *self._overloads, @@ -1463,14 +1464,14 @@ def constrain_and_bind_variables( )(sub) sub.add_subtyping_provenance((self, supertype)) return sub - except ConcatTypeError: - continue + except ConcatTypeError as e: + exceptions.append(e) finally: subtyping_assumptions[:] = subtyping_assumptions_copy raise ConcatTypeError( 'no overload of {} is a subtype of {}'.format(self, supertype) - ) + ) from exceptions[0] class _PythonOverloadedType(Type): @@ -1689,6 +1690,9 @@ def _apply(self, t: Type) -> Type: def unroll(self) -> Type: if self._unrolled_ty is None: self._unrolled_ty = self._apply(self) + if self._internal_name is not None: + self._unrolled_ty.set_internal_name(self._internal_name) + self._unrolled_ty._type_id = self._type_id return self._unrolled_ty def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: From ac8816dcb77c34a5061bc0c8f23e9a241082d598 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Wed, 5 Jun 2024 21:50:06 -0700 Subject: [PATCH 27/61] Add verbose flag to print traceback for type errors --- concat/__main__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/concat/__main__.py b/concat/__main__.py index 221ad15..679128e 100644 --- a/concat/__main__.py +++ b/concat/__main__.py @@ -46,6 +46,12 @@ def func(name: str) -> IO[AnyStr]: default=False, help='turn stack debugging on', ) +arg_parser.add_argument( + '--verbose', + action='store_true', + default=False, + help='print internal logs and errors', +) arg_parser.add_argument( '--tokenize', action='store_true', @@ -94,6 +100,8 @@ def func(name: str) -> IO[AnyStr]: else: print(get_line_at(args.file, e.location), end='') print(' ' * e.location[1] + '^') + if args.verbose: + raise except concat.parser_combinators.ParseError as e: print('Parse Error:') print( From 48e911f9d36f1485ec66d3f8a68e6cc5e8747ad5 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Wed, 5 Jun 2024 23:21:14 -0700 Subject: [PATCH 28/61] Add type stub for itertools.islice --- concat/typecheck/__init__.py | 42 ++++++++++--------- concat/typecheck/builtin_stubs/itertools.cati | 6 +++ 2 files changed, 29 insertions(+), 19 deletions(-) create mode 100644 concat/typecheck/builtin_stubs/itertools.cati diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index ed20d2b..b52385b 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -398,7 +398,7 @@ def infer( # FIXME: Infer the type of elements in the list based on # ALL the elements. if element_type == no_return_type: - assert isinstance(collected_type[-1], IndividualType) + assert collected_type[-1] != SequenceKind() element_type = collected_type[-1] # drop the top of the stack to use as the item collected_type = collected_type[:-1] @@ -455,20 +455,28 @@ def infer( module_parts = node.value.split('.') module_spec = None path = None - for module_prefix in itertools.accumulate( - module_parts, lambda a, b: f'{a}.{b}' - ): - for finder in sys.meta_path: - module_spec = finder.find_spec(module_prefix, path) - if module_spec is not None: - path = module_spec.submodule_search_locations - break - assert module_spec is not None - module_path = module_spec.origin - if module_path is None: - raise TypeError(f'Cannot find path of module {node.value}') - # For now, assume the module's written in Python. - stub_path = pathlib.Path(module_path).with_suffix('.cati') + if module_parts[0] in sys.builtin_module_names: + stub_path = pathlib.Path(__file__) / '../builtin_stubs' + for part in module_parts: + stub_path = stub_path / part + else: + for module_prefix in itertools.accumulate( + module_parts, lambda a, b: f'{a}.{b}' + ): + for finder in sys.meta_path: + module_spec = finder.find_spec(module_prefix, path) + if module_spec is not None: + path = module_spec.submodule_search_locations + break + assert module_spec is not None + module_path = module_spec.origin + if module_path is None: + raise TypeError( + f'Cannot find path of module {node.value}' + ) + # For now, assume the module's written in Python. + stub_path = pathlib.Path(module_path) + stub_path = stub_path.with_suffix('.cati') stub_env = _check_stub(stub_path) imported_type = stub_env.get(node.imported_name) if imported_type is None: @@ -836,10 +844,6 @@ def to_type(self, env: Environment) -> Tuple[IndividualType, Environment]: arg_as_type, env = arg.to_type(env) args.append(arg_as_type) generic_type, env = self._generic_type.to_type(env) - if isinstance(generic_type, GenericType): - if generic_type.is_variadic: - args = (TypeSequence(args),) - return generic_type[args], env if isinstance(generic_type.kind, GenericTypeKind): return generic_type[args], env raise TypeError('{} is not a generic type'.format(generic_type)) diff --git a/concat/typecheck/builtin_stubs/itertools.cati b/concat/typecheck/builtin_stubs/itertools.cati new file mode 100644 index 0000000..24fea25 --- /dev/null +++ b/concat/typecheck/builtin_stubs/itertools.cati @@ -0,0 +1,6 @@ +def islice(--) @cast (py_overloaded[ + py_function[(iterable[`t] Optional[int]), iterator[`t]], + py_function[(iterable[`t] Optional[int] Optional[int]), iterator[`t]], + py_function[(iterable[`t] Optional[int] Optional[int] Optional[int]), iterator[`t]] # TODO: allow trailing commas +]): + () From 8075125242fe677c9b213395f4a752e449e87bd4 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Thu, 6 Jun 2024 00:39:17 -0700 Subject: [PATCH 29/61] Add type stubs for to_slice and abs --- concat/stdlib/pyinterop/__init__.cati | 3 +++ concat/stdlib/pyinterop/__init__.py | 14 -------------- concat/typecheck/__init__.py | 12 +++++------- .../builtins.cati} | 4 ++++ concat/typecheck/preamble_types.py | 5 +++++ 5 files changed, 17 insertions(+), 21 deletions(-) rename concat/typecheck/{py_builtins.cati => builtin_stubs/builtins.cati} (94%) diff --git a/concat/stdlib/pyinterop/__init__.cati b/concat/stdlib/pyinterop/__init__.cati index d8f2d26..18f9bcd 100644 --- a/concat/stdlib/pyinterop/__init__.cati +++ b/concat/stdlib/pyinterop/__init__.cati @@ -9,3 +9,6 @@ def to_str(*stack_type_var errors:object encoding:object obj:object -- *stack_ty def to_py_function(*rest_var f:(*rest_var_2 -- *rest_var_3) -- *rest_var py_f:py_function[*any_args, `any_ret]): () + +def to_slice(*stack_type_var step:Optional[`x] stop:Optional[`y] start:Optional[`z] -- *stack_type_var slice_:slice[`z, `y, `x]): + () diff --git a/concat/stdlib/pyinterop/__init__.py b/concat/stdlib/pyinterop/__init__.py index 9cad93c..78d2695 100644 --- a/concat/stdlib/pyinterop/__init__.py +++ b/concat/stdlib/pyinterop/__init__.py @@ -80,20 +80,6 @@ TypeSequence([_stack_type_var, dict_type[_x, _y]]), ), ), - 'to_slice': GenericType( - [_stack_type_var, _x, _y, _z], - StackEffect( - TypeSequence( - [ - _stack_type_var, - optional_type[_x,], - optional_type[_y,], - optional_type[_z,], - ] - ), - TypeSequence([_stack_type_var, slice_type[_z, _y, _x]]), - ), - ), } diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index b52385b..b37057c 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -188,10 +188,11 @@ def __hash__(self) -> int: import concat.parse +_builtins_stub_path = pathlib.Path(__file__) / '../builtin_stubs/builtins.cati' + + def load_builtins_and_preamble() -> Environment: - env = _check_stub( - pathlib.Path(__file__).with_name('py_builtins.cati'), is_builtins=True, - ) + env = _check_stub(_builtins_stub_path, is_builtins=True,) env = Environment( { **env, @@ -217,10 +218,7 @@ def check( builtins_stub_env = Environment() preamble_stub_env = Environment() if _should_load_builtins: - builtins_stub_env = _check_stub( - pathlib.Path(__file__).with_name('py_builtins.cati'), - is_builtins=True, - ) + builtins_stub_env = _check_stub(_builtins_stub_path, is_builtins=True,) if _should_load_preamble: preamble_stub_env = _check_stub( pathlib.Path(__file__).with_name('preamble.cati'), diff --git a/concat/typecheck/py_builtins.cati b/concat/typecheck/builtin_stubs/builtins.cati similarity index 94% rename from concat/typecheck/py_builtins.cati rename to concat/typecheck/builtin_stubs/builtins.cati index 58c1e26..115ea96 100644 --- a/concat/typecheck/py_builtins.cati +++ b/concat/typecheck/builtin_stubs/builtins.cati @@ -98,3 +98,7 @@ class list[`element]: () !@@concat.typecheck.builtin_list list + +# SupportAbs is the name of the protocol in Python +def abs(--) @cast (py_function[(SupportsAbs[`t]), `t]): + () diff --git a/concat/typecheck/preamble_types.py b/concat/typecheck/preamble_types.py index 520d54c..0db16c5 100644 --- a/concat/typecheck/preamble_types.py +++ b/concat/typecheck/preamble_types.py @@ -1,6 +1,7 @@ from concat.typecheck.types import ( GenericType, IndividualVariable, + ObjectType, SequenceVariable, StackEffect, TypeSequence, @@ -143,4 +144,8 @@ TypeSequence([_stack_type_var, not_implemented_type]), ), ), + 'SupportsAbs': GenericType( + [_a_var], + ObjectType({'__abs__': py_function_type[TypeSequence([]), _a_var],},), + ), } From 44266aa4f28f78ed45bf6382a6e4a4ce9aaf2d3b Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Thu, 6 Jun 2024 16:34:08 -0700 Subject: [PATCH 30/61] Handle variadic generic type arguments in GenericType --- concat/stdlib/pyinterop/__init__.py | 4 +--- concat/tests/typecheck/test_types.py | 5 +---- concat/typecheck/__init__.py | 5 +---- concat/typecheck/types.py | 32 +++++++++++++++++++--------- 4 files changed, 25 insertions(+), 21 deletions(-) diff --git a/concat/stdlib/pyinterop/__init__.py b/concat/stdlib/pyinterop/__init__.py index 78d2695..99568b8 100644 --- a/concat/stdlib/pyinterop/__init__.py +++ b/concat/stdlib/pyinterop/__init__.py @@ -72,9 +72,7 @@ TypeSequence( [ _stack_type_var, - optional_type[ - iterable_type[tuple_type[TypeSequence([_x, _y]),],], - ], + optional_type[iterable_type[tuple_type[_x, _y],],], ] ), TypeSequence([_stack_type_var, dict_type[_x, _y]]), diff --git a/concat/tests/typecheck/test_types.py b/concat/tests/typecheck/test_types.py index 59affa3..76aa14f 100644 --- a/concat/tests/typecheck/test_types.py +++ b/concat/tests/typecheck/test_types.py @@ -120,10 +120,7 @@ def test_stack_effect_input_supertype(self) -> None: class TestFix(unittest.TestCase): fix_var = BoundVariable(IndividualKind()) linked_list = Fix( - fix_var, - optional_type[ - tuple_type[TypeSequence([get_object_type(), fix_var]),], - ], + fix_var, optional_type[tuple_type[get_object_type(), fix_var],], ) def test_unroll_supertype(self) -> None: diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index b37057c..7da6b8c 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -440,10 +440,7 @@ def infer( StackEffect( i, TypeSequence( - [ - *collected_type, - tuple_type[TypeSequence(element_types),], - ] + [*collected_type, tuple_type[element_types],] ), ) ), diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 603adf4..8af1fc5 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -510,18 +510,29 @@ def constrain_and_bind_variables( if self is supertype or _contains_assumption( subtyping_assumptions, self, supertype ): - return Substitutions([]) + return Substitutions() + # HACK: KIND_POLY I should use kind polymorphism instead to get the + # continuation example to typecheck. + if supertype._type_id == get_object_type()._type_id: + return Substitutions() + if isinstance(supertype, IndividualVariable) and supertype not in rigid_variables: + return Substitutions([(supertype, self)]) + if supertype.kind == IndividualKind(): + return self.instantiate().constrain_and_bind_variables(supertype, rigid_variables, subtyping_assumptions) if self.kind != supertype.kind: + # HACK: KIND_POLY + if isinstance(supertype.kind, GenericTypeKind): + return self.instantiate().constrain_and_bind_variables(supertype.instantiate(), rigid_variables, subtyping_assumptions) raise ConcatTypeError( f'{self} has kind {self.kind} but {supertype} has kind {supertype.kind}' ) - if not isinstance(supertype, GenericType): - raise NotImplementedError(supertype) shared_vars = [type(var)() for var in self._type_parameters] self_instance = self[shared_vars] supertype_instance = supertype[shared_vars] rigid_variables = ( rigid_variables + # QUESTION: The parameters have been substituted already. Does this + # make sense? Maybe the shared_vars should be rigid. | set(self._type_parameters) | set(supertype._type_parameters) ) @@ -539,7 +550,11 @@ def apply_substitution(self, sub: 'Substitutions') -> 'GenericType': if var not in self._type_parameters } ) - ty = GenericType(self._type_parameters, sub(self._body)) + ty = GenericType( + self._type_parameters, + sub(self._body), + is_variadic=self.is_variadic, + ) return ty @property @@ -562,6 +577,9 @@ def __init__(self, sequence: Sequence[Type]) -> None: else: self._rest = None self._individual_types = sequence + for ty in self._individual_types: + if ty.kind == SequenceKind(): + raise ConcatTypeError(f'{ty} cannot be a sequence type') def as_sequence(self) -> Sequence[Type]: if self._rest is not None: @@ -758,12 +776,6 @@ class _Function(IndividualType): def __init__( self, input_types: TypeSequence, output_types: TypeSequence, ) -> None: - for ty in input_types[1:]: - if ty.kind != IndividualKind(): - raise ConcatTypeError(f'{ty} must be an individual type') - for ty in output_types[1:]: - if ty.kind != IndividualKind(): - raise ConcatTypeError(f'{ty} must be an individual type') super().__init__() self.input = input_types self.output = output_types From da0b9a6f45211b1b8ed163194647e35aa8d1708a Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Thu, 6 Jun 2024 16:45:39 -0700 Subject: [PATCH 31/61] Add cast because type variable failed to be unified --- concat/examples/strstr.cat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/concat/examples/strstr.cat b/concat/examples/strstr.cat index 9820723..84ade23 100644 --- a/concat/examples/strstr.cat +++ b/concat/examples/strstr.cat @@ -17,6 +17,7 @@ def simple_str(obj:object -- string:str): None swap None swap to_str def strstr(haystack:str needle:str -- index:int): - [(), None, None] None to_dict swap pick$.find py_call cast (int) nip + # FIXME: These casts should be unnecessary + [(), None, None] None to_dict swap pick cast (str) $.find py_call cast (int) nip get_input get_input strstr simple_str put_output From 8b8670e15abd388cc66e2857a526aab0e6e61d3f Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Thu, 6 Jun 2024 16:55:47 -0700 Subject: [PATCH 32/61] Allow some (probably broken) subtyping between type of different arities I did this to get continuation.cat to typecheck. But I don't want that kind of subtyping anymore. In the future I will use kind polymorphism. --- concat/typecheck/__init__.py | 19 ++++++------------- concat/typecheck/types.py | 20 +++++++++++++++----- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index 7da6b8c..a94c56c 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -72,11 +72,12 @@ def __init__( ] = {}, ) -> None: self._sub = dict(sub) - for variable, ty in self._sub.items(): - if variable.kind != ty.kind: - raise TypeError( - f'{variable} is being substituted by {ty}, which has the wrong kind ({variable.kind} vs {ty.kind})' - ) + # See HACK KIND_POLY + # for variable, ty in self._sub.items(): + # if variable.kind != ty.kind: + # raise TypeError( + # f'{variable} is being substituted by {ty}, which has the wrong kind ({variable.kind} vs {ty.kind})' + # ) self._cache: Dict[int, Type] = {} # innermost first self.subtyping_provenance: List[Any] = [] @@ -319,10 +320,6 @@ def infer( if isinstance(child, concat.parse.AttributeWordNode): top = o1[-1] attr_type = top.get_type_of_attribute(child.value) - if not isinstance(attr_type, IndividualType): - raise TypeError( - f'{attr_type} must be an individual type' - ) if should_instantiate: attr_type = attr_type.instantiate() rest_types = o1[:-1] @@ -339,10 +336,6 @@ def infer( name_type = gamma[child.value] if isinstance(name_type, ForwardTypeReference): raise NameError(child) - if not isinstance(name_type, IndividualType): - raise TypeError( - f'{name_type} must be an individual type' - ) if should_instantiate: name_type = name_type.instantiate() current_effect = StackEffect( diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 8af1fc5..be788b6 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -242,10 +242,9 @@ class _Variable(Type, abc.ABC): def apply_substitution( self, sub: 'concat.typecheck.Substitutions' - ) -> Union['IndividualType', '_Variable', 'TypeSequence']: + ) -> Type: if self in sub: result = sub[self] - assert self.kind == result.kind, f'{self!r} --> {result!r}' return result # type: ignore return self @@ -468,6 +467,8 @@ def __getitem__(self, type_arguments: 'TypeArguments') -> 'Type': if type_argument_ids in self._instantiations: return self._instantiations[type_argument_ids] expected_kinds = [var.kind for var in self._type_parameters] + if self.is_variadic: + type_arguments = [TypeSequence(type_arguments)] actual_kinds = [ty.kind for ty in type_arguments] if expected_kinds != actual_kinds: raise ConcatTypeError( @@ -515,14 +516,23 @@ def constrain_and_bind_variables( # continuation example to typecheck. if supertype._type_id == get_object_type()._type_id: return Substitutions() - if isinstance(supertype, IndividualVariable) and supertype not in rigid_variables: + if ( + isinstance(supertype, IndividualVariable) + and supertype not in rigid_variables + ): return Substitutions([(supertype, self)]) if supertype.kind == IndividualKind(): - return self.instantiate().constrain_and_bind_variables(supertype, rigid_variables, subtyping_assumptions) + return self.instantiate().constrain_and_bind_variables( + supertype, rigid_variables, subtyping_assumptions + ) if self.kind != supertype.kind: # HACK: KIND_POLY if isinstance(supertype.kind, GenericTypeKind): - return self.instantiate().constrain_and_bind_variables(supertype.instantiate(), rigid_variables, subtyping_assumptions) + return self.instantiate().constrain_and_bind_variables( + supertype.instantiate(), + rigid_variables, + subtyping_assumptions, + ) raise ConcatTypeError( f'{self} has kind {self.kind} but {supertype} has kind {supertype.kind}' ) From 9a11cbf4bfc589f9b946d5da7383ac1e4a443d81 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Fri, 19 Jul 2024 16:51:08 -0700 Subject: [PATCH 33/61] Implement kind subtyping --- concat/kind-poly.md | 55 +++ concat/parse.py | 76 ++-- concat/stdlib/continuation.py | 8 - concat/stdlib/pyinterop/__init__.py | 13 +- concat/tests/strategies.py | 5 +- concat/tests/test_typecheck.py | 11 +- concat/tests/typecheck/test_types.py | 26 +- concat/typecheck/__init__.py | 85 +++-- concat/typecheck/preamble.cati | 6 +- concat/typecheck/preamble_types.py | 10 +- concat/typecheck/types.py | 527 ++++++++++++++------------- 11 files changed, 448 insertions(+), 374 deletions(-) create mode 100644 concat/kind-poly.md diff --git a/concat/kind-poly.md b/concat/kind-poly.md new file mode 100644 index 0000000..f223cfb --- /dev/null +++ b/concat/kind-poly.md @@ -0,0 +1,55 @@ +# Kind Polymorphism + +* Will probably want kind variables in the future + * Introduce a kind `Kind` at that point +* And kind aliases, too + +Probably, the easiest migration is to take the individual variable syntax: + +``` +`t +``` + +And use it to mean item-kinded variables instead. + +## Prior art + +* [Adding kind-polymorphism to the Scala programming language](https://codesync.global/media/adding-kind-polymorphism-to-the-scala-programming-language/) + * [`AnyKind`](https://www.scala-lang.org/api/current/scala/AnyKind.html) + +## Kinds + +* (?) Item: the kind of types of stack items + * Just use Individual? + * I use kinds for arity checking too, so I think that would make it harder + * Needed to exclude Sequence from kind-polymorphic type parameters where a + Sequence is invalid, e.g: + * `def drop[t : Item](*s t -- *s)` (possible syntax) +* Individual: the kind of zero-arity types of values +* Generic: type constructors + * they have arities + * they always construct an individual type + * I should change this + * they can be the type of a value, e.g. polymorphic functions +* Sequence: stack types + +### Subkinding + +``` + Item + / \ +Individual Generic + + +Sequence +``` + +## Syntax + +* `def drop[t : Item](*s t -- *s)` + * First thing I thought of + * Looks more natural to me + * Similar to type parameter syntax for classes +* `def drop(forall (t : Item). (*s t -- *s)):` + * Uses existing forall syntax, but extended + * Opens the door to allowing any type syntax as a type annotation diff --git a/concat/parse.py b/concat/parse.py index a083b18..6131c15 100644 --- a/concat/parse.py +++ b/concat/parse.py @@ -30,8 +30,6 @@ def word_ext(parsers): import operator from typing import ( Any, - Callable, - Dict, Generator, Iterable, Iterator, @@ -307,6 +305,7 @@ class FuncdefStatementNode(StatementNode): def __init__( self, name: 'Token', + type_parameters: Sequence[Tuple['Token', Node]], decorators: Iterable[WordNode], annotation: Optional[Iterable[WordNode]], body: 'WordsOrStatements', @@ -316,25 +315,21 @@ def __init__( super().__init__() self.location = location self.name = name.value + self.type_parameters = type_parameters self.decorators = decorators self.annotation = annotation self.body = body + self.stack_effect = stack_effect self.children = [ + *self.type_parameters, *self.decorators, *(self.annotation or []), + self.stack_effect, *self.body, ] - self.stack_effect = stack_effect def __repr__(self) -> str: - return 'FuncdefStatementNode(decorators={!r}, name={!r}, annotation={!r}, body={!r}, stack_effect={!r}, location={!r})'.format( - self.decorators, - self.name, - self.annotation, - self.body, - self.stack_effect, - self.location, - ) + return f'FuncdefStatementNode(decorators={self.decorators!r}, name={self.name!r}, type_parameters={self.type_parameters!r}, annotation={self.annotation!r}, body={self.body!r}, stack_effect={self.stack_effect!r}, location={self.location!r})' class FromImportStatementNode(ImportStatementNode): @@ -429,10 +424,6 @@ def top_level_parser() -> Generator[ # statement = import statement ; parsers['statement'] = parsers.ref_parser('import-statement') - ImportStatementParserGenerator = Generator[ - concat.parser_combinators.Parser, Any, ImportStatementNode - ] - # This parses one of many types of word. # The specific word node is returned. # word = @@ -577,26 +568,49 @@ def word_list_parser() -> Generator: ) # This parses a function definition. - # funcdef statement = DEF, NAME, stack effect, decorator*, + # funcdef statement = DEF, NAME, [ type parameters ], stack effect, decorator*, # [ annotation ], COLON, suite ; # decorator = AT, word ; # annotation = RARROW, word* ; # suite = NEWLINE, INDENT, (word | statement, NEWLINE)+, DEDENT | statement # | word+ ; + # type parameters = "[", [ type parameter ], + # (",", type parameter)*, [ "," ], "]" ; + # type parameter = NAME, COLON, type ; # The stack effect syntax is defined within the typecheck module. @concat.parser_combinators.generate def funcdef_statement_parser() -> Generator: location = (yield token('DEF')).start name = yield token('NAME') + type_params = (yield type_parameters.optional()) or [] effect_ast = yield parsers['stack-effect-type'] decorators = yield decorator.many() annotation = yield annotation_parser.optional() yield token('COLON') body = yield suite return FuncdefStatementNode( - name, decorators, annotation, body, location, effect_ast + name, + type_params, + decorators, + annotation, + body, + location, + effect_ast, ) + @concat.parser_combinators.generate + def type_parameter() -> Generator: + name = yield token('NAME') + yield token('COLON') + ty = yield parsers['type'] + return (name, ty) + + type_parameters = bracketed( + token('LSQB'), + type_parameter.sep_by(token('COMMA')) << token('COMMA').optional(), + token('RQSB'), + ).map(handle_recovery) + parsers['funcdef-statement'] = funcdef_statement_parser.desc( 'funcdef statement' ) @@ -701,20 +715,6 @@ def ellispis_verify( return concat.parser_combinators.success(None) return concat.parser_combinators.fail('a literal ellispis (...)') - def handle_recovery( - x: Union[ - Sequence[Node], - Tuple[Any, concat.parser_combinators.Result[Any]], - ] - ) -> Sequence[Node]: - if ( - isinstance(x, tuple) - and len(x) > 1 - and isinstance(x[1], concat.parser_combinators.Result) - ): - return [ParseError(x[1])] - return x - ellispis_parser = token('NAME').bind(ellispis_verify) type_parameters = ( yield bracketed( @@ -818,3 +818,17 @@ def freeze_word_parser() -> Generator: # parsers['word'] |= parsers['freeze-word'].should_fail( # 'not a freeze word, which has polymorphic type' # ) + + +def handle_recovery( + x: Union[ + Sequence[Node], Tuple[Any, concat.parser_combinators.Result[Any]], + ] +) -> Sequence[Node]: + if ( + isinstance(x, tuple) + and len(x) > 1 + and isinstance(x[1], concat.parser_combinators.Result) + ): + return [ParseError(x[1])] + return x diff --git a/concat/stdlib/continuation.py b/concat/stdlib/continuation.py index c53980b..2393390 100644 --- a/concat/stdlib/continuation.py +++ b/concat/stdlib/continuation.py @@ -1,8 +1,4 @@ from concat.common_types import ConcatFunction -from concat.typecheck.types import ( - IndividualVariable, - SequenceVariable, -) from typing import Any, Callable, Generic, List, NoReturn, Type, TypeVar, cast @@ -86,10 +82,6 @@ def compose( # Concat API -_s, _t, _u = SequenceVariable(), SequenceVariable(), SequenceVariable() -_a, _b, _r = IndividualVariable(), IndividualVariable(), IndividualVariable() - - def call_with_current_continuation( stack: List[object], stash: List[object] ) -> None: diff --git a/concat/stdlib/pyinterop/__init__.py b/concat/stdlib/pyinterop/__init__.py index 99568b8..e94c31a 100644 --- a/concat/stdlib/pyinterop/__init__.py +++ b/concat/stdlib/pyinterop/__init__.py @@ -3,15 +3,14 @@ import concat.stdlib.ski from concat.typecheck.types import ( GenericType, - IndividualVariable, - ObjectType, + IndividualKind, + ItemVariable, SequenceVariable, StackEffect, TypeSequence, dict_type, iterable_type, optional_type, - slice_type, subscriptable_type, tuple_type, ) @@ -39,9 +38,9 @@ _rest_var = SequenceVariable() _rest_var_2 = SequenceVariable() _rest_var_3 = SequenceVariable() -_x = IndividualVariable() -_y = IndividualVariable() -_z = IndividualVariable() +_x = ItemVariable(IndividualKind) +_y = ItemVariable(IndividualKind) +_z = ItemVariable(IndividualKind) globals()['@@types'] = { 'getitem': GenericType( [_stack_type_var, _x, _y], @@ -337,7 +336,7 @@ def python_f(x: object) -> object: f(stack, stash) return stack.pop() - stack.append(map(python_f, iterable)) + stack.append(builtins.map(python_f, iterable)) def open(stack: List[object], stash: List[object]) -> None: diff --git a/concat/tests/strategies.py b/concat/tests/strategies.py index d8b4e90..834d747 100644 --- a/concat/tests/strategies.py +++ b/concat/tests/strategies.py @@ -1,6 +1,7 @@ from concat.typecheck.types import ( + IndividualKind, IndividualType, - IndividualVariable, + ItemVariable, ObjectType, PythonFunctionType, SequenceVariable, @@ -72,7 +73,7 @@ def _mark_individual_type_strategy( _individual_type_strategy = recursive( _mark_individual_type_strategy( - from_type(IndividualVariable), IndividualVariable + builds(ItemVariable, just(IndividualKind)), ItemVariable ) | _mark_individual_type_strategy( just(no_return_type), type(no_return_type) diff --git a/concat/tests/test_typecheck.py b/concat/tests/test_typecheck.py index 50beb36..a4fcf4f 100644 --- a/concat/tests/test_typecheck.py +++ b/concat/tests/test_typecheck.py @@ -9,7 +9,8 @@ Fix, IndividualKind, IndividualType, - IndividualVariable, + IndividualKind, + ItemVariable, ObjectType, StackEffect, Type as ConcatType, @@ -192,8 +193,8 @@ def test_cast_word(self) -> None: class TestStackEffectParser(unittest.TestCase): _a_bar = concat.typecheck.SequenceVariable() _d_bar = concat.typecheck.SequenceVariable() - _b = concat.typecheck.IndividualVariable() - _c = concat.typecheck.IndividualVariable() + _b = concat.typecheck.ItemVariable(IndividualKind) + _c = concat.typecheck.ItemVariable(IndividualKind) examples: Dict[str, StackEffect] = { 'a b -- b a': StackEffect( TypeSequence([_a_bar, _b, _c]), TypeSequence([_a_bar, _c, _b]) @@ -273,7 +274,7 @@ def test_builtin_name_does_not_exist_in_empty_environment(self) -> None: ) @example( named_type_node=concat.typecheck.NamedTypeNode((0, 0), ''), - type=IndividualVariable(), + type=ItemVariable(IndividualKind), ) def test_name_does_exist(self, named_type_node, type) -> None: env = concat.typecheck.Environment({named_type_node.name: type}) @@ -392,7 +393,7 @@ def test_class_subtype_of_stack_effect(self, effect) -> None: ) ) def test_class_subtype_of_py_function(self, type1, type2) -> None: - x = IndividualVariable() + x = ItemVariable(IndividualKind) py_function = py_function_type[TypeSequence([type1]), type2] unbound_py_function = py_function_type[TypeSequence([x, type1]), type2] cls = Fix(x, ClassType({'__init__': unbound_py_function})) diff --git a/concat/tests/typecheck/test_types.py b/concat/tests/typecheck/test_types.py index 76aa14f..fdaf577 100644 --- a/concat/tests/typecheck/test_types.py +++ b/concat/tests/typecheck/test_types.py @@ -9,7 +9,7 @@ Fix, ForwardTypeReference, IndividualKind, - IndividualVariable, + ItemVariable, ObjectType, SequenceVariable, StackEffect, @@ -29,68 +29,68 @@ class TestIndividualVariableConstrain(unittest.TestCase): def test_individual_variable_subtype(self) -> None: - v = IndividualVariable() + v = ItemVariable(IndividualKind) ty = get_int_type() sub = v.constrain_and_bind_variables(ty, set(), []) self.assertEqual(ty, sub(v)) def test_individual_variable_supertype(self) -> None: - v = IndividualVariable() + v = ItemVariable(IndividualKind) ty = get_int_type() sub = ty.constrain_and_bind_variables(v, set(), []) self.assertEqual(ty, sub(v)) def test_attribute_subtype(self) -> None: - v = IndividualVariable() + v = ItemVariable(IndividualKind) attr_ty = ObjectType({'__add__': v}) ty = get_int_type() with self.assertRaises(ConcatTypeError): attr_ty.constrain_and_bind_variables(ty, set(), []) def test_attribute_supertype(self) -> None: - v = IndividualVariable() + v = ItemVariable(IndividualKind) attr_ty = ObjectType({'__add__': v}) ty = get_int_type() sub = ty.constrain_and_bind_variables(attr_ty, set(), []) self.assertEqual(ty.get_type_of_attribute('__add__'), sub(v)) def test_py_function_return_subtype(self) -> None: - v = IndividualVariable() + v = ItemVariable(IndividualKind) py_fun_ty = py_function_type[TypeSequence([get_int_type()]), v] ty = get_int_type().get_type_of_attribute('__add__') sub = py_fun_ty.constrain_and_bind_variables(ty, set(), []) self.assertEqual(get_int_type(), sub(v)) def test_py_function_return_supertype(self) -> None: - v = IndividualVariable() + v = ItemVariable(IndividualKind) py_fun_ty = py_function_type[TypeSequence([get_int_type()]), v] ty = get_int_type().get_type_of_attribute('__add__') sub = ty.constrain_and_bind_variables(py_fun_ty, set(), []) self.assertEqual(get_int_type(), sub(v)) def test_type_sequence_subtype(self) -> None: - v = IndividualVariable() + v = ItemVariable(IndividualKind) seq_ty = TypeSequence([v]) ty = TypeSequence([get_int_type()]) sub = seq_ty.constrain_and_bind_variables(ty, set(), []) self.assertEqual(get_int_type(), sub(v)) def test_type_sequence_supertype(self) -> None: - v = IndividualVariable() + v = ItemVariable(IndividualKind) seq_ty = TypeSequence([v]) ty = TypeSequence([get_int_type()]) sub = ty.constrain_and_bind_variables(seq_ty, set(), []) self.assertEqual(get_int_type(), sub(v)) def test_int_addable(self) -> None: - v = IndividualVariable() + v = ItemVariable(IndividualKind) sub = get_int_type().constrain_and_bind_variables( addable_type[v, v], set(), [] ) self.assertEqual(get_int_type(), sub(v)) def test_int__add__addable__add__(self) -> None: - v = IndividualVariable() + v = ItemVariable(IndividualKind) int_add = get_int_type().get_type_of_attribute('__add__') addable_add = addable_type[v, v].get_type_of_attribute('__add__') sub = int_add.constrain_and_bind_variables(addable_add, set(), []) @@ -118,7 +118,7 @@ def test_stack_effect_input_supertype(self) -> None: class TestFix(unittest.TestCase): - fix_var = BoundVariable(IndividualKind()) + fix_var = BoundVariable(IndividualKind) linked_list = Fix( fix_var, optional_type[tuple_type[get_object_type(), fix_var],], ) @@ -146,7 +146,7 @@ def test_unroll_equal(self) -> None: class TestForwardReferences(unittest.TestCase): env = Environment({'ty': get_object_type()}) - ty = ForwardTypeReference(IndividualKind(), 'ty', env) + ty = ForwardTypeReference(IndividualKind, 'ty', env) def test_resolve_supertype(self) -> None: self.assertEqual( diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index a94c56c..91f6d6a 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -35,7 +35,7 @@ if TYPE_CHECKING: import concat.astutils from concat.orderedset import InsertionOrderedSet - from concat.typecheck.types import Type, _Variable + from concat.typecheck.types import Type, Variable class Environment(Dict[str, 'Type']): @@ -52,7 +52,7 @@ def copy(self) -> 'Environment': def apply_substitution(self, sub: 'Substitutions') -> 'Environment': return Environment({name: sub(t) for name, t in self.items()}) - def free_type_variables(self) -> 'InsertionOrderedSet[_Variable]': + def free_type_variables(self) -> 'InsertionOrderedSet[Variable]': return free_type_variables_of_mapping(self) @@ -64,11 +64,11 @@ def apply_substitution(self, sub: 'Substitutions') -> _Result: pass -class Substitutions(Mapping['_Variable', 'Type']): +class Substitutions(Mapping['Variable', 'Type']): def __init__( self, sub: Union[ - Iterable[Tuple['_Variable', 'Type']], Mapping['_Variable', 'Type'] + Iterable[Tuple['Variable', 'Type']], Mapping['Variable', 'Type'] ] = {}, ) -> None: self._sub = dict(sub) @@ -78,7 +78,7 @@ def __init__( # raise TypeError( # f'{variable} is being substituted by {ty}, which has the wrong kind ({variable.kind} vs {ty.kind})' # ) - self._cache: Dict[int, Type] = {} + self._cache: Dict[int, 'Type'] = {} # innermost first self.subtyping_provenance: List[Any] = [] @@ -87,10 +87,10 @@ def add_subtyping_provenance( ) -> None: self.subtyping_provenance.append(subtyping_query) - def __getitem__(self, var: '_Variable') -> 'Type': + def __getitem__(self, var: 'Variable') -> 'Type': return self._sub[var] - def __iter__(self) -> Iterator['_Variable']: + def __iter__(self) -> Iterator['Variable']: return iter(self._sub) def __len__(self) -> int: @@ -118,7 +118,7 @@ def __call__(self, arg: _Substitutable[_Result]) -> _Result: result = arg.apply_substitution(self) return result - def _dom(self) -> Set['_Variable']: + def _dom(self) -> Set['Variable']: return {*self} def __str__(self) -> str: @@ -154,7 +154,7 @@ def __hash__(self) -> int: GenericTypeKind, IndividualKind, IndividualType, - IndividualVariable, + ItemVariable, Kind, ObjectType, QuotationType, @@ -162,18 +162,16 @@ def __hash__(self) -> int: SequenceVariable, StackEffect, StackItemType, - Type, TypeSequence, free_type_variables_of_mapping, get_int_type, get_list_type, get_object_type, get_str_type, + ItemKind, module_type, no_return_type, py_function_type, - subscriptable_type, - subtractable_type, tuple_type, ) import abc @@ -270,11 +268,11 @@ def infer( for node in e: if isinstance(node, concat.parse.ClassdefStatementNode): type_name = node.class_name - kind: Kind = IndividualKind() + kind: Kind = IndividualKind type_parameters = [] parameter_kinds: Sequence[Kind] if node.is_variadic: - parameter_kinds = [SequenceKind()] + parameter_kinds = [SequenceKind] else: for param in node.type_parameters: if isinstance(param, TypeNode): @@ -375,7 +373,7 @@ def infer( elif isinstance(node, concat.parse.ListWordNode): phi = S collected_type = o - element_type: IndividualType = no_return_type + element_type: 'Type' = no_return_type for item in node.list_children: phi1, fun_type, _ = infer( phi(gamma), @@ -389,7 +387,7 @@ def infer( # FIXME: Infer the type of elements in the list based on # ALL the elements. if element_type == no_return_type: - assert collected_type[-1] != SequenceKind() + assert collected_type[-1] != SequenceKind element_type = collected_type[-1] # drop the top of the stack to use as the item collected_type = collected_type[:-1] @@ -494,7 +492,6 @@ def infer( ) elif isinstance(node, concat.parse.FuncdefStatementNode): S = current_subs - f = current_effect name = node.name # NOTE: To continue the "bidirectional" bent, we will require a ghg # type annotation. @@ -654,7 +651,7 @@ def infer( elif not check_bodies and isinstance( node, concat.parse.ClassdefStatementNode ): - type_parameters: List[_Variable] = [] + type_parameters: List[Variable] = [] temp_gamma = gamma.copy() if node.is_variadic: type_parameters.append(SequenceVariable()) @@ -665,7 +662,7 @@ def infer( param, temp_gamma = param_node.to_type(temp_gamma) type_parameters.append(param) - kind: Kind = IndividualKind() + kind: Kind = IndividualKind if type_parameters: kind = GenericTypeKind([v.kind for v in type_parameters]) self_type = BoundVariable(kind) @@ -765,9 +762,10 @@ def _check_stub( class TypeNode(concat.parse.Node, abc.ABC): def __init__(self, location: concat.astutils.Location) -> None: self.location = location + self.children: Sequence[concat.parse.Node] @abc.abstractmethod - def to_type(self, env: Environment) -> Tuple[Type, Environment]: + def to_type(self, env: Environment) -> Tuple['Type', Environment]: pass @@ -777,23 +775,24 @@ def __init__(self, location: concat.astutils.Location) -> None: super().__init__(location) @abc.abstractmethod - def to_type(self, env: Environment) -> Tuple[IndividualType, Environment]: + def to_type(self, env: Environment) -> Tuple['Type', Environment]: pass # A dataclass is not used here because making this a subclass of an abstract # class does not work without overriding __init__ even when it's a dataclass. class NamedTypeNode(TypeNode): - def __init__(self, location: tuple, name: str) -> None: + def __init__(self, location: concat.astutils.Location, name: str) -> None: super().__init__(location) self.name = name + self.children = [] def __repr__(self) -> str: return '{}({!r}, {!r})'.format( type(self).__qualname__, self.location, self.name ) - def to_type(self, env: Environment) -> Tuple[Type, Environment]: + def to_type(self, env: Environment) -> Tuple['Type', Environment]: type = env.get(self.name, None) if type is None: raise NameError(self.name, self.location) @@ -819,12 +818,15 @@ class _GenericTypeNode(IndividualTypeNode): def __init__( self, location: concat.astutils.Location, + end_location: concat.astutils.Location, generic_type: IndividualTypeNode, type_arguments: Sequence[IndividualTypeNode], ) -> None: super().__init__(location) self._generic_type = generic_type self._type_arguments = type_arguments + self.end_location = end_location + self.children = [generic_type] + list(type_arguments) def to_type(self, env: Environment) -> Tuple[IndividualType, Environment]: args = [] @@ -848,6 +850,7 @@ def __init__( super().__init__(location) self._name = None if args[0] is None else args[0].value self._type = args[1] + self.children = [n for n in args if isinstance(n, TypeNode)] # QUESTION: Should I have a separate space for the temporary associated names? def to_type(self, env: Environment) -> Tuple[IndividualType, Environment]: @@ -891,6 +894,8 @@ def __init__( super().__init__(location or (-1, -1)) self._sequence_variable = seq_var self._individual_type_items = tuple(individual_type_items) + self.children = [] if seq_var is None else [seq_var] + self.children.extend(self._individual_type_items) def to_type(self, env: Environment) -> Tuple[TypeSequence, Environment]: sequence: List[StackItemType] = [] @@ -932,6 +937,7 @@ def __init__( self.input = [(i.name, i.type) for i in input.individual_type_items] self.output_sequence_variable = output.sequence_variable self.output = [(o.name, o.type) for o in output.individual_type_items] + self.children = [input, output] def __repr__(self) -> str: return '{}({!r}, {!r}, {!r}, {!r}, location={!r})'.format( @@ -987,30 +993,29 @@ def to_type(self, env: Environment) -> Tuple[StackEffect, Environment]: ) -class _IndividualVariableNode(IndividualTypeNode): - """The AST type for individual type variables.""" +class _ItemVariableNode(TypeNode): + """The AST type for item type variables.""" def __init__(self, name: Token) -> None: super().__init__(name.start) self._name = name.value + self.children = [] - def to_type( - self, env: Environment - ) -> Tuple[IndividualVariable, Environment]: + def to_type(self, env: Environment) -> Tuple['Variable', Environment]: # QUESTION: Should callers be expected to have already introduced the # name into the context? if self._name in env: ty = env[self._name] - if not isinstance(ty, IndividualVariable): + if not (ty.kind <= ItemKind): error = TypeError( - f'{self._name} is not an individual type variable' + f'{self._name} is not an item type variable (has kind {ty.kind})' ) error.location = self.location raise error return ty, env env = env.copy() - var = IndividualVariable() + var = BoundVariable(ItemKind) env[self._name] = var return var, env @@ -1025,6 +1030,7 @@ class _SequenceVariableNode(TypeNode): def __init__(self, name: Token) -> None: super().__init__(name.start) self._name = name.value + self.children = [] def to_type( self, env: Environment @@ -1058,15 +1064,16 @@ def __init__( self, location: 'concat.astutils.Location', type_variables: Sequence[ - Union[_IndividualVariableNode, _SequenceVariableNode] + Union[_ItemVariableNode, _SequenceVariableNode] ], ty: TypeNode, ) -> None: super().__init__(location) self._type_variables = type_variables self._type = ty + self.children = list(type_variables) + [ty] - def to_type(self, env: Environment) -> Tuple[Type, Environment]: + def to_type(self, env: Environment) -> Tuple['Type', Environment]: temp_env = env.copy() variables = [] for var in self._type_variables: @@ -1127,7 +1134,7 @@ def possibly_nullary_generic_type_parser() -> Generator: end_location = (yield concat.parse.token('RSQB')).end return _GenericTypeNode( type_constructor_name.location, - # end_location, + end_location, type_constructor_name, type_arguments, ) @@ -1138,7 +1145,7 @@ def individual_type_variable_parser() -> Generator: yield concat.parse.token('BACKTICK') name = yield non_star_name_parser - return _IndividualVariableNode(name) + return _ItemVariableNode(name) @concat.parser_combinators.generate def sequence_type_variable_parser() -> Generator: @@ -1347,10 +1354,10 @@ def _ensure_type( if obj_name and obj_name in known_stack_item_names: type = cast(StackItemType, known_stack_item_names[obj_name]) elif typename is None: - # NOTE: This could lead type varibles in the output of a function that - # are unconstrained. In other words, it would basically become an Any - # type. - type = IndividualVariable() + # NOTE: This could lead type to variables in the output of a function + # that are unconstrained. In other words, it would basically become an + # Any type. + type = ItemVariable(IndividualKind) elif isinstance(typename, TypeNode): type, env = cast( Tuple[StackItemType, Environment], typename.to_type(env), diff --git a/concat/typecheck/preamble.cati b/concat/typecheck/preamble.cati index beefbbc..34b3bde 100644 --- a/concat/typecheck/preamble.cati +++ b/concat/typecheck/preamble.cati @@ -19,13 +19,13 @@ def to_list(*rest_var i:iterable[`a_var] -- *rest_var l:list[`a_var]): def py_call(*rest_var kwargs:iterable[object] args:iterable[object] f:py_function[*seq_var, `a_var] -- *rest_var res:`a_var): () -def nip(*rest_var a:object b:`a_var -- *rest_var b:`a_var): +def nip(*rest_var a:`_ b:`a_var -- *rest_var b:`a_var): () -def nip_2(*rest_var a:object b:object c:`a_var -- *rest_var c:`a_var): +def nip_2(*rest_var a:`_1 b:`_2 c:`a_var -- *rest_var c:`a_var): () -def drop(*rest_var a:object -- *rest_var): +def drop(*rest_var a:`_ -- *rest_var): () def open(*rest_var kwargs:dict[str, object] path:str -- *rest_var f:file): diff --git a/concat/typecheck/preamble_types.py b/concat/typecheck/preamble_types.py index 0db16c5..cfb61fb 100644 --- a/concat/typecheck/preamble_types.py +++ b/concat/typecheck/preamble_types.py @@ -1,6 +1,7 @@ from concat.typecheck.types import ( + BoundVariable, GenericType, - IndividualVariable, + ItemKind, ObjectType, SequenceVariable, StackEffect, @@ -29,10 +30,9 @@ _seq_var = SequenceVariable() _stack_var = SequenceVariable() _stack_type_var = SequenceVariable() -_a_var = IndividualVariable() -_b_var = IndividualVariable() -_c_var = IndividualVariable() -_x = IndividualVariable() +_a_var = BoundVariable(ItemKind) +_b_var = BoundVariable(ItemKind) +_c_var = BoundVariable(ItemKind) types = { 'addable': addable_type, diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index be788b6..b7f5048 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -1,3 +1,4 @@ +import abc from concat.orderedset import InsertionOrderedSet import concat.typecheck from concat.typecheck.errors import ( @@ -6,6 +7,7 @@ StaticAnalysisError, TypeError as ConcatTypeError, ) +import functools from typing import ( AbstractSet, Any, @@ -24,8 +26,6 @@ cast, overload, ) -from typing_extensions import Self -import abc if TYPE_CHECKING: @@ -63,10 +63,9 @@ class Type(abc.ABC): def __init__(self) -> None: self._free_type_variables_cached: Optional[ - InsertionOrderedSet[_Variable] + InsertionOrderedSet[Variable] ] = None self._internal_name: Optional[str] = None - self._forward_references_resolved = False self._type_id = Type._next_type_id Type._next_type_id += 1 @@ -87,7 +86,7 @@ def __eq__(self, other: object) -> bool: if not isinstance(other, Type): return NotImplemented # QUESTION: Define == separately from is_subtype_of? - return self.is_subtype_of(other) and other.is_subtype_of(self) + return self.is_subtype_of(other) and other.is_subtype_of(self) # type: ignore # NOTE: Avoid hashing types. I might I'm having correctness issues related # to hashing that I'd rather avoid entirely. Maybe one day I'll introduce @@ -112,10 +111,10 @@ def attributes(self) -> Mapping[str, 'Type']: return {} @abc.abstractmethod - def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: + def _free_type_variables(self) -> InsertionOrderedSet['Variable']: pass - def free_type_variables(self) -> InsertionOrderedSet['_Variable']: + def free_type_variables(self) -> InsertionOrderedSet['Variable']: if self._free_type_variables_cached is None: # Break circular references. Recusring into the same type won't add # new FTVs, so we can pretend there are none we finish finding the @@ -134,7 +133,7 @@ def apply_substitution( def constrain_and_bind_variables( self, supertype: 'Type', - rigid_variables: AbstractSet['_Variable'], + rigid_variables: AbstractSet['Variable'], subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': raise NotImplementedError @@ -152,11 +151,6 @@ def constrain(self, supertype: 'Type') -> None: def instantiate(self) -> 'Type': return self - @abc.abstractmethod - def resolve_forward_references(self) -> 'Type': - self._forward_references_resolved = True - return self - @abc.abstractproperty def kind(self) -> 'Kind': pass @@ -181,7 +175,7 @@ def instantiate(self) -> 'IndividualType': @property def kind(self) -> 'Kind': - return IndividualKind() + return IndividualKind @property def attributes(self) -> Mapping[str, Type]: @@ -221,19 +215,14 @@ def __str__(self) -> str: def __repr__(self) -> str: return f'StuckTypeApplication({self._head!r}, {self._args!r})' - def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: + def _free_type_variables(self) -> InsertionOrderedSet['Variable']: ftv = self._head.free_type_variables() for arg in self._args: ftv |= arg.free_type_variables() return ftv - def resolve_forward_references(self) -> Type: - head = self._head.resolve_forward_references() - args = [arg.resolve_forward_references() for arg in self._args] - return head[args] - -class _Variable(Type, abc.ABC): +class Variable(Type, abc.ABC): """Objects that represent type variables. Every type variable object is assumed to be unique. Thus, fresh type @@ -248,7 +237,7 @@ def apply_substitution( return result # type: ignore return self - def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: + def _free_type_variables(self) -> InsertionOrderedSet['Variable']: return InsertionOrderedSet([self]) def __lt__(self, other) -> bool: @@ -262,9 +251,6 @@ def __gt__(self, other) -> bool: def __eq__(self, other) -> bool: return self is other - def resolve_forward_references(self) -> '_Variable': - return self - # NOTE: This hash impl is kept because sets of variables are fine and # variables are simple. def __hash__(self) -> int: @@ -275,8 +261,12 @@ def __hash__(self) -> int: return hash(id(self)) + @abc.abstractmethod + def freshen(self) -> 'Variable': + pass + -class BoundVariable(_Variable): +class BoundVariable(Variable): def __init__(self, kind: 'Kind') -> None: super().__init__() self._kind = kind @@ -288,11 +278,22 @@ def kind(self) -> 'Kind': def constrain_and_bind_variables( self, supertype, rigid_variables, subtyping_assumptions ) -> 'Substitutions': - raise TypeError('Cannot constrain bound variables') + raise ConcatTypeError( + f'Cannot constrain bound variable {self} to {supertype}' + ) def __getitem__(self, args: 'TypeArguments') -> Type: - assert isinstance(self.kind, GenericTypeKind) - assert list(self.kind.parameter_kinds) == [t.kind for t in args] + if not isinstance(self.kind, GenericTypeKind): + raise ConcatTypeError(f'{self} is not a generic type') + if len(self.kind.parameter_kinds) != len(args): + raise ConcatTypeError( + f'{self} was given {len(args)} arguments but expected {len(self.kind.parameter_kinds)}' + ) + for a, p in zip(args, self.kind.parameter_kinds): + if not (a.kind <= p): + raise ConcatTypeError( + f'{a} has kind {a.kind} but expected kind {p}' + ) return StuckTypeApplication(self, args) def __repr__(self) -> str: @@ -305,32 +306,41 @@ def __str__(self) -> str: def attributes(self) -> Mapping[str, Type]: raise TypeError('Cannot get attributes of bound variables') + def freshen(self) -> 'Variable': + if self._kind <= ItemKind: + return ItemVariable(self._kind) + return SequenceVariable() + + +class ItemVariable(Variable): + def __init__(self, kind: 'Kind') -> None: + super().__init__() + self._kind = kind -class IndividualVariable(_Variable, IndividualType): def constrain_and_bind_variables( self, supertype: Type, - rigid_variables: AbstractSet['_Variable'], + rigid_variables: AbstractSet['Variable'], subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': from concat.typecheck import Substitutions - if self is supertype or supertype is get_object_type(): + if ( + self is supertype + or self.kind == IndividualKind + and supertype is get_object_type() + ): return Substitutions() - if supertype.kind != IndividualKind(): - raise ConcatTypeError( - '{} must be an individual type: expected {}'.format( - supertype, self - ) - ) - mapping: Mapping[_Variable, Type] if ( - isinstance(supertype, IndividualVariable) + self.kind <= supertype.kind and supertype not in rigid_variables + and isinstance(supertype, Variable) + ): + return Substitutions([(supertype, self)]) + mapping: Mapping[Variable, Type] + if self.kind is IndividualKind and isinstance( + supertype, _OptionalType ): - mapping = {supertype: self} - return Substitutions(mapping) - if isinstance(supertype, _OptionalType): try: return self.constrain_and_bind_variables( supertype.type_arguments[0], @@ -345,34 +355,34 @@ def constrain_and_bind_variables( raise ConcatTypeError( f'{self} is considered fixed here and cannot become a subtype of {supertype}' ) - mapping = {self: supertype} - return Substitutions(mapping) + if self.kind >= supertype.kind: + mapping = {self: supertype} + return Substitutions(mapping) + raise ConcatTypeError( + f'{self} has kind {self.kind}, but {supertype} has kind {supertype.kind}' + ) def __str__(self) -> str: - return '`t_{}'.format(id(self)) + return 't_{}'.format(id(self)) def __repr__(self) -> str: - return ''.format(id(self)) - - def apply_substitution( - self, sub: 'concat.typecheck.Substitutions' - ) -> IndividualType: - return cast(IndividualType, super().apply_substitution(sub)) + return ''.format(id(self)) @property def attributes(self) -> NoReturn: raise ConcatTypeError( - '{} is an individual type variable, so its attributes are unknown'.format( - self - ) + f'{self} is an item type variable, so its attributes are unknown' ) @property def kind(self) -> 'Kind': - return IndividualKind() + return self._kind + def freshen(self) -> 'ItemVariable': + return ItemVariable(self._kind) -class SequenceVariable(_Variable): + +class SequenceVariable(Variable): def __init__(self) -> None: super().__init__() @@ -385,7 +395,7 @@ def __repr__(self) -> str: def constrain_and_bind_variables( self, supertype: Type, - rigid_variables: AbstractSet['_Variable'], + rigid_variables: AbstractSet['Variable'], subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': from concat.typecheck import Substitutions @@ -426,20 +436,23 @@ def attributes(self) -> NoReturn: @property def kind(self) -> 'Kind': - return SequenceKind() + return SequenceKind + + def freshen(self) -> 'SequenceVariable': + return SequenceVariable() class GenericType(Type): def __init__( self, - type_parameters: Sequence['_Variable'], + type_parameters: Sequence['Variable'], body: Type, is_variadic: bool = False, ) -> None: super().__init__() assert type_parameters self._type_parameters = type_parameters - if body.kind != IndividualKind(): + if body.kind != IndividualKind: raise ConcatTypeError( f'Cannot be polymorphic over non-individual type {body}' ) @@ -470,7 +483,9 @@ def __getitem__(self, type_arguments: 'TypeArguments') -> 'Type': if self.is_variadic: type_arguments = [TypeSequence(type_arguments)] actual_kinds = [ty.kind for ty in type_arguments] - if expected_kinds != actual_kinds: + if len(expected_kinds) != len(actual_kinds) or not ( + expected_kinds >= actual_kinds + ): raise ConcatTypeError( f'A type argument to {self} has the wrong kind, type arguments: {type_arguments}, expected kinds: {expected_kinds}' ) @@ -490,20 +505,16 @@ def kind(self) -> 'Kind': kinds = [var.kind for var in self._type_parameters] return GenericTypeKind(kinds) - def resolve_forward_references(self) -> 'GenericType': - body = self._body.resolve_forward_references() - return GenericType(self._type_parameters, body, self.is_variadic) - def instantiate(self) -> Type: - fresh_vars: Sequence[_Variable] = [ - type(var)() for var in self._type_parameters + fresh_vars: Sequence[Variable] = [ + var.freshen() for var in self._type_parameters ] return self[fresh_vars] def constrain_and_bind_variables( self, supertype: 'Type', - rigid_variables: AbstractSet['_Variable'], + rigid_variables: AbstractSet['Variable'], subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': from concat.typecheck import Substitutions @@ -512,39 +523,30 @@ def constrain_and_bind_variables( subtyping_assumptions, self, supertype ): return Substitutions() - # HACK: KIND_POLY I should use kind polymorphism instead to get the - # continuation example to typecheck. - if supertype._type_id == get_object_type()._type_id: - return Substitutions() + # EXCEPTION: I should be able to instantiate a generic type when + # required. + if supertype.kind is IndividualKind: + return self.instantiate().constrain_and_bind_variables( + supertype, rigid_variables, subtyping_assumptions + ) if ( - isinstance(supertype, IndividualVariable) + isinstance(supertype, ItemVariable) + and supertype.kind >= self.kind and supertype not in rigid_variables ): return Substitutions([(supertype, self)]) - if supertype.kind == IndividualKind(): - return self.instantiate().constrain_and_bind_variables( - supertype, rigid_variables, subtyping_assumptions - ) - if self.kind != supertype.kind: - # HACK: KIND_POLY - if isinstance(supertype.kind, GenericTypeKind): - return self.instantiate().constrain_and_bind_variables( - supertype.instantiate(), - rigid_variables, - subtyping_assumptions, - ) + if not (self.kind <= supertype.kind): raise ConcatTypeError( f'{self} has kind {self.kind} but {supertype} has kind {supertype.kind}' ) - shared_vars = [type(var)() for var in self._type_parameters] + shared_vars = [var.freshen() for var in self._type_parameters] self_instance = self[shared_vars] supertype_instance = supertype[shared_vars] rigid_variables = ( rigid_variables - # QUESTION: The parameters have been substituted already. Does this - # make sense? Maybe the shared_vars should be rigid. - | set(self._type_parameters) - | set(supertype._type_parameters) + # The parameters have been substituted already. The shared_vars + # should be rigid. + | set(shared_vars) ) return self_instance.constrain_and_bind_variables( supertype_instance, rigid_variables, subtyping_assumptions @@ -573,7 +575,7 @@ def attributes(self) -> NoReturn: 'Generic types do not have attributes; maybe you forgot type arguments?' ) - def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: + def _free_type_variables(self) -> InsertionOrderedSet['Variable']: return self._body.free_type_variables() - set(self._type_parameters) @@ -588,7 +590,7 @@ def __init__(self, sequence: Sequence[Type]) -> None: self._rest = None self._individual_types = sequence for ty in self._individual_types: - if ty.kind == SequenceKind(): + if ty.kind == SequenceKind: raise ConcatTypeError(f'{ty} cannot be a sequence type') def as_sequence(self) -> Sequence[Type]: @@ -612,7 +614,7 @@ def apply_substitution(self, sub) -> 'TypeSequence': def constrain_and_bind_variables( self, supertype: Type, - rigid_variables: AbstractSet['_Variable'], + rigid_variables: AbstractSet['Variable'], subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': """Check that self is a subtype of supertype. @@ -729,8 +731,8 @@ def constrain_and_bind_variables( f'{self} is a sequence type, not {supertype}' ) - def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: - ftv: InsertionOrderedSet[_Variable] = InsertionOrderedSet([]) + def _free_type_variables(self) -> InsertionOrderedSet['Variable']: + ftv: InsertionOrderedSet[Variable] = InsertionOrderedSet([]) for t in self: ftv |= t.free_type_variables() return ftv @@ -769,16 +771,9 @@ def __repr__(self) -> str: def __iter__(self) -> Iterator[Type]: return iter(self.as_sequence()) - def resolve_forward_references(self) -> 'TypeSequence': - individual_types = [ - t.resolve_forward_references() for t in self._individual_types - ] - rest = [] if self._rest is None else [self._rest] - return TypeSequence(rest + individual_types) - @property def kind(self) -> 'Kind': - return SequenceKind() + return SequenceKind # TODO: Rename to StackEffect at all use sites. @@ -802,7 +797,7 @@ def generalized_wrt(self, gamma: 'Environment') -> Type: def constrain_and_bind_variables( self, supertype: Type, - rigid_variables: AbstractSet['_Variable'], + rigid_variables: AbstractSet['Variable'], subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': from concat.typecheck import Substitutions @@ -815,7 +810,8 @@ def constrain_and_bind_variables( return Substitutions() if ( - isinstance(supertype, IndividualVariable) + isinstance(supertype, ItemVariable) + and supertype.kind is IndividualKind and supertype not in rigid_variables ): return Substitutions([(supertype, self)]) @@ -838,7 +834,7 @@ def constrain_and_bind_variables( )(sub) return sub - def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: + def _free_type_variables(self) -> InsertionOrderedSet['Variable']: return ( self.input.free_type_variables() | self.output.free_type_variables() @@ -887,11 +883,6 @@ def apply_substitution( def bind(self) -> '_Function': return _Function(self.input[:-1], self.output) - def resolve_forward_references(self) -> 'StackEffect': - input = self.input.resolve_forward_references() - output = self.output.resolve_forward_references() - return StackEffect(input, output) - class QuotationType(_Function): def __init__(self, fun_type: _Function) -> None: @@ -900,7 +891,7 @@ def __init__(self, fun_type: _Function) -> None: def constrain_and_bind_variables( self, supertype: Type, - rigid_variables: AbstractSet['_Variable'], + rigid_variables: AbstractSet['Variable'], subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': if ( @@ -909,8 +900,8 @@ def constrain_and_bind_variables( ): # FIXME: Don't present new variables every time. # FIXME: Account for the types of the elements of the quotation. - in_var = IndividualVariable() - out_var = IndividualVariable() + in_var = ItemVariable(IndividualKind) + out_var = ItemVariable(IndividualKind) quotation_iterable_type = iterable_type[ StackEffect(TypeSequence([in_var]), TypeSequence([out_var])), ] @@ -932,8 +923,8 @@ def apply_substitution( def free_type_variables_of_mapping( attributes: Mapping[str, Type] -) -> InsertionOrderedSet[_Variable]: - ftv: InsertionOrderedSet[_Variable] = InsertionOrderedSet([]) +) -> InsertionOrderedSet[Variable]: + ftv: InsertionOrderedSet[Variable] = InsertionOrderedSet([]) for sigma in attributes.values(): ftv |= sigma.free_type_variables() return ftv @@ -947,7 +938,10 @@ def _contains_assumption( assumptions: Sequence[Tuple[Type, Type]], subtype: Type, supertype: Type ) -> bool: for sub, sup in assumptions: - if sub is subtype and sup is supertype: + if ( + sub._type_id == subtype._type_id + and sup._type_id == supertype._type_id + ): return True return False @@ -970,7 +964,7 @@ def __init__( self._attributes = attributes for t in nominal_supertypes: - if t.kind != IndividualKind(): + if t.kind != IndividualKind: raise ConcatTypeError( f'{t} must be an individual type, but has kind {t.kind}' ) @@ -987,27 +981,13 @@ def __init__( def nominal(self) -> bool: return self._nominal - def resolve_forward_references(self) -> 'ObjectType': - attributes = { - attr: t.resolve_forward_references() - for attr, t in self._attributes.items() - } - nominal_supertypes = [ - t.resolve_forward_references() for t in self._nominal_supertypes - ] - return ObjectType( - attributes, nominal_supertypes, self.nominal, self._head - ) - @property def kind(self) -> 'Kind': - return IndividualKind() + return IndividualKind def apply_substitution( self, sub: 'concat.typecheck.Substitutions', ) -> 'ObjectType': - from concat.typecheck import Substitutions - # if no free type vars will be substituted, just return self if not any(free_var in sub for free_var in self.free_type_variables()): return self @@ -1034,7 +1014,7 @@ def apply_substitution( def constrain_and_bind_variables( self, supertype: Type, - rigid_variables: AbstractSet['_Variable'], + rigid_variables: AbstractSet['Variable'], subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': from concat.typecheck import Substitutions @@ -1049,7 +1029,8 @@ def constrain_and_bind_variables( # obj <: `t, `t is not rigid # --> `t = obj if ( - isinstance(supertype, IndividualVariable) + isinstance(supertype, Variable) + and supertype.kind >= IndividualKind and supertype not in rigid_variables ): sub = Substitutions([(supertype, self)]) @@ -1064,7 +1045,7 @@ def constrain_and_bind_variables( ) ) - if self.kind != supertype.kind: + if not (self.kind <= supertype.kind): raise ConcatTypeError( f'{self} has kind {self.kind}, but {supertype} has kind {supertype.kind}' ) @@ -1119,7 +1100,7 @@ def constrain_and_bind_variables( sub.add_subtyping_provenance((self, supertype)) return sub if not isinstance(supertype, ObjectType): - raise NotImplementedError(supertype) + raise NotImplementedError(repr(supertype)) # every object type is a subtype of object_type if supertype._type_id == get_object_type()._type_id: sub = Substitutions() @@ -1155,7 +1136,7 @@ def __repr__(self) -> str: head = None if self._head is self else self._head return f'{type(self).__qualname__}(attributes={self._attributes!r}, nominal_supertypes={self._nominal_supertypes!r}, nominal={self._nominal!r}, _head={head!r})' - def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: + def _free_type_variables(self) -> InsertionOrderedSet[Variable]: ftv = free_type_variables_of_mapping(self.attributes) # QUESTION: Include supertypes? return ftv @@ -1211,7 +1192,7 @@ class PythonFunctionType(IndividualType): def __init__( self, _overloads: Sequence[Tuple[Type, Type]] = (), - type_parameters: Sequence[_Variable] = (), + type_parameters: Sequence[Variable] = (), _type_arguments: Sequence[Type] = (), ) -> None: super().__init__() @@ -1230,43 +1211,39 @@ def __init__( ) if self._arity == 0: i, o = _type_arguments - if i.kind != SequenceKind(): + if i.kind != SequenceKind: raise ConcatTypeError( f'{i} must be a sequence type, but has kind {i.kind}' ) # HACK: Sequence variables are introduced by the type sequence AST nodes - if ( - isinstance(i, TypeSequence) - and i - and i[0].kind == SequenceKind() - ): + if isinstance(i, TypeSequence) and i and i[0].kind == SequenceKind: i = TypeSequence(i.as_sequence()[1:]) _type_arguments = i, o - if o.kind != IndividualKind(): + if not (o.kind <= ItemKind): raise ConcatTypeError( - f'{o} must be an individual type, but has kind {o.kind}' + f'{o} must be an item type, but has kind {o.kind}' ) _fixed_overloads: List[Tuple[Type, Type]] = [] for i, o in _overloads: - if i.kind != SequenceKind(): + if i.kind != SequenceKind: raise ConcatTypeError( f'{i} must be a sequence type, but has kind {i.kind}' ) if ( isinstance(i, TypeSequence) and i - and i[0].kind == SequenceKind() + and i[0].kind == SequenceKind ): i = TypeSequence(i.as_sequence()[1:]) - if o.kind != IndividualKind(): + if not (o.kind <= ItemKind): raise ConcatTypeError( - f'{o} must be an individual type, but has kind {o.kind}' + f'{o} must be an item type, but has kind {o.kind}' ) _fixed_overloads.append((i, o)) self._overloads = _fixed_overloads self._type_arguments = _type_arguments - def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: + def _free_type_variables(self) -> InsertionOrderedSet[Variable]: if self._arity == 0: ftv = self.input.free_type_variables() ftv |= self.output.free_type_variables() @@ -1277,37 +1254,8 @@ def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: @property def kind(self) -> 'Kind': if self._arity == 0: - return IndividualKind() - return GenericTypeKind([SequenceKind(), IndividualKind()]) - - def resolve_forward_references(self) -> 'PythonFunctionType': - if self._arity == 2: - return self - overloads: List[Tuple[Type, Type]] = [] - for args, ret in overloads: - overloads.append( - ( - args.resolve_forward_references(), - ret.resolve_forward_references(), - ) - ) - type_arguments = list( - t.resolve_forward_references() for t in self._type_arguments - ) - return PythonFunctionType( - _overloads=overloads, - type_parameters=[], - _type_arguments=type_arguments, - ) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, PythonFunctionType): - return False - if self.kind != other.kind: - return False - if isinstance(self.kind, GenericTypeKind): - return True - return self.input == other.input and self.output == other.output + return IndividualKind + return GenericTypeKind([SequenceKind, IndividualKind]) def __repr__(self) -> str: # QUESTION: Is it worth using type(self)? @@ -1333,13 +1281,13 @@ def __getitem__( ) input = arguments[0] output = arguments[1] - if input.kind != SequenceKind(): + if input.kind != SequenceKind: raise ConcatTypeError( f'First argument to {self} must be a sequence type of function arguments' ) - if output.kind != IndividualKind(): + if not (output.kind <= ItemKind): raise ConcatTypeError( - f'Second argument to {self} must be an individual type for the return type' + f'Second argument to {self} (the return type) must be an item type' ) return PythonFunctionType( _type_arguments=(input, output), type_parameters=(), _overloads=[], @@ -1406,7 +1354,7 @@ def bind(self) -> 'PythonFunctionType': def constrain_and_bind_variables( self, supertype: Type, - rigid_variables: AbstractSet['_Variable'], + rigid_variables: AbstractSet['Variable'], subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': from concat.typecheck import Substitutions @@ -1421,13 +1369,14 @@ def constrain_and_bind_variables( raise ConcatTypeError( f'{self} has kind {self.kind} but {supertype} has kind {supertype.kind}' ) - if self.kind == IndividualKind(): + if self.kind == IndividualKind: if supertype is get_object_type(): sub = Substitutions() sub.add_subtyping_provenance((self, supertype)) return sub if ( - isinstance(supertype, IndividualVariable) + isinstance(supertype, ItemVariable) + and supertype.kind is IndividualKind and supertype not in rigid_variables ): sub = Substitutions([(supertype, self)]) @@ -1522,7 +1471,7 @@ def __getitem__(self, args: Sequence[Type]) -> 'PythonFunctionType': def attributes(self) -> Mapping[str, 'Type']: raise ConcatTypeError('py_overloaded does not have attributes') - def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: + def _free_type_variables(self) -> InsertionOrderedSet['Variable']: return InsertionOrderedSet([]) def apply_substitution( @@ -1538,17 +1487,14 @@ def instantiate(self) -> PythonFunctionType: def constrain_and_bind_variables( self, supertype: 'Type', - rigid_variables: AbstractSet['_Variable'], + rigid_variables: AbstractSet['Variable'], subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': raise ConcatTypeError('py_overloaded is a generic type') - def resolve_forward_references(self) -> '_PythonOverloadedType': - return self - @property def kind(self) -> 'Kind': - return GenericTypeKind([SequenceKind()]) + return GenericTypeKind([SequenceKind]) def __eq__(self, other: object) -> bool: return isinstance(other, type(self)) @@ -1573,19 +1519,16 @@ def apply_substitution( def __repr__(self) -> str: return '{}()'.format(type(self).__qualname__) - def _free_type_variables(self) -> InsertionOrderedSet['_Variable']: + def _free_type_variables(self) -> InsertionOrderedSet['Variable']: return InsertionOrderedSet([]) - def resolve_forward_references(self) -> Self: - return self - class _OptionalType(IndividualType): def __init__(self, type_argument: Type) -> None: super().__init__() - if type_argument.kind != IndividualKind(): + if not (type_argument.kind <= ItemKind): raise ConcatTypeError( - f'{type_argument} must be an individual type, but has kind {type_argument.kind}' + f'{type_argument} must be an item type, but has kind {type_argument.kind}' ) while isinstance(type_argument, _OptionalType): type_argument = type_argument._type_argument @@ -1597,7 +1540,7 @@ def __repr__(self) -> str: def __str__(self) -> str: return f'optional_type[{self._type_argument}]' - def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: + def _free_type_variables(self) -> InsertionOrderedSet[Variable]: return self._type_argument.free_type_variables() def __eq__(self, other: object) -> bool: @@ -1639,10 +1582,6 @@ def constrain_and_bind_variables( ) return sub - def resolve_forward_references(self) -> '_OptionalType': - type_argument = self._type_argument.resolve_forward_references() - return _OptionalType(type_argument) - def apply_substitution( self, sub: 'concat.typecheck.Substitutions' ) -> '_OptionalType': @@ -1653,36 +1592,98 @@ def type_arguments(self) -> Sequence[Type]: return [self._type_argument] +@functools.total_ordering class Kind(abc.ABC): @abc.abstractmethod def __eq__(self, other: object) -> bool: pass + @abc.abstractmethod + def __lt__(self, other: 'Kind') -> bool: + pass + + +class _ItemKind(Kind): + __instance: Optional['_ItemKind'] = None + + def __new__(cls) -> '_ItemKind': + if cls.__instance is None: + cls.__instance = super().__new__(cls) + return cls.__instance -class IndividualKind(Kind): def __eq__(self, other: object) -> bool: - return isinstance(other, IndividualKind) + return self is other + + def __lt__(self, other: Kind) -> bool: + return False -class SequenceKind(Kind): +ItemKind = _ItemKind() + + +class _IndividualKind(Kind): + __instance: Optional['_IndividualKind'] = None + + def __new__(cls) -> '_IndividualKind': + if cls.__instance is None: + cls.__instance = super().__new__(cls) + return cls.__instance + def __eq__(self, other: object) -> bool: - return isinstance(other, SequenceKind) + return self is other + + def __lt__(self, other: Kind) -> bool: + return other is ItemKind + + +IndividualKind = _IndividualKind() + + +class _SequenceKind(Kind): + __instance: Optional['_SequenceKind'] = None + + def __new__(cls) -> '_SequenceKind': + if cls.__instance is None: + cls.__instance = super().__new__(cls) + return cls.__instance + + def __eq__(self, other: object) -> bool: + return self is other + + def __lt__(self, other: Kind) -> bool: + return other is ItemKind + + +SequenceKind = _SequenceKind() class GenericTypeKind(Kind): def __init__(self, parameter_kinds: Sequence[Kind]) -> None: - assert parameter_kinds + if not parameter_kinds: + raise ConcatTypeError( + 'Generic type kinds cannot have empty parameters' + ) self.parameter_kinds = parameter_kinds def __eq__(self, other: object) -> bool: - return ( - isinstance(other, GenericTypeKind) - and self.parameter_kinds == other.parameter_kinds - ) + return isinstance(other, GenericTypeKind) and list( + self.parameter_kinds + ) == list(other.parameter_kinds) + + def __lt__(self, other: Kind) -> bool: + if not isinstance(other, Kind): + return NotImplemented + if other is ItemKind: + return True + if not isinstance(other, GenericTypeKind): + return False + if len(self.parameter_kinds) != len(other.parameter_kinds): + return False + return list(self.parameter_kinds) > list(other.parameter_kinds) class Fix(Type): - def __init__(self, var: _Variable, body: Type) -> None: + def __init__(self, var: Variable, body: Type) -> None: super().__init__() assert var.kind == body.kind self._var = var @@ -1717,7 +1718,7 @@ def unroll(self) -> Type: self._unrolled_ty._type_id = self._type_id return self._unrolled_ty - def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: + def _free_type_variables(self) -> InsertionOrderedSet[Variable]: return self._body.free_type_variables() - {self._var} def apply_substitution(self, sub: 'Substitutions') -> Type: @@ -1740,7 +1741,7 @@ def constrain_and_bind_variables( ) -> 'Substitutions': from concat.typecheck import Substitutions - if supertype is get_object_type() or _contains_assumption( + if supertype._type_id == get_object_type()._type_id or _contains_assumption( subtyping_assumptions, self, supertype ): sub = Substitutions() @@ -1749,6 +1750,8 @@ def constrain_and_bind_variables( if isinstance(supertype, Fix): unrolled = supertype.unroll() + # BUG: The unrolled types have the same type ids, so the assumption + # is used immediately, which is unsound. sub = self.unroll().constrain_and_bind_variables( unrolled, rigid_variables, @@ -1769,10 +1772,6 @@ def constrain_and_bind_variables( def kind(self) -> Kind: return self._var.kind - def resolve_forward_references(self) -> Type: - body = self._body.resolve_forward_references() - return Fix(self._var, body) - def __getitem__(self, args: Any) -> Any: return self.unroll()[args] @@ -1821,7 +1820,7 @@ def __getitem__(self, args: TypeArguments) -> Type: f'Type argument has kind {arg.kind}, expected kind {kind}' ) return ForwardTypeReference( - IndividualKind(), + IndividualKind, self._name_to_resolve, self._resolution_env, _type_arguments=args, @@ -1853,7 +1852,7 @@ def attributes(self) -> Mapping[str, Type]: def constrain_and_bind_variables( self, supertype: Type, - rigid_variables: AbstractSet['_Variable'], + rigid_variables: AbstractSet['Variable'], subtyping_assumptions: List[Tuple['Type', 'Type']], ) -> 'Substitutions': if self is supertype or _contains_assumption( @@ -1867,7 +1866,7 @@ def constrain_and_bind_variables( subtyping_assumptions + [(self, supertype)], ) - def _free_type_variables(self) -> InsertionOrderedSet[_Variable]: + def _free_type_variables(self) -> InsertionOrderedSet[Variable]: return InsertionOrderedSet([]) @property @@ -1892,7 +1891,7 @@ def _mapping_to_str(mapping: Mapping) -> str: # expose _Function as StackEffect StackEffect = _Function -_x = BoundVariable(kind=IndividualKind()) +_x = BoundVariable(kind=IndividualKind) float_type = ObjectType({}, nominal=True) no_return_type = _NoReturnType() @@ -1951,13 +1950,13 @@ def set_int_type(ty: Type) -> None: _arg_type_var = SequenceVariable() -_return_type_var = IndividualVariable() +_return_type_var = ItemVariable(IndividualKind) py_function_type = PythonFunctionType( type_parameters=[_arg_type_var, _return_type_var] ) py_function_type.set_internal_name('py_function_type') -_invert_result_var = IndividualVariable() +_invert_result_var = ItemVariable(IndividualKind) invertible_type = GenericType( [_invert_result_var], ObjectType( @@ -1965,8 +1964,8 @@ def set_int_type(ty: Type) -> None: ), ) -_sub_operand_type = IndividualVariable() -_sub_result_type = IndividualVariable() +_sub_operand_type = BoundVariable(ItemKind) +_sub_result_type = BoundVariable(ItemKind) # FIXME: Add reverse_substractable_type for __rsub__ subtractable_type = GenericType( [_sub_operand_type, _sub_result_type], @@ -1980,8 +1979,8 @@ def set_int_type(ty: Type) -> None: ) subtractable_type.set_internal_name('subtractable_type') -_add_other_operand_type = IndividualVariable() -_add_result_type = IndividualVariable() +_add_other_operand_type = BoundVariable(ItemKind) +_add_result_type = BoundVariable(ItemKind) addable_type = GenericType( [_add_other_operand_type, _add_result_type], @@ -2002,7 +2001,7 @@ def set_int_type(ty: Type) -> None: # QUESTION: Allow comparison methods to return any object? -_other_type = IndividualVariable() +_other_type = BoundVariable(ItemKind) geq_comparable_type = GenericType( [_other_type], ObjectType( @@ -2030,7 +2029,7 @@ def set_int_type(ty: Type) -> None: none_type = ObjectType({}, nominal=True) none_type.set_internal_name('none_type') -_result_type = IndividualVariable() +_result_type = BoundVariable(ItemKind) iterator_type = GenericType( [_result_type], @@ -2070,32 +2069,35 @@ def set_int_type(ty: Type) -> None: ) context_manager_type.set_internal_name('context_manager_type') -_optional_type_var = IndividualVariable() +_optional_type_var = BoundVariable(ItemKind) optional_type = GenericType( [_optional_type_var], _OptionalType(_optional_type_var) ) optional_type.set_internal_name('optional_type') -_key_type_var = IndividualVariable() -_value_type_var = IndividualVariable() -dict_type = ObjectType( - { - '__iter__': py_function_type[ - TypeSequence([]), iterator_type[_key_type_var,] - ] - }, +_key_type_var = BoundVariable(kind=IndividualKind) +_value_type_var = BoundVariable(kind=IndividualKind) +dict_type = GenericType( [_key_type_var, _value_type_var], - nominal=True, + ObjectType( + { + '__iter__': py_function_type[ + TypeSequence([]), iterator_type[_key_type_var,] + ] + }, + nominal=True, + ), ) dict_type.set_internal_name('dict_type') _start_type_var, _stop_type_var, _step_type_var = ( - IndividualVariable(), - IndividualVariable(), - IndividualVariable(), + BoundVariable(ItemKind), + BoundVariable(ItemKind), + BoundVariable(ItemKind), ) -slice_type = ObjectType( - {}, [_start_type_var, _stop_type_var, _step_type_var], nominal=True +slice_type = GenericType( + [_start_type_var, _stop_type_var, _step_type_var], + ObjectType({}, nominal=True), ) slice_type.set_internal_name('slice_type') @@ -2117,19 +2119,22 @@ def set_int_type(ty: Type) -> None: base_exception_type = ObjectType({}, nominal=True) module_type = ObjectType({}, nominal=True) -_index_type_var = IndividualVariable() -_result_type_var = IndividualVariable() -subscriptable_type = ObjectType( - { - '__getitem__': py_function_type[ - TypeSequence([_index_type_var]), _result_type_var - ], - }, +_index_type_var = BoundVariable(ItemKind) +_result_type_var = BoundVariable(ItemKind) +subscriptable_type = GenericType( [_index_type_var, _result_type_var], + ObjectType( + { + '__getitem__': py_function_type[ + TypeSequence([_index_type_var]), _result_type_var + ], + }, + ), ) -_answer_type_var = IndividualVariable() -continuation_monad_type = ObjectType( - {}, [_result_type_var, _answer_type_var], nominal=True +_answer_type_var = BoundVariable(ItemKind) +continuation_monad_type = GenericType( + [_result_type_var, _answer_type_var], + ObjectType(attributes={}, nominal=True,), ) continuation_monad_type.set_internal_name('continuation_monad_type') From 1d317860a488a8751376eea78e56b0350ad08c4f Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Fri, 19 Jul 2024 20:44:48 -0700 Subject: [PATCH 34/61] Provide user-friendly __str__ for kinds --- concat/typecheck/types.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index b7f5048..302f015 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -1602,6 +1602,10 @@ def __eq__(self, other: object) -> bool: def __lt__(self, other: 'Kind') -> bool: pass + @abc.abstractmethod + def __str__(self) -> str: + pass + class _ItemKind(Kind): __instance: Optional['_ItemKind'] = None @@ -1617,6 +1621,9 @@ def __eq__(self, other: object) -> bool: def __lt__(self, other: Kind) -> bool: return False + def __str__(self) -> str: + return 'Item' + ItemKind = _ItemKind() @@ -1635,6 +1642,9 @@ def __eq__(self, other: object) -> bool: def __lt__(self, other: Kind) -> bool: return other is ItemKind + def __str__(self) -> str: + return 'Individual' + IndividualKind = _IndividualKind() @@ -1653,6 +1663,9 @@ def __eq__(self, other: object) -> bool: def __lt__(self, other: Kind) -> bool: return other is ItemKind + def __str__(self) -> str: + return 'Sequence' + SequenceKind = _SequenceKind() @@ -1681,6 +1694,9 @@ def __lt__(self, other: Kind) -> bool: return False return list(self.parameter_kinds) > list(other.parameter_kinds) + def __str__(self) -> str: + return f'Generic[{", ".join(map(str, self.parameter_kinds))}]' + class Fix(Type): def __init__(self, var: Variable, body: Type) -> None: From 7b3e02b1f2b5351e74f5b969faa45e739ff71ea7 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Sat, 20 Jul 2024 01:24:35 -0700 Subject: [PATCH 35/61] Implement subsumption between polytypes using regeneralization Also, generic types can now have a type of any kind as the body. --- concat/poly-subsumption.md | 55 ++++++++ concat/tests/typecheck/test_types.py | 18 +++ concat/typecheck/__init__.py | 6 +- concat/typecheck/types.py | 188 ++++++++++++++++----------- 4 files changed, 189 insertions(+), 78 deletions(-) create mode 100644 concat/poly-subsumption.md diff --git a/concat/poly-subsumption.md b/concat/poly-subsumption.md new file mode 100644 index 0000000..6894b24 --- /dev/null +++ b/concat/poly-subsumption.md @@ -0,0 +1,55 @@ +Subsumption of a polymorphic type by another type may involve instantiation and +reordering of type variables. Subsumption can be checked using +"regeneralization." + +I think a correct(-ish?) way of doing regeneralization in Concat is the +following (substitution notation might be backwards): + +``` + t[fresh(a+)/a+] <: s with b+ rigid + b+ all not in ftv(forall a+. t) +------------------------------------------------------ [FORALL<:FORALL] + forall a+. t <: forall b+. s +``` + +In the `HMV_InstG` rule of [Visible Type Application (Extended +version)](https://www.seas.upenn.edu/~sweirich/papers/type-app-extended.pdf), in +Fig. 4, there's a condition that the bs are not free in `forall a.... t`. + +`FORALL<:FORALL` means that `forall a. int <: forall a, b. int`. But `forall (a : +Individual). a /<: forall (b : Item). b` because we would have to solve +`fresh(a) <: b`, which requires the substitution `[b/a]` because the kind of +`b` >= the kind of `a`. But we made `b` rigid! + +``` + (s can be generic, but not a forall) + forall a+, b*. t : Generic[k+, l*] + s : Generic[m*] + l* :> m* + forall b*. t[fresh(a+)/a+] <: s +---------------------------------------- [FORALL<:INST] + forall a+, b*. t <: s + + (b is a unification variable) + forall a+. t : Generic[k+] + b : l + Generic[k+] <: l +-------------------------------------- [FORALL<:VAR] +forall a+. t <: b --> [b/forall a+. t] +``` + +The following might not make sense: + +``` +(s can be generic, but not a forall) + forall a+. t : Generic[k+] + s : l + Generic[k+] <: l + b+ = fresh(a+) +t[b+/a+] <: s[b+] (type application) +------------------------------------ [FORALL<:KIND-SUB] + forall a+. t <: s +``` + +Other papers mentioning regeneralization: +- [Guarded impredicative polymorphism](https://www.microsoft.com/en-us/research/uploads/prod/2017/07/impred-pldi18-submission.pdf) diff --git a/concat/tests/typecheck/test_types.py b/concat/tests/typecheck/test_types.py index fdaf577..d75f68b 100644 --- a/concat/tests/typecheck/test_types.py +++ b/concat/tests/typecheck/test_types.py @@ -8,7 +8,9 @@ BoundVariable, Fix, ForwardTypeReference, + GenericType, IndividualKind, + ItemKind, ItemVariable, ObjectType, SequenceVariable, @@ -180,3 +182,19 @@ def test_constrain_empty(self) -> None: def test_empty_equal(self) -> None: self.assertEqual(TypeSequence([]), TypeSequence([])) + + +class TestGeneric(unittest.TestCase): + def test_generalize(self) -> None: + a, b = BoundVariable(ItemKind), BoundVariable(ItemKind) + subtype = GenericType([a], get_int_type()) + supertype = GenericType([a, b], get_int_type()) + subtype.constrain_and_bind_variables(supertype, set(), []) + + def test_parameter_kinds(self) -> None: + ind = BoundVariable(IndividualKind) + item = BoundVariable(ItemKind) + subtype = GenericType([ind], ind) + supertype = GenericType([item], item) + with self.assertRaises(ConcatTypeError): + subtype.constrain_and_bind_variables(supertype, set(), []) diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index 91f6d6a..e4fdd0c 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -283,7 +283,7 @@ def infer( parameter_kinds = [ variable.kind for variable in type_parameters ] - kind = GenericTypeKind(parameter_kinds) + kind = GenericTypeKind(parameter_kinds, IndividualKind) gamma[type_name] = ForwardTypeReference(kind, type_name, gamma) for node in e: @@ -664,7 +664,9 @@ def infer( kind: Kind = IndividualKind if type_parameters: - kind = GenericTypeKind([v.kind for v in type_parameters]) + kind = GenericTypeKind( + [v.kind for v in type_parameters], IndividualKind + ) self_type = BoundVariable(kind) temp_gamma[node.class_name] = self_type _, _, body_attrs = infer( diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 302f015..91ec734 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -452,10 +452,6 @@ def __init__( super().__init__() assert type_parameters self._type_parameters = type_parameters - if body.kind != IndividualKind: - raise ConcatTypeError( - f'Cannot be polymorphic over non-individual type {body}' - ) self._body = body self._instantiations: Dict[Tuple[int, ...], Type] = {} self.is_variadic = is_variadic @@ -501,9 +497,9 @@ def __getitem__(self, type_arguments: 'TypeArguments') -> 'Type': return instance @property - def kind(self) -> 'Kind': + def kind(self) -> 'GenericTypeKind': kinds = [var.kind for var in self._type_parameters] - return GenericTypeKind(kinds) + return GenericTypeKind(kinds, self._body.kind) def instantiate(self) -> Type: fresh_vars: Sequence[Variable] = [ @@ -523,33 +519,67 @@ def constrain_and_bind_variables( subtyping_assumptions, self, supertype ): return Substitutions() - # EXCEPTION: I should be able to instantiate a generic type when - # required. - if supertype.kind is IndividualKind: - return self.instantiate().constrain_and_bind_variables( - supertype, rigid_variables, subtyping_assumptions - ) + # NOTE: Here, we implement subsumption of polytypes, so the kinds don't + # need to be the same. See concat/poly-subsumption.md for more + # information. if ( - isinstance(supertype, ItemVariable) - and supertype.kind >= self.kind + isinstance(supertype, Variable) and supertype not in rigid_variables + and self.kind <= supertype.kind ): return Substitutions([(supertype, self)]) - if not (self.kind <= supertype.kind): + if not isinstance(supertype, GenericType): + supertype_parameter_kinds: Sequence[Kind] + if isinstance(supertype.kind, GenericTypeKind): + supertype_parameter_kinds = supertype.kind.parameter_kinds + elif self.kind.result_kind <= supertype.kind: + supertype_parameter_kinds = [] + else: + raise ConcatTypeError( + f'{self} has kind {self.kind} but {supertype} has kind {supertype.kind}' + ) + params_to_inst = len(self.kind.parameter_kinds) - len( + supertype_parameter_kinds + ) + param_kinds_left = self.kind.parameter_kinds[ + -len(supertype_parameter_kinds) : + ] + if params_to_inst < 0 or not ( + param_kinds_left >= supertype_parameter_kinds + ): + raise ConcatTypeError( + f'{self} has kind {self.kind} but {supertype} has kind {supertype.kind}' + ) + sub = Substitutions( + [ + (t, t.freshen()) + for t in self._type_parameters[:params_to_inst] + ] + ) + parameters_left = self._type_parameters[params_to_inst:] + inst: Type + if parameters_left: + inst = GenericType(parameters_left, sub(self._body)) + else: + inst = sub(self._body) + return inst.constrain_and_bind_variables( + supertype, rigid_variables, subtyping_assumptions + ) + # supertype is a GenericType + # QUESTION: Should I care about is_variadic? + if any( + map( + lambda t: t in self.free_type_variables(), + supertype._type_parameters, + ) + ): raise ConcatTypeError( - f'{self} has kind {self.kind} but {supertype} has kind {supertype.kind}' + f'Type parameters {supertype._type_parameters} cannot appear free in {self}' ) - shared_vars = [var.freshen() for var in self._type_parameters] - self_instance = self[shared_vars] - supertype_instance = supertype[shared_vars] - rigid_variables = ( - rigid_variables - # The parameters have been substituted already. The shared_vars - # should be rigid. - | set(shared_vars) - ) - return self_instance.constrain_and_bind_variables( - supertype_instance, rigid_variables, subtyping_assumptions + return self.instantiate().constrain_and_bind_variables( + supertype._body, + rigid_variables | set(supertype._type_parameters), + subtyping_assumptions, ) def apply_substitution(self, sub: 'Substitutions') -> 'GenericType': @@ -657,7 +687,7 @@ def constrain_and_bind_variables( # error else: raise StackMismatchError(self, supertype) - elif not self._individual_types: + if not self._individual_types: # *a <: [], *a is not rigid # --> *a = [] if supertype._is_empty() and self._rest not in rigid_variables: @@ -683,49 +713,47 @@ def constrain_and_bind_variables( sub = Substitutions([(self._rest, supertype)]) sub.add_subtyping_provenance((self, supertype)) return sub - else: - raise StackMismatchError(self, supertype) - else: - # *a? `t... `t_n <: [] - # error - if supertype._is_empty(): - raise StackMismatchError(self, supertype) - # *a? `t... `t_n <: *b, *b is not rigid, *b is not free in LHS - # --> *b = LHS - elif ( - not supertype._individual_types - and supertype._rest - and supertype._rest not in self.free_type_variables() - and supertype._rest not in rigid_variables - ): - sub = Substitutions([(supertype._rest, self)]) - sub.add_subtyping_provenance((self, supertype)) - return sub - # `t_n <: `s_m *a? `t... <: *b? `s... - # --- - # *a? `t... `t_n <: *b? `s... `s_m - elif supertype._individual_types: - sub = self._individual_types[ - -1 - ].constrain_and_bind_variables( - supertype._individual_types[-1], + # else: + # print(rigid_variables) + # raise StackMismatchError(self, supertype) + # *a? `t... `t_n <: [] + # error + if supertype._is_empty(): + raise StackMismatchError(self, supertype) + # *a? `t... `t_n <: *b, *b is not rigid, *b is not free in LHS + # --> *b = LHS + elif ( + not supertype._individual_types + and supertype._rest + and supertype._rest not in self.free_type_variables() + and supertype._rest not in rigid_variables + ): + sub = Substitutions([(supertype._rest, self)]) + sub.add_subtyping_provenance((self, supertype)) + return sub + # `t_n <: `s_m *a? `t... <: *b? `s... + # --- + # *a? `t... `t_n <: *b? `s... `s_m + elif supertype._individual_types: + sub = self._individual_types[-1].constrain_and_bind_variables( + supertype._individual_types[-1], + rigid_variables, + subtyping_assumptions, + ) + try: + sub = sub(self[:-1]).constrain_and_bind_variables( + sub(supertype[:-1]), rigid_variables, subtyping_assumptions, - ) - try: - sub = sub(self[:-1]).constrain_and_bind_variables( - sub(supertype[:-1]), - rigid_variables, - subtyping_assumptions, - )(sub) - # sub.add_subtyping_provenance((self, supertype)) - return sub - except StackMismatchError: - # TODO: Add info about occurs check and rigid - # variables. - raise StackMismatchError(self, supertype) - else: + )(sub) + return sub + except StackMismatchError: + # TODO: Add info about occurs check and rigid + # variables. raise StackMismatchError(self, supertype) + else: + raise StackMismatchError(self, supertype) + raise StackMismatchError(self, supertype) else: raise ConcatTypeError( f'{self} is a sequence type, not {supertype}' @@ -1255,7 +1283,7 @@ def _free_type_variables(self) -> InsertionOrderedSet[Variable]: def kind(self) -> 'Kind': if self._arity == 0: return IndividualKind - return GenericTypeKind([SequenceKind, IndividualKind]) + return GenericTypeKind([SequenceKind, IndividualKind], IndividualKind) def __repr__(self) -> str: # QUESTION: Is it worth using type(self)? @@ -1494,7 +1522,7 @@ def constrain_and_bind_variables( @property def kind(self) -> 'Kind': - return GenericTypeKind([SequenceKind]) + return GenericTypeKind([SequenceKind], IndividualKind) def __eq__(self, other: object) -> bool: return isinstance(other, type(self)) @@ -1671,17 +1699,22 @@ def __str__(self) -> str: class GenericTypeKind(Kind): - def __init__(self, parameter_kinds: Sequence[Kind]) -> None: + def __init__( + self, parameter_kinds: Sequence[Kind], result_kind: Kind + ) -> None: if not parameter_kinds: raise ConcatTypeError( 'Generic type kinds cannot have empty parameters' ) self.parameter_kinds = parameter_kinds + self.result_kind = result_kind def __eq__(self, other: object) -> bool: - return isinstance(other, GenericTypeKind) and list( - self.parameter_kinds - ) == list(other.parameter_kinds) + return ( + isinstance(other, GenericTypeKind) + and list(self.parameter_kinds) == list(other.parameter_kinds) + and self.result_kind == other.result_kind + ) def __lt__(self, other: Kind) -> bool: if not isinstance(other, Kind): @@ -1692,10 +1725,13 @@ def __lt__(self, other: Kind) -> bool: return False if len(self.parameter_kinds) != len(other.parameter_kinds): return False - return list(self.parameter_kinds) > list(other.parameter_kinds) + return ( + list(self.parameter_kinds) > list(other.parameter_kinds) + and self.result_kind < other.result_kind + ) def __str__(self) -> str: - return f'Generic[{", ".join(map(str, self.parameter_kinds))}]' + return f'Generic[{", ".join(map(str, self.parameter_kinds))}, {self.result_kind}]' class Fix(Type): From b56be5cd559f418ee0d73b2f5ea49578ae0888f2 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Sat, 20 Jul 2024 17:36:06 -0700 Subject: [PATCH 36/61] Fail to constrain a bound variable only when it has to be substituted --- concat/typecheck/types.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 91ec734..18f3ac0 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -278,6 +278,19 @@ def kind(self) -> 'Kind': def constrain_and_bind_variables( self, supertype, rigid_variables, subtyping_assumptions ) -> 'Substitutions': + from concat.typecheck import Substitutions + + if ( + supertype._type_id == get_object_type()._type_id + or (self, supertype) in subtyping_assumptions + ): + return Substitutions() + if ( + isinstance(supertype, Variable) + and self.kind <= supertype.kind + and supertype not in rigid_variables + ): + return Substitutions([(supertype, self)]) raise ConcatTypeError( f'Cannot constrain bound variable {self} to {supertype}' ) From 97eb347ff954f541d3db632cea1e2ac9b37f8184 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Sat, 20 Jul 2024 17:47:09 -0700 Subject: [PATCH 37/61] Fix function definition transpiler test --- concat/tests/test_transpile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/concat/tests/test_transpile.py b/concat/tests/test_transpile.py index 79aca29..d90f071 100644 --- a/concat/tests/test_transpile.py +++ b/concat/tests/test_transpile.py @@ -1,12 +1,11 @@ import concat.visitors -from concat.astutils import get_explicit_positional_function_parameters from concat.lex import Token import concat.parse import concat.transpile from concat.typecheck import StackEffectTypeNode, TypeSequenceNode import unittest import ast -from typing import Iterable, Iterator, List, Sequence, Type, cast +from typing import Iterable, Iterator, Type import astunparse # type: ignore @@ -56,6 +55,7 @@ def test_funcdef_statement_visitor(self) -> None: [], [], [], + [], (0, 0), StackEffectTypeNode( (0, 0), From 8755dd607e3b4e3bc2e132a49383579970955eb2 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Sat, 20 Jul 2024 21:40:05 -0700 Subject: [PATCH 38/61] Fix some of the subtyping tests --- concat/tests/test_typecheck.py | 20 ++++--- concat/typecheck/types.py | 98 +++++++++++++++++----------------- 2 files changed, 64 insertions(+), 54 deletions(-) diff --git a/concat/tests/test_typecheck.py b/concat/tests/test_typecheck.py index a4fcf4f..5150f8f 100644 --- a/concat/tests/test_typecheck.py +++ b/concat/tests/test_typecheck.py @@ -9,7 +9,6 @@ Fix, IndividualKind, IndividualType, - IndividualKind, ItemVariable, ObjectType, StackEffect, @@ -279,7 +278,7 @@ def test_builtin_name_does_not_exist_in_empty_environment(self) -> None: def test_name_does_exist(self, named_type_node, type) -> None: env = concat.typecheck.Environment({named_type_node.name: type}) expected_type = named_type_node.to_type(env)[0] - note((expected_type, type)) + note(str((expected_type, type))) self.assertEqual(named_type_node.to_type(env)[0], type) @@ -309,7 +308,9 @@ def test_reflexive_equality(self, type): class TestSubtyping(unittest.TestCase): def test_int_not_subtype_of_float(self) -> None: """Differ from Reticulated Python: !(int <= float).""" - self.assertFalse(get_int_type().is_subtype_of(float_type)) + ex = get_int_type().is_subtype_of(float_type) + print(ex) + self.assertFalse(ex) @given(from_type(IndividualType), from_type(IndividualType)) @settings(suppress_health_check=(HealthCheck.filter_too_much,)) @@ -328,7 +329,10 @@ def test_no_return_is_bottom_type(self, type) -> None: @given(from_type(IndividualType)) @settings(suppress_health_check=(HealthCheck.filter_too_much,)) def test_object_is_top_type(self, type) -> None: - self.assertTrue(type.is_subtype_of(get_object_type())) + ex = type.is_subtype_of(get_object_type()) + note(repr(type)) + note(str(ex)) + self.assertTrue(ex) __attributes_generator = dictionaries( text(max_size=25), from_type(IndividualType), max_size=5 # type: ignore @@ -355,7 +359,11 @@ def test_class_structural_subtyping( ) -> None: object1 = ClassType({**other_attributes, **attributes}) object2 = ClassType(attributes) - self.assertTrue(object1.is_subtype_of(object2)) + note(repr(object1)) + note(repr(object2)) + ex = object1.is_subtype_of(object2) + note(str(ex)) + self.assertTrue(ex) @given(from_type(StackEffect)) @settings(suppress_health_check=(HealthCheck.filter_too_much,)) @@ -377,7 +385,7 @@ def test_object_subtype_of_py_function(self, type1, type2) -> None: @given(from_type(StackEffect)) def test_class_subtype_of_stack_effect(self, effect) -> None: - x = BoundVariable(kind=IndividualKind()) + x = BoundVariable(kind=IndividualKind) # NOTE: self-last convention is modelled after Factor. unbound_effect = StackEffect( TypeSequence([*effect.input, x]), effect.output diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 18f3ac0..973ab39 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -340,14 +340,15 @@ def constrain_and_bind_variables( if ( self is supertype - or self.kind == IndividualKind - and supertype is get_object_type() + # QUESTION: subsumption of polytypes? + or self.kind is IndividualKind + and supertype._type_id is get_object_type()._type_id ): return Substitutions() if ( - self.kind <= supertype.kind + isinstance(supertype, Variable) + and self.kind <= supertype.kind and supertype not in rigid_variables - and isinstance(supertype, Variable) ): return Substitutions([(supertype, self)]) mapping: Mapping[Variable, Type] @@ -726,9 +727,6 @@ def constrain_and_bind_variables( sub = Substitutions([(self._rest, supertype)]) sub.add_subtyping_provenance((self, supertype)) return sub - # else: - # print(rigid_variables) - # raise StackMismatchError(self, supertype) # *a? `t... `t_n <: [] # error if supertype._is_empty(): @@ -1406,11 +1404,11 @@ def constrain_and_bind_variables( sub = Substitutions() sub.add_subtyping_provenance((self, supertype)) return sub - if self.kind != supertype.kind: + if not (self.kind <= supertype.kind): raise ConcatTypeError( f'{self} has kind {self.kind} but {supertype} has kind {supertype.kind}' ) - if self.kind == IndividualKind: + if self.kind is IndividualKind: if supertype is get_object_type(): sub = Substitutions() sub.add_subtyping_provenance((self, supertype)) @@ -1445,45 +1443,49 @@ def constrain_and_bind_variables( ) sub.add_subtyping_provenance((self, supertype)) return sub - if isinstance(supertype, PythonFunctionType): - if isinstance(self.kind, GenericTypeKind): - sub = Substitutions() - sub.add_subtyping_provenance((self, supertype)) - return sub - - # No need to extend the rigid variables, we know both types have no - # parameters at this point. - - # Support overloading the subtype. - exceptions = [] - for overload in [ - (self.input, self.output), - *self._overloads, - ]: - try: - subtyping_assumptions_copy = subtyping_assumptions[:] - self_input_types = overload[0] - supertype_input_types = supertype.input - sub = supertype_input_types.constrain_and_bind_variables( - self_input_types, - rigid_variables, - subtyping_assumptions_copy, + if isinstance(supertype, PythonFunctionType): + # No need to extend the rigid variables, we know both types have no + # parameters at this point. + + # Support overloading the subtype. + exceptions = [] + for overload in [ + (self.input, self.output), + *self._overloads, + ]: + try: + subtyping_assumptions_copy = subtyping_assumptions[:] + self_input_types = overload[0] + supertype_input_types = supertype.input + sub = supertype_input_types.constrain_and_bind_variables( + self_input_types, + rigid_variables, + subtyping_assumptions_copy, + ) + sub = sub(self.output).constrain_and_bind_variables( + sub(supertype.output), + rigid_variables, + subtyping_assumptions_copy, + )(sub) + sub.add_subtyping_provenance((self, supertype)) + return sub + except ConcatTypeError as e: + exceptions.append(e) + finally: + subtyping_assumptions[:] = subtyping_assumptions_copy + raise ConcatTypeError( + 'no overload of {} is a subtype of {}'.format( + self, supertype ) - sub = sub(self.output).constrain_and_bind_variables( - sub(supertype.output), - rigid_variables, - subtyping_assumptions_copy, - )(sub) - sub.add_subtyping_provenance((self, supertype)) - return sub - except ConcatTypeError as e: - exceptions.append(e) - finally: - subtyping_assumptions[:] = subtyping_assumptions_copy - - raise ConcatTypeError( - 'no overload of {} is a subtype of {}'.format(self, supertype) - ) from exceptions[0] + ) from exceptions[0] + raise ConcatTypeError(f'{self} is not a subtype of {supertype}') + # TODO: Remove generic type responsibility from this class + if isinstance(supertype, PythonFunctionType) and isinstance( + supertype.kind, GenericTypeKind + ): + sub = Substitutions() + sub.add_subtyping_provenance((self, supertype)) + return sub class _PythonOverloadedType(Type): @@ -1780,7 +1782,7 @@ def unroll(self) -> Type: self._unrolled_ty = self._apply(self) if self._internal_name is not None: self._unrolled_ty.set_internal_name(self._internal_name) - self._unrolled_ty._type_id = self._type_id + # self._unrolled_ty._type_id = self._type_id return self._unrolled_ty def _free_type_variables(self) -> InsertionOrderedSet[Variable]: From 5bab4da5546ccbee19fd74d6a9e95591e7269c58 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Sun, 21 Jul 2024 01:04:35 -0700 Subject: [PATCH 39/61] Fix typechecker tests --- concat/tests/test_typecheck.py | 5 +++-- concat/typecheck/__init__.py | 15 +++++---------- concat/typecheck/types.py | 27 +++++++++++++++------------ 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/concat/tests/test_typecheck.py b/concat/tests/test_typecheck.py index 5150f8f..1e2ebc9 100644 --- a/concat/tests/test_typecheck.py +++ b/concat/tests/test_typecheck.py @@ -9,6 +9,7 @@ Fix, IndividualKind, IndividualType, + ItemKind, ItemVariable, ObjectType, StackEffect, @@ -192,8 +193,8 @@ def test_cast_word(self) -> None: class TestStackEffectParser(unittest.TestCase): _a_bar = concat.typecheck.SequenceVariable() _d_bar = concat.typecheck.SequenceVariable() - _b = concat.typecheck.ItemVariable(IndividualKind) - _c = concat.typecheck.ItemVariable(IndividualKind) + _b = concat.typecheck.ItemVariable(ItemKind) + _c = concat.typecheck.ItemVariable(ItemKind) examples: Dict[str, StackEffect] = { 'a b -- b a': StackEffect( TypeSequence([_a_bar, _b, _c]), TypeSequence([_a_bar, _c, _b]) diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index e4fdd0c..4949095 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -1351,19 +1351,14 @@ def _ensure_type( env: Environment, obj_name: Optional[str], known_stack_item_names: Environment, -) -> Tuple[StackItemType, Environment]: - type: StackItemType +) -> Tuple['Type', Environment]: + type: Type if obj_name and obj_name in known_stack_item_names: - type = cast(StackItemType, known_stack_item_names[obj_name]) + type = known_stack_item_names[obj_name] elif typename is None: - # NOTE: This could lead type to variables in the output of a function - # that are unconstrained. In other words, it would basically become an - # Any type. - type = ItemVariable(IndividualKind) + type = ItemVariable(ItemKind) elif isinstance(typename, TypeNode): - type, env = cast( - Tuple[StackItemType, Environment], typename.to_type(env), - ) + type, env = typename.to_type(env) else: raise NotImplementedError( 'Cannot turn {!r} into a type'.format(typename) diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 973ab39..6d26838 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -71,11 +71,16 @@ def __init__(self) -> None: # QUESTION: Do I need this? def is_subtype_of(self, supertype: 'Type') -> SubtypeExplanation: + from concat.typecheck import Substitutions + try: sub = self.constrain_and_bind_variables(supertype, set(), []) except ConcatTypeError as e: return SubtypeExplanation(e) - return SubtypeExplanation(sub) + ftv = self.free_type_variables() | supertype.free_type_variables() + sub1 = Substitutions({v: t for v, t in sub.items() if v in ftv}) + sub1.subtyping_provenance = sub.subtyping_provenance + return SubtypeExplanation(sub1) # No <= implementation using subtyping, because variables overload that for # sort by identity. @@ -281,7 +286,8 @@ def constrain_and_bind_variables( from concat.typecheck import Substitutions if ( - supertype._type_id == get_object_type()._type_id + self._type_id == supertype._type_id + or supertype._type_id == get_object_type()._type_id or (self, supertype) in subtyping_assumptions ): return Substitutions() @@ -844,13 +850,13 @@ def constrain_and_bind_variables( if ( self is supertype or _contains_assumption(subtyping_assumptions, self, supertype) - or supertype is get_object_type() + or supertype._type_id == get_object_type()._type_id ): return Substitutions() if ( isinstance(supertype, ItemVariable) - and supertype.kind is IndividualKind + and supertype.kind <= ItemKind and supertype not in rigid_variables ): return Substitutions([(supertype, self)]) @@ -1058,13 +1064,15 @@ def constrain_and_bind_variables( ) -> 'Substitutions': from concat.typecheck import Substitutions - if self is supertype or _contains_assumption( - subtyping_assumptions, self, supertype + # every object type is a subtype of object_type + if ( + self is supertype + or supertype._type_id == get_object_type()._type_id + or _contains_assumption(subtyping_assumptions, self, supertype) ): sub = Substitutions() sub.add_subtyping_provenance((self, supertype)) return sub - # obj <: `t, `t is not rigid # --> `t = obj if ( @@ -1140,11 +1148,6 @@ def constrain_and_bind_variables( return sub if not isinstance(supertype, ObjectType): raise NotImplementedError(repr(supertype)) - # every object type is a subtype of object_type - if supertype._type_id == get_object_type()._type_id: - sub = Substitutions() - sub.add_subtyping_provenance((self, supertype)) - return sub # Don't forget that there's nominal subtyping too. if supertype._nominal: if supertype in self._nominal_supertypes: From f424567766854be966a24502d83fecfdba528eb7 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Sun, 1 Sep 2024 20:33:17 -0700 Subject: [PATCH 40/61] Move test dependencies to pyproject.toml and remove tox The version of tox I have to use doesn't work on my machine, so I won't worry about it. I wasn't doing much with it anyways. --- pyproject.toml | 20 ++++++++++++++++---- tox.ini | 27 --------------------------- 2 files changed, 16 insertions(+), 31 deletions(-) delete mode 100644 tox.ini diff --git a/pyproject.toml b/pyproject.toml index e6333aa..649551e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,13 +27,21 @@ readme = "README.md" [project.optional-dependencies] dev = [ "axblack==20220330", - "hypothesis>=6.75.1,<7", + "importlib-metadata<5", "mypy>=1.1.1", "pre-commit>=2.6.0,<3", + "pylsp-mypy", + "python-lsp-server[all]==1.7.2", "snakeviz", - "tox>=4.5.1,<5", + "virtualenv<=20.17.1", +] +test = [ + "coverage>=6.4.4,<7", + "hypothesis>=6.75.1,<7", + "nose2==0.13.0", + "pywinpty>=2.0.7,<3; platform_system==\"Windows\"", + "scripttest", ] - [project.urls] "Source code" = "https://github.com/jmanuel1/concat" @@ -44,6 +52,10 @@ pattern = "version = '(?P.*)'" [tool.hatch.envs.default] features = [ - "dev" + "dev", + "test", ] python = "3.7" + +[tool.hatch.metadata] +allow-direct-references = true diff --git a/tox.ini b/tox.ini deleted file mode 100644 index d12cce6..0000000 --- a/tox.ini +++ /dev/null @@ -1,27 +0,0 @@ -[tox] -requires = - tox>=4 -env_list = py37 - -[testenv] -description = run tests with coverage -deps = - coverage>=6.4.4,<7 - hypothesis>=6.75.1,<7 - nose2==0.13.0 - pywinpty>=2.0.7,<3; platform_system=="Windows" - scripttest -commands = - coverage erase - coverage run -m nose2 -v --pretty-assert {posargs:concat.tests} - coverage combine - coverage xml - coverage lcov -passenv = - CARGO_BUILD_TARGET -setenv = - PYTHONWARNINGS = default - -[flake8] -ignore = E741, F402 -max-complexity = 15 From 5e1800bc03c9b653a5c406b2fa87cf2bf9d95fb4 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Sun, 1 Sep 2024 21:57:15 -0700 Subject: [PATCH 41/61] Remove definition in Python of bool And let methods like __lt__ return any type to untie a knot with bool --- concat/typecheck/__init__.py | 3 ++ concat/typecheck/builtin_stubs/builtins.cati | 2 + concat/typecheck/preamble.cati | 6 +-- concat/typecheck/types.py | 46 +++++++++++++++----- 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index 4949095..bd046c8 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -306,6 +306,9 @@ def infer( if pragma == 'builtin_int': name = node.args[0] concat.typecheck.types.set_int_type(gamma[name]) + if pragma == 'builtin_bool': + name = node.args[0] + concat.typecheck.types.set_bool_type(gamma[name]) elif isinstance(node, concat.parse.PushWordNode): S1, (i1, o1) = S, (i, o) child = node.children[0] diff --git a/concat/typecheck/builtin_stubs/builtins.cati b/concat/typecheck/builtin_stubs/builtins.cati index 115ea96..20c8488 100644 --- a/concat/typecheck/builtin_stubs/builtins.cati +++ b/concat/typecheck/builtin_stubs/builtins.cati @@ -6,6 +6,8 @@ class object: class bool: () +!@@concat.typecheck.builtin_bool bool + class int: def __add__(--) @cast (py_function[(int), int]): () diff --git a/concat/typecheck/preamble.cati b/concat/typecheck/preamble.cati index 34b3bde..82d24b5 100644 --- a/concat/typecheck/preamble.cati +++ b/concat/typecheck/preamble.cati @@ -79,18 +79,18 @@ def loop(*rest_var body:(*rest_var -- *rest_var flag:bool) -- *rest_var): # Rule 1: first operand has __ge__(type(second operand)) # Rule 2: second operand has __le__(type(first operand)) # FIXME: Implement the second type rule -def >=(*stack_type_var a:geq_comparable[`b_var] b:`b_var -- *stack_type_var res:bool): +def >=(*stack_type_var a:geq_comparable[`b_var, `ret] b:`b_var -- *stack_type_var res:`ret): () # Rule 1: first operand has __lt__(type(second operand)) # Rule 2: second operand has __gt__(type(first operand)) # FIXME: Implement the second type rule # Also look at Python's note about when reflected method get's priority. -def <(*stack_type_var a:lt_comparable[`b_var] b:`b_var -- *stack_type_var res:bool): +def <(*stack_type_var a:lt_comparable[`b_var, `ret] b:`b_var -- *stack_type_var res:`ret): () # FIXME: Implement the second type rule -def <=(*stack_type_var a:leq_comparable[`b_var] b:`b_var -- *stack_type_var res:bool): +def <=(*stack_type_var a:leq_comparable[`b_var, `ret] b:`b_var -- *stack_type_var res:`ret): () def choose(*rest_var b:bool t:(*rest_var -- *seq_var) f:(*rest_var -- *seq_var) -- *seq_var): diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 6d26838..3b21ffb 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -2019,6 +2019,19 @@ def set_int_type(ty: Type) -> None: _int_type = ty +_bool_type: Optional[Type] = None + + +def get_bool_type() -> Type: + assert _bool_type is not None + return _bool_type + + +def set_bool_type(ty: Type) -> None: + global _bool_type + _bool_type = ty + + _arg_type_var = SequenceVariable() _return_type_var = ItemVariable(IndividualKind) py_function_type = PythonFunctionType( @@ -2066,36 +2079,49 @@ def set_int_type(ty: Type) -> None: ) addable_type.set_internal_name('addable_type') -bool_type = ObjectType({}, nominal=True) -bool_type.set_internal_name('bool_type') - -# QUESTION: Allow comparison methods to return any object? +# NOTE: Allow comparison methods to return any object. I don't think Python +# stops it. Plus, these definitions don't have to depend on bool, which is +# defined in builtins.cati. _other_type = BoundVariable(ItemKind) +_return_type = BoundVariable(ItemKind) geq_comparable_type = GenericType( - [_other_type], + [_other_type, _return_type], ObjectType( - {'__ge__': py_function_type[TypeSequence([_other_type]), bool_type]}, + { + '__ge__': py_function_type[ + TypeSequence([_other_type]), _return_type + ] + }, ), ) geq_comparable_type.set_internal_name('geq_comparable_type') leq_comparable_type = GenericType( - [_other_type], + [_other_type, _return_type], ObjectType( - {'__le__': py_function_type[TypeSequence([_other_type]), bool_type]}, + { + '__le__': py_function_type[ + TypeSequence([_other_type]), _return_type + ] + }, ), ) leq_comparable_type.set_internal_name('leq_comparable_type') lt_comparable_type = GenericType( - [_other_type], + [_other_type, _return_type], ObjectType( - {'__lt__': py_function_type[TypeSequence([_other_type]), bool_type]}, + { + '__lt__': py_function_type[ + TypeSequence([_other_type]), _return_type + ] + }, ), ) lt_comparable_type.set_internal_name('lt_comparable_type') + none_type = ObjectType({}, nominal=True) none_type.set_internal_name('none_type') From 7fc0c46ff0b07cb48ee36431b46edb5b6b55986c Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Mon, 2 Sep 2024 19:13:32 -0700 Subject: [PATCH 42/61] Move all pyinterop.__init__ types to stub --- concat/stdlib/pyinterop/__init__.cati | 3 ++ concat/stdlib/pyinterop/__init__.py | 59 --------------------------- 2 files changed, 3 insertions(+), 59 deletions(-) diff --git a/concat/stdlib/pyinterop/__init__.cati b/concat/stdlib/pyinterop/__init__.cati index 18f9bcd..6f65d92 100644 --- a/concat/stdlib/pyinterop/__init__.cati +++ b/concat/stdlib/pyinterop/__init__.cati @@ -12,3 +12,6 @@ def to_py_function(*rest_var f:(*rest_var_2 -- *rest_var_3) -- *rest_var py_f:py def to_slice(*stack_type_var step:Optional[`x] stop:Optional[`y] start:Optional[`z] -- *stack_type_var slice_:slice[`z, `y, `x]): () + +def map(*rest_var f:(*rest_var x:`y -- *rest_var fx:`z) it:iterable[`y] -- *rest_var fit:iterable[`z]): + () diff --git a/concat/stdlib/pyinterop/__init__.py b/concat/stdlib/pyinterop/__init__.py index e94c31a..a742afa 100644 --- a/concat/stdlib/pyinterop/__init__.py +++ b/concat/stdlib/pyinterop/__init__.py @@ -1,19 +1,6 @@ """Concat-Python interoperation helpers.""" from concat.common_types import ConcatFunction import concat.stdlib.ski -from concat.typecheck.types import ( - GenericType, - IndividualKind, - ItemVariable, - SequenceVariable, - StackEffect, - TypeSequence, - dict_type, - iterable_type, - optional_type, - subscriptable_type, - tuple_type, -) import builtins import importlib import os @@ -34,52 +21,6 @@ ) -_stack_type_var = SequenceVariable() -_rest_var = SequenceVariable() -_rest_var_2 = SequenceVariable() -_rest_var_3 = SequenceVariable() -_x = ItemVariable(IndividualKind) -_y = ItemVariable(IndividualKind) -_z = ItemVariable(IndividualKind) -globals()['@@types'] = { - 'getitem': GenericType( - [_stack_type_var, _x, _y], - StackEffect( - TypeSequence([_stack_type_var, subscriptable_type[_x, _y], _x,]), - TypeSequence([_stack_type_var, _y]), - ), - ), - 'map': GenericType( - [_rest_var, _y, _z], - StackEffect( - TypeSequence( - [ - _rest_var, - StackEffect( - TypeSequence([_rest_var, _y]), - TypeSequence([_rest_var, _z]), - ), - iterable_type[_y,], - ] - ), - TypeSequence([_rest_var, iterable_type[_z,]]), - ), - ), - 'to_dict': GenericType( - [_stack_type_var, _x, _y], - StackEffect( - TypeSequence( - [ - _stack_type_var, - optional_type[iterable_type[tuple_type[_x, _y],],], - ] - ), - TypeSequence([_stack_type_var, dict_type[_x, _y]]), - ), - ), -} - - def to_py_function(stack: List[object], stash: List[object]) -> None: func = cast(Callable[[List[object], List[object]], None], stack.pop()) From c411d78369ff1bbe89f844311c93b983978f8f15 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Mon, 2 Sep 2024 19:39:24 -0700 Subject: [PATCH 43/61] Make sure missing stub causes error pointing to import*ing* file --- concat/examples/continuation.cat | 1 - concat/typecheck/__init__.py | 28 +++++++++++++++------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/concat/examples/continuation.cat b/concat/examples/continuation.cat index 137593a..0bcd7d5 100644 --- a/concat/examples/continuation.cat +++ b/concat/examples/continuation.cat @@ -10,7 +10,6 @@ from concat.stdlib.continuation import map_cont from concat.stdlib.continuation import bind_cont from concat.stdlib.continuation import cont_from_cps from concat.stdlib.pyinterop import to_dict -# from concat.stdlib.execution import loop # FIXME: This line should cause an error when there's no @@types in that module def abort(k:forall *s. (*s x:`a -- *s n:none) -- n:none): diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index bd046c8..2c5f5b9 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -740,25 +740,27 @@ def _check_stub_resolved_path( for failure in recovered_parsing_failures: print('Parse Error:') print(create_parsing_failure_message(file, tokens, failure)) - return check( - env, - concat_ast.children, - str(path.parent), - _should_check_bodies=False, - _should_load_builtins=not is_builtins, - _should_load_preamble=not is_preamble and not is_builtins, - ) + try: + env = check( + env, + concat_ast.children, + str(path.parent), + _should_check_bodies=False, + _should_load_builtins=not is_builtins, + _should_load_preamble=not is_preamble and not is_builtins, + ) + except StaticAnalysisError as e: + e.set_path_if_missing(path) + raise + _module_namespaces[path] = env + return env def _check_stub( path: pathlib.Path, is_builtins: bool = False, is_preamble: bool = False ) -> 'Environment': path = path.resolve() - try: - return _check_stub_resolved_path(path, is_builtins, is_preamble) - except StaticAnalysisError as e: - e.set_path_if_missing(path) - raise + return _check_stub_resolved_path(path, is_builtins, is_preamble) # Parsing type annotations From 00a9217ceeaa9a0f5d96e2ee5434d24327876c89 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Mon, 2 Sep 2024 22:39:20 -0700 Subject: [PATCH 44/61] Clean up imports in __main__ --- concat/__main__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/concat/__main__.py b/concat/__main__.py index 679128e..26865ee 100644 --- a/concat/__main__.py +++ b/concat/__main__.py @@ -10,12 +10,10 @@ import concat.parser_combinators import concat.stdlib.repl import concat.typecheck -import io import json import os.path import sys -import textwrap -from typing import Callable, IO, AnyStr, Sequence, TextIO +from typing import Callable, IO, AnyStr filename = '' From c3b1fa3dcfc845718fdb17918f4747a7e383fed2 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Mon, 2 Sep 2024 23:42:47 -0700 Subject: [PATCH 45/61] Move as many preamble types as possible to stubs, separate nominal & structural types --- concat/stdlib/repl.py | 2 +- concat/tests/strategies.py | 5 +- concat/tests/test_typecheck.py | 21 +- concat/tests/typecheck/test_types.py | 4 +- concat/transpile.py | 5 +- concat/typecheck/__init__.py | 124 ++++---- concat/typecheck/builtin_stubs/builtins.cati | 2 + concat/typecheck/builtin_stubs/types.cati | 4 + concat/typecheck/preamble.cati | 36 +++ concat/typecheck/preamble0.cati | 4 + concat/typecheck/preamble_types.py | 107 ------- concat/typecheck/types.py | 282 ++++++++++++------- 12 files changed, 313 insertions(+), 283 deletions(-) create mode 100644 concat/typecheck/builtin_stubs/types.cati create mode 100644 concat/typecheck/preamble0.cati diff --git a/concat/stdlib/repl.py b/concat/stdlib/repl.py index e7e214d..176c1be 100644 --- a/concat/stdlib/repl.py +++ b/concat/stdlib/repl.py @@ -151,7 +151,7 @@ def show_var(stack: List[object], stash: List[object]): 'visible_vars': set(), 'show_var': show_var, 'concat': concat, - '@@extra_env': concat.typecheck.Environment(), + '@@extra_env': concat.typecheck.load_builtins_and_preamble(), **initial_globals, } locals: Dict[str, object] = {} diff --git a/concat/tests/strategies.py b/concat/tests/strategies.py index 834d747..12a6bcd 100644 --- a/concat/tests/strategies.py +++ b/concat/tests/strategies.py @@ -8,7 +8,6 @@ StackEffect, StuckTypeApplication, TypeSequence, - none_type, no_return_type, optional_type, py_function_type, @@ -47,13 +46,11 @@ def _object_type_strategy( builds( ObjectType, attributes=dictionaries(text(), individual_type_strategy), - nominal_supertypes=lists(individual_type_strategy), _head=none(), ), lambda children: builds( ObjectType, attributes=dictionaries(text(), individual_type_strategy), - nominal_supertypes=lists(individual_type_strategy), _head=children, ), max_leaves=10, @@ -100,7 +97,7 @@ def _mark_individual_type_strategy( ) | _mark_individual_type_strategy( builds(lambda args: optional_type[args], tuples(children),), - type(optional_type[none_type,]), + type(optional_type[ObjectType({}),]), ), max_leaves=50, ) diff --git a/concat/tests/test_typecheck.py b/concat/tests/test_typecheck.py index 1e2ebc9..1ed88c4 100644 --- a/concat/tests/test_typecheck.py +++ b/concat/tests/test_typecheck.py @@ -15,13 +15,11 @@ StackEffect, Type as ConcatType, TypeSequence, - ellipsis_type, float_type, get_object_type, get_int_type, no_return_type, - none_type, - not_implemented_type, + get_none_type, optional_type, py_function_type, ) @@ -117,9 +115,7 @@ def test_call_inference(self) -> None: try_prog = '$(42) call\n' tree = parse(try_prog) _, type, _ = concat.typecheck.infer( - concat.typecheck.Environment( - concat.typecheck.preamble_types.types - ), + concat.typecheck.load_builtins_and_preamble(), tree.children, is_top_level=True, ) @@ -129,17 +125,16 @@ def test_call_inference(self) -> None: @given(sampled_from(['None', '...', 'NotImplemented'])) def test_constants(self, constant_name) -> None: + env = concat.typecheck.load_builtins_and_preamble() _, effect, _ = concat.typecheck.infer( - concat.typecheck.Environment( - concat.typecheck.preamble_types.types - ), + env, [concat.parse.NameWordNode(lex.Token(value=constant_name))], initial_stack=TypeSequence([]), ) expected_types = { - 'None': none_type, - 'NotImplemented': not_implemented_type, - '...': ellipsis_type, + 'None': get_none_type(), + 'NotImplemented': env['not_implemented'], + '...': env['ellipsis'], } expected_type = expected_types[constant_name] self.assertEqual(list(effect.output), [expected_type]) @@ -413,7 +408,7 @@ def test_none_subtype_of_optional(self, ty: IndividualType) -> None: opt_ty = optional_type[ ty, ] - self.assertTrue(none_type.is_subtype_of(opt_ty)) + self.assertTrue(get_none_type().is_subtype_of(opt_ty)) @given(from_type(IndividualType)) def test_type_subtype_of_optional(self, ty: IndividualType) -> None: diff --git a/concat/tests/typecheck/test_types.py b/concat/tests/typecheck/test_types.py index d75f68b..39acbcd 100644 --- a/concat/tests/typecheck/test_types.py +++ b/concat/tests/typecheck/test_types.py @@ -21,7 +21,7 @@ get_object_type, optional_type, py_function_type, - tuple_type, + get_tuple_type, ) import unittest @@ -122,7 +122,7 @@ def test_stack_effect_input_supertype(self) -> None: class TestFix(unittest.TestCase): fix_var = BoundVariable(IndividualKind) linked_list = Fix( - fix_var, optional_type[tuple_type[get_object_type(), fix_var],], + fix_var, optional_type[get_tuple_type()[get_object_type(), fix_var],], ) def test_unroll_supertype(self) -> None: diff --git a/concat/transpile.py b/concat/transpile.py index da76138..e18308b 100644 --- a/concat/transpile.py +++ b/concat/transpile.py @@ -51,9 +51,8 @@ def parse(tokens: Sequence[Token]) -> concat.parse.TopLevelNode: def typecheck(concat_ast: concat.parse.TopLevelNode, source_dir: str) -> None: # FIXME: Consider the type of everything entered interactively beforehand. - concat.typecheck.check( - concat.typecheck.Environment(), concat_ast.children, source_dir - ) + env = concat.typecheck.load_builtins_and_preamble() + concat.typecheck.check(env, concat_ast.children, source_dir) def transpile(code: str, source_dir: str = '.') -> ast.Module: diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index 2c5f5b9..2fdbf4a 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -148,14 +148,17 @@ def __hash__(self) -> int: from concat.typecheck.types import ( BoundVariable, + Brand, Fix, ForwardTypeReference, GenericType, GenericTypeKind, IndividualKind, IndividualType, + ItemKind, ItemVariable, Kind, + NominalType, ObjectType, QuotationType, SequenceKind, @@ -168,16 +171,14 @@ def __hash__(self) -> int: get_list_type, get_object_type, get_str_type, - ItemKind, - module_type, + get_tuple_type, + get_module_type, no_return_type, py_function_type, - tuple_type, ) import abc from concat.error_reporting import create_parsing_failure_message from concat.lex import Token -import functools import importlib import importlib.util import itertools @@ -191,15 +192,18 @@ def __hash__(self) -> int: def load_builtins_and_preamble() -> Environment: - env = _check_stub(_builtins_stub_path, is_builtins=True,) - env = Environment( - { - **env, - **_check_stub( - pathlib.Path(__file__).with_name('preamble.cati'), - is_preamble=True, - ), - } + env = _check_stub( + pathlib.Path(__file__).with_name('preamble0.cati'), is_preamble=True, + ) + env = _check_stub(_builtins_stub_path, is_builtins=True, initial_env=env) + env = _check_stub( + pathlib.Path(__file__).with_name('preamble.cati'), + is_preamble=True, + initial_env=env, + ) + # pick up ModuleType + _check_stub( + _builtins_stub_path.with_name('types.cati'), initial_env=env.copy(), ) return env @@ -209,27 +213,11 @@ def check( program: concat.astutils.WordsOrStatements, source_dir: str = '.', _should_check_bodies: bool = True, - _should_load_builtins: bool = True, - _should_load_preamble: bool = True, ) -> Environment: import concat.typecheck.preamble_types - builtins_stub_env = Environment() - preamble_stub_env = Environment() - if _should_load_builtins: - builtins_stub_env = _check_stub(_builtins_stub_path, is_builtins=True,) - if _should_load_preamble: - preamble_stub_env = _check_stub( - pathlib.Path(__file__).with_name('preamble.cati'), - is_preamble=True, - ) environment = Environment( - { - **builtins_stub_env, - **preamble_stub_env, - **concat.typecheck.preamble_types.types, - **environment, - } + {**concat.typecheck.preamble_types.types, **environment,} ) res = infer( environment, @@ -239,6 +227,7 @@ def check( source_dir, check_bodies=_should_check_bodies, ) + return res[2] @@ -309,6 +298,15 @@ def infer( if pragma == 'builtin_bool': name = node.args[0] concat.typecheck.types.set_bool_type(gamma[name]) + if pragma == 'builtin_tuple': + name = node.args[0] + concat.typecheck.types.set_tuple_type(gamma[name]) + if pragma == 'builtin_none': + name = node.args[0] + concat.typecheck.types.set_none_type(gamma[name]) + if pragma == 'builtin_module': + name = node.args[0] + concat.typecheck.types.set_module_type(gamma[name]) elif isinstance(node, concat.parse.PushWordNode): S1, (i1, o1) = S, (i, o) child = node.children[0] @@ -434,7 +432,10 @@ def infer( StackEffect( i, TypeSequence( - [*collected_type, tuple_type[element_types],] + [ + *collected_type, + get_tuple_type()[element_types], + ] ), ) ), @@ -466,7 +467,9 @@ def infer( # For now, assume the module's written in Python. stub_path = pathlib.Path(module_path) stub_path = stub_path.with_suffix('.cati') - stub_env = _check_stub(stub_path) + stub_env = _check_stub( + stub_path, initial_env=load_builtins_and_preamble() + ) imported_type = stub_env.get(node.imported_name) if imported_type is None: raise TypeError( @@ -688,15 +691,15 @@ def infer( if name not in temp_gamma } ) - ty: Type = ObjectType( - attributes=body_attrs, nominal_supertypes=(), nominal=True, + ty: Type = NominalType( + Brand(node.class_name, IndividualKind, []), + ObjectType(attributes=body_attrs,), ) if type_parameters: ty = GenericType( type_parameters, ty, is_variadic=node.is_variadic ) ty = Fix(self_type, ty) - ty.set_internal_name(node.class_name) gamma[node.class_name] = ty # elif isinstance(node, concat.parse.TypeAliasStatementNode): # gamma[node.name], _ = node.type_node.to_type(gamma) @@ -710,10 +713,17 @@ def infer( return current_subs, current_effect, gamma -@functools.lru_cache(maxsize=None) +_module_namespaces: Dict[pathlib.Path, 'Environment'] = {} + + def _check_stub_resolved_path( - path: pathlib.Path, is_builtins: bool = False, is_preamble: bool = False + path: pathlib.Path, + is_builtins: bool = False, + is_preamble: bool = False, + initial_env: Optional['Environment'] = None, ) -> 'Environment': + if path in _module_namespaces: + return _module_namespaces[path] try: source = path.read_text() except FileNotFoundError as e: @@ -721,7 +731,7 @@ def _check_stub_resolved_path( except IOError as e: raise TypeError(f'Failed to read type stubs at {path}') from e tokens = concat.lex.tokenize(source) - env = Environment() + env = initial_env or Environment() from concat.transpile import parse try: @@ -734,6 +744,7 @@ def _check_stub_resolved_path( file, tokens, e.args[0].failures ) ) + _module_namespaces[path] = env return env recovered_parsing_failures = concat_ast.parsing_failures with path.open() as file: @@ -746,8 +757,6 @@ def _check_stub_resolved_path( concat_ast.children, str(path.parent), _should_check_bodies=False, - _should_load_builtins=not is_builtins, - _should_load_preamble=not is_preamble and not is_builtins, ) except StaticAnalysisError as e: e.set_path_if_missing(path) @@ -757,10 +766,15 @@ def _check_stub_resolved_path( def _check_stub( - path: pathlib.Path, is_builtins: bool = False, is_preamble: bool = False + path: pathlib.Path, + is_builtins: bool = False, + is_preamble: bool = False, + initial_env: Optional['Environment'] = None, ) -> 'Environment': path = path.resolve() - return _check_stub_resolved_path(path, is_builtins, is_preamble) + return _check_stub_resolved_path( + path, is_builtins, is_preamble, initial_env + ) # Parsing type annotations @@ -1320,7 +1334,11 @@ def _generate_type_of_innermost_module( elif callable(getattr(module, name)): attribute_type = py_function_type module_attributes[name] = attribute_type - module_t = ObjectType(module_attributes, nominal_supertypes=[module_type],) + module_type_brand = get_module_type().unroll().brand # type: ignore + brand = Brand( + f'type({qualified_name})', IndividualKind, [module_type_brand] + ) + module_t = NominalType(brand, ObjectType(module_attributes)) return StackEffect( TypeSequence([_seq_var]), TypeSequence([_seq_var, module_t]) ) @@ -1332,13 +1350,19 @@ def _generate_module_type( if _full_name is None: _full_name = '.'.join(components) if len(components) > 1: - module_t = ObjectType( - { - components[1]: _generate_module_type( - components[1:], _full_name, source_dir - )[_seq_var,], - }, - nominal_supertypes=[module_type], + module_type_brand = get_module_type().unroll().brand # type: ignore + brand = Brand( + f'type({_full_name})', IndividualKind, [module_type_brand] + ) + module_t = NominalType( + brand, + ObjectType( + { + components[1]: _generate_module_type( + components[1:], _full_name, source_dir + )[_seq_var,], + } + ), ) effect = StackEffect( TypeSequence([_seq_var]), TypeSequence([_seq_var, module_t]) diff --git a/concat/typecheck/builtin_stubs/builtins.cati b/concat/typecheck/builtin_stubs/builtins.cati index 20c8488..70dea0a 100644 --- a/concat/typecheck/builtin_stubs/builtins.cati +++ b/concat/typecheck/builtin_stubs/builtins.cati @@ -82,6 +82,8 @@ class tuple[`t...]: def __getitem__(--) @cast (py_function[(int), object]): () +!@@concat.typecheck.builtin_tuple tuple + class dict[`key, `value]: def __iter__(--) @cast (py_function[(), iterator[`key]]): () diff --git a/concat/typecheck/builtin_stubs/types.cati b/concat/typecheck/builtin_stubs/types.cati new file mode 100644 index 0000000..eb89cac --- /dev/null +++ b/concat/typecheck/builtin_stubs/types.cati @@ -0,0 +1,4 @@ +class ModuleType: + () + +!@@concat.typecheck.builtin_module ModuleType diff --git a/concat/typecheck/preamble.cati b/concat/typecheck/preamble.cati index 82d24b5..4c14111 100644 --- a/concat/typecheck/preamble.cati +++ b/concat/typecheck/preamble.cati @@ -13,6 +13,12 @@ class file: def __exit__(--) @cast (py_function[*any, `any2]): () +class ellipsis: + () + +class not_implemented: + () + def to_list(*rest_var i:iterable[`a_var] -- *rest_var l:list[`a_var]): () @@ -101,3 +107,33 @@ def if_not(*rest_var b:bool body:(*rest_var -- *rest_var) -- *rest_var): def if_then(*rest_var b:bool body:(*rest_var -- *rest_var) -- *rest_var): () + +def swap(*rest_var x:`a_var y:`b_var -- *rest_var y x): + () + +def pick(*rest_var x:`a_var y:`b_var z:`c_var -- *rest_var x y z x): + () + +def dup(*rest_var x:`a_var -- x x): + () + +def over(*rest_var x:`a_var y:`b_var -- *rest_var x y x): + () + +def curry(*rest_var x:`a_var f:(*seq_var x:`a_var -- *stack_var) -- *rest_var g:(*seq_var -- *stack_var)): + () + +def call(*rest_var f:(*rest_var -- *seq_var) -- *seq_var): + () + +def None(*stack_type_var -- *stack_type_var none:none): + () + +def ...(*stack_type_var -- *stack_type_var ...:ellipsis): + () + +def Ellipsis(*stack_type_var -- *stack_type_var ...:ellipsis): + () + +def NotImplemented(*stack_type_var -- *stack_type_var not_impl:not_implemented): + () diff --git a/concat/typecheck/preamble0.cati b/concat/typecheck/preamble0.cati new file mode 100644 index 0000000..71f4baf --- /dev/null +++ b/concat/typecheck/preamble0.cati @@ -0,0 +1,4 @@ +class none: + () + +!@@concat.typecheck.builtin_none none diff --git a/concat/typecheck/preamble_types.py b/concat/typecheck/preamble_types.py index cfb61fb..51fa448 100644 --- a/concat/typecheck/preamble_types.py +++ b/concat/typecheck/preamble_types.py @@ -3,21 +3,15 @@ GenericType, ItemKind, ObjectType, - SequenceVariable, - StackEffect, TypeSequence, addable_type, context_manager_type, - ellipsis_type, geq_comparable_type, iterable_type, iterator_type, leq_comparable_type, lt_comparable_type, - module_type, no_return_type, - none_type, - not_implemented_type, optional_type, py_function_type, py_overloaded_type, @@ -26,84 +20,13 @@ ) -_rest_var = SequenceVariable() -_seq_var = SequenceVariable() -_stack_var = SequenceVariable() -_stack_type_var = SequenceVariable() _a_var = BoundVariable(ItemKind) -_b_var = BoundVariable(ItemKind) -_c_var = BoundVariable(ItemKind) types = { 'addable': addable_type, 'leq_comparable': leq_comparable_type, 'lt_comparable': lt_comparable_type, 'geq_comparable': geq_comparable_type, - 'swap': GenericType( - [_rest_var, _a_var, _b_var], - StackEffect( - TypeSequence([_rest_var, _a_var, _b_var]), - TypeSequence([_rest_var, _b_var, _a_var]), - ), - ), - 'pick': GenericType( - [_rest_var, _a_var, _b_var, _c_var], - StackEffect( - TypeSequence([_rest_var, _a_var, _b_var, _c_var]), - TypeSequence([_rest_var, _a_var, _b_var, _c_var, _a_var]), - ), - ), - 'dup': GenericType( - [_rest_var, _a_var], - StackEffect( - TypeSequence([_rest_var, _a_var]), - TypeSequence([_rest_var, _a_var, _a_var]), - ), - ), - 'over': GenericType( - [_rest_var, _a_var, _b_var], - StackEffect( - TypeSequence([_rest_var, _a_var, _b_var]), - TypeSequence([_rest_var, _a_var, _b_var, _a_var]), - ), - ), - 'curry': GenericType( - [_rest_var, _seq_var, _stack_var, _a_var], - StackEffect( - TypeSequence( - [ - _rest_var, - _a_var, - StackEffect( - TypeSequence([_seq_var, _a_var]), - TypeSequence([_stack_var]), - ), - ] - ), - TypeSequence( - [ - _rest_var, - StackEffect( - TypeSequence([_seq_var]), TypeSequence([_stack_var]) - ), - ] - ), - ), - ), - 'call': GenericType( - [_rest_var, _seq_var], - StackEffect( - TypeSequence( - [ - _rest_var, - StackEffect( - TypeSequence([_rest_var]), TypeSequence([_seq_var]) - ), - ] - ), - TypeSequence([_seq_var]), - ), - ), # TODO: Separate type-check-time environment from runtime environment. 'iterable': iterable_type, 'NoReturn': no_return_type, @@ -111,39 +34,9 @@ 'subtractable': subtractable_type, 'context_manager': context_manager_type, 'iterator': iterator_type, - 'module': module_type, 'py_function': py_function_type, 'py_overloaded': py_overloaded_type, 'Optional': optional_type, - 'none': none_type, - 'None': GenericType( - [_stack_type_var], - StackEffect( - TypeSequence([_stack_type_var]), - TypeSequence([_stack_type_var, none_type]), - ), - ), - '...': GenericType( - [_stack_type_var], - StackEffect( - TypeSequence([_stack_type_var]), - TypeSequence([_stack_type_var, ellipsis_type]), - ), - ), - 'Ellipsis': GenericType( - [_stack_type_var], - StackEffect( - TypeSequence([_stack_type_var]), - TypeSequence([_stack_type_var, ellipsis_type]), - ), - ), - 'NotImplemented': GenericType( - [_stack_type_var], - StackEffect( - TypeSequence([_stack_type_var]), - TypeSequence([_stack_type_var, not_implemented_type]), - ), - ), 'SupportsAbs': GenericType( [_a_var], ObjectType({'__abs__': py_function_type[TypeSequence([]), _a_var],},), diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 3b21ffb..6cb201a 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -369,7 +369,7 @@ def constrain_and_bind_variables( ) except ConcatTypeError: return self.constrain_and_bind_variables( - none_type, rigid_variables, subtyping_assumptions + get_none_type(), rigid_variables, subtyping_assumptions ) if self in rigid_variables: raise ConcatTypeError( @@ -991,41 +991,144 @@ def _contains_assumption( return False -class ObjectType(IndividualType): - """The representation of types of objects, based on a gradual typing paper. +# The representation of types of objects. + +# Originally, it was based on a gradual typing paper. That paper is "Design and +# Evaluation of Gradual Typing for Python" (Vitousek et al. 2014). But now +# nominal and structural subtyping will be separated internally by using +# brands, like in "Integrating Nominal and Structural Subtyping" (Malayeri & +# Aldrich 2008). - That paper is "Design and Evaluation of Gradual Typing for Python" - (Vitousek et al. 2014).""" +# http://reports-archive.adm.cs.cmu.edu/anon/anon/home/ftp/usr0/ftp/2008/CMU-CS-08-120.pdf +# not using functools.total_ordering because == should be only identity. +class Brand: def __init__( - self, - attributes: Mapping[str, Type], - nominal_supertypes: Sequence[Type] = (), - nominal: bool = False, - _head: Optional['ObjectType'] = None, + self, user_name: str, kind: 'Kind', superbrands: Sequence['Brand'] ) -> None: + self._user_name = user_name + self.kind = kind + # for t in superbrands: + # if t.kind != kind: + # raise ConcatTypeError( + # f'{t} must have kind {kind}, but has kind {t.kind}' + # ) + self._superbrands = superbrands + + def __str__(self) -> str: + return self._user_name + + def __repr__(self) -> str: + return f'Brand({self._user_name!r}, {self.kind}, {self._superbrands!r})@{id(self)}' + + def __lt__(self, other: 'Brand') -> bool: + object_brand = get_object_type().unroll().brand # type: ignore + return ( + (self is not other and other is object_brand) + or other in self._superbrands + or any(brand <= other for brand in self._superbrands) + ) + + def __le__(self, other: 'Brand') -> bool: + return self is other or self < other + + +class NominalType(Type): + def __init__(self, brand: Brand, ty: Type) -> None: super().__init__() - self._attributes = attributes + self._brand = brand + self._ty = ty + # assert brand.kind == ty.kind - for t in nominal_supertypes: - if t.kind != IndividualKind: + def _free_type_variables(self) -> InsertionOrderedSet[Variable]: + return self._ty.free_type_variables() + + def apply_substitution(self, sub: 'Substitutions') -> 'NominalType': + return NominalType(self._brand, sub(self._ty)) + + @property + def attributes(self) -> Mapping[str, Type]: + return self._ty.attributes + + def constrain_and_bind_variables( + self, supertype, rigid_variables, subtyping_assumptions + ) -> 'Substitutions': + if isinstance(supertype, NominalType): + if self._brand <= supertype._brand: + return concat.typecheck.Substitutions() + raise ConcatTypeError(f'{self} is not a subtype of {supertype}') + # TODO: Find a way to force myself to handle these different cases. + # Visitor pattern? singledispatch? + if isinstance(supertype, _OptionalType): + try: + return self.constrain_and_bind_variables( + get_none_type(), rigid_variables, subtyping_assumptions + ) + except ConcatTypeError: + return self.constrain_and_bind_variables( + supertype.type_arguments[0], + rigid_variables, + subtyping_assumptions, + ) + if isinstance(supertype, Fix): + return self.constrain_and_bind_variables( + supertype.unroll(), + rigid_variables, + subtyping_assumptions + [(self, supertype)], + ) + if isinstance(supertype, ForwardTypeReference): + return self.constrain_and_bind_variables( + supertype.resolve_forward_references(), + rigid_variables, + subtyping_assumptions + [(self, supertype)], + ) + if isinstance(supertype, Variable): + if supertype in rigid_variables: raise ConcatTypeError( - f'{t} must be an individual type, but has kind {t.kind}' + f'{self} is not a subtype of rigid variable {supertype}' ) - self._nominal_supertypes = nominal_supertypes + if not (self.kind <= supertype.kind): + raise ConcatTypeError( + f'{self} has kind {self.kind}, but {supertype} has kind {supertype.kind}' + ) + return concat.typecheck.Substitutions([(supertype, self)]) + return self._ty.constrain_and_bind_variables( + supertype, rigid_variables, subtyping_assumptions + ) + + @property + def kind(self) -> 'Kind': + return self._ty.kind + + @property + def brand(self) -> Brand: + return self._brand + + def __str__(self) -> str: + return str(self._brand) + + def __repr__(self) -> str: + return f'NominalType({self._brand!r}, {self._ty!r})' + + +class ObjectType(IndividualType): + """Structural record types.""" - self._nominal = nominal + def __init__( + self, + attributes: Mapping[str, Type], + _head: Optional['ObjectType'] = None, + ) -> None: + super().__init__() + + self._attributes = attributes self._head = _head or self self._internal_name: Optional[str] = None self._internal_name = self._head._internal_name - @property - def nominal(self) -> bool: - return self._nominal - @property def kind(self) -> 'Kind': return IndividualKind @@ -1041,13 +1144,8 @@ def apply_substitution( Dict[str, IndividualType], {attr: sub(t) for attr, t in self._attributes.items()}, ) - nominal_supertypes = [ - sub(supertype) for supertype in self._nominal_supertypes - ] subbed_type = type(self)( attributes, - nominal_supertypes=nominal_supertypes, - nominal=self._nominal, # head is only used to keep track of where a type came from, so # there's no need to substitute it _head=self._head, @@ -1110,7 +1208,7 @@ def constrain_and_bind_variables( if isinstance(supertype, _OptionalType): try: sub = self.constrain_and_bind_variables( - none_type, + get_none_type(), rigid_variables, subtyping_assumptions + [(self, supertype)], ) @@ -1146,18 +1244,13 @@ def constrain_and_bind_variables( ) sub.add_subtyping_provenance((self, supertype)) return sub + # Don't forget that there's nominal subtyping too. + if isinstance(supertype, NominalType): + raise concat.typecheck.errors.TypeError( + f'structural type {self} cannot be a subtype of nominal type {supertype}' + ) if not isinstance(supertype, ObjectType): raise NotImplementedError(repr(supertype)) - # Don't forget that there's nominal subtyping too. - if supertype._nominal: - if supertype in self._nominal_supertypes: - sub = Substitutions() - sub.add_subtyping_provenance((self, supertype)) - return sub - if self._head is not supertype._head: - raise ConcatTypeError( - '{} is not a subtype of {}'.format(self, supertype) - ) subtyping_assumptions = subtyping_assumptions + [(self, supertype)] @@ -1176,7 +1269,7 @@ def constrain_and_bind_variables( def __repr__(self) -> str: head = None if self._head is self else self._head - return f'{type(self).__qualname__}(attributes={self._attributes!r}, nominal_supertypes={self._nominal_supertypes!r}, nominal={self._nominal!r}, _head={head!r})' + return f'{type(self).__qualname__}(attributes={self._attributes!r}, _head={head!r})' def _free_type_variables(self) -> InsertionOrderedSet[Variable]: ftv = free_type_variables_of_mapping(self.attributes) @@ -1186,7 +1279,7 @@ def _free_type_variables(self) -> InsertionOrderedSet[Variable]: def __str__(self) -> str: if self._internal_name is not None: return self._internal_name - return f'ObjectType({_mapping_to_str(self._attributes)}, {_iterable_to_str(self._nominal_supertypes)}, {self._nominal}, {None if self._head is self else self._head})' + return f'ObjectType({_mapping_to_str(self._attributes)}, {None if self._head is self else self._head})' @property def attributes(self) -> Mapping[str, Type]: @@ -1196,10 +1289,6 @@ def attributes(self) -> Mapping[str, Type]: def head(self) -> 'ObjectType': return self._head - @property - def nominal_supertypes(self) -> Sequence[Type]: - return self._nominal_supertypes - # QUESTION: Should this exist, or should I use ObjectType? class ClassType(ObjectType): @@ -1617,10 +1706,13 @@ def constrain_and_bind_variables( f'{self} is an individual type, but {supertype} has kind {supertype.kind}' ) # FIXME: optional[none] should simplify to none - if self._type_argument is none_type and supertype is none_type: + if ( + self._type_argument is get_none_type() + and supertype is get_none_type() + ): return Substitutions() - sub = none_type.constrain_and_bind_variables( + sub = get_none_type().constrain_and_bind_variables( supertype, rigid_variables, subtyping_assumptions ) sub = sub(self._type_argument).constrain_and_bind_variables( @@ -1638,6 +1730,7 @@ def type_arguments(self) -> Sequence[Type]: return [self._type_argument] +# FIXME: Not a total order, using total_ordering might be very unsound. @functools.total_ordering class Kind(abc.ABC): @abc.abstractmethod @@ -1963,7 +2056,7 @@ def _mapping_to_str(mapping: Mapping) -> str: _x = BoundVariable(kind=IndividualKind) -float_type = ObjectType({}, nominal=True) +float_type = NominalType(Brand('float', IndividualKind, []), ObjectType({})) no_return_type = _NoReturnType() @@ -1977,6 +2070,7 @@ def get_object_type() -> Type: def set_object_type(ty: Type) -> None: global _object_type + assert _object_type is None _object_type = ty @@ -2006,6 +2100,19 @@ def set_str_type(ty: Type) -> None: _str_type = ty +_tuple_type: Optional[Type] = None + + +def get_tuple_type() -> Type: + assert _tuple_type is not None + return _tuple_type + + +def set_tuple_type(ty: Type) -> None: + global _tuple_type + _tuple_type = ty + + _int_type: Optional[Type] = None @@ -2032,6 +2139,32 @@ def set_bool_type(ty: Type) -> None: _bool_type = ty +_none_type: Optional[Type] = None + + +def get_none_type() -> Type: + assert _none_type is not None + return _none_type + + +def set_none_type(ty: Type) -> None: + global _none_type + _none_type = ty + + +_module_type: Optional[Type] = None + + +def get_module_type() -> Type: + assert _module_type is not None + return _module_type + + +def set_module_type(ty: Type) -> None: + global _module_type + _module_type = ty + + _arg_type_var = SequenceVariable() _return_type_var = ItemVariable(IndividualKind) py_function_type = PythonFunctionType( @@ -2121,10 +2254,6 @@ def set_bool_type(ty: Type) -> None: ) lt_comparable_type.set_internal_name('lt_comparable_type') - -none_type = ObjectType({}, nominal=True) -none_type.set_internal_name('none_type') - _result_type = BoundVariable(ItemKind) iterator_type = GenericType( @@ -2134,9 +2263,7 @@ def set_bool_type(ty: Type) -> None: ObjectType( { '__iter__': py_function_type[TypeSequence([]), _x], - '__next__': py_function_type[ - TypeSequence([none_type,]), _result_type - ], + '__next__': py_function_type[TypeSequence([]), _result_type], }, ), ), @@ -2171,50 +2298,6 @@ def set_bool_type(ty: Type) -> None: ) optional_type.set_internal_name('optional_type') -_key_type_var = BoundVariable(kind=IndividualKind) -_value_type_var = BoundVariable(kind=IndividualKind) -dict_type = GenericType( - [_key_type_var, _value_type_var], - ObjectType( - { - '__iter__': py_function_type[ - TypeSequence([]), iterator_type[_key_type_var,] - ] - }, - nominal=True, - ), -) -dict_type.set_internal_name('dict_type') - -_start_type_var, _stop_type_var, _step_type_var = ( - BoundVariable(ItemKind), - BoundVariable(ItemKind), - BoundVariable(ItemKind), -) -slice_type = GenericType( - [_start_type_var, _stop_type_var, _step_type_var], - ObjectType({}, nominal=True), -) -slice_type.set_internal_name('slice_type') - -ellipsis_type = ObjectType({}, nominal=True) -not_implemented_type = ObjectType({}, nominal=True) - -_element_types_var = SequenceVariable() -tuple_type = GenericType( - [_element_types_var], - ObjectType( - {'__getitem__': py_function_type}, - nominal=True, - # iterable_type is a structural supertype - ), - is_variadic=True, -) -tuple_type.set_internal_name('tuple_type') - -base_exception_type = ObjectType({}, nominal=True) -module_type = ObjectType({}, nominal=True) - _index_type_var = BoundVariable(ItemKind) _result_type_var = BoundVariable(ItemKind) subscriptable_type = GenericType( @@ -2227,10 +2310,3 @@ def set_bool_type(ty: Type) -> None: }, ), ) - -_answer_type_var = BoundVariable(ItemKind) -continuation_monad_type = GenericType( - [_result_type_var, _answer_type_var], - ObjectType(attributes={}, nominal=True,), -) -continuation_monad_type.set_internal_name('continuation_monad_type') From a10dbf120adeb0757915d2d7cd797dddcdd9a0ac Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Sat, 7 Sep 2024 09:58:34 -0700 Subject: [PATCH 46/61] Remove remaining module type guessing code and use stubs instead Now we don't need to import (and execute!) modules imported by Concat source within the type checker. --- concat/typecheck/__init__.py | 56 ++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index 2fdbf4a..4c9eec9 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -713,6 +713,32 @@ def infer( return current_subs, current_effect, gamma +def _find_stub_path(module_parts: Sequence[str]) -> pathlib.Path: + module_spec = None + path = None + if module_parts[0] in sys.builtin_module_names: + stub_path = pathlib.Path(__file__) / '../builtin_stubs' + for part in module_parts: + stub_path = stub_path / part + else: + for module_prefix in itertools.accumulate( + module_parts, lambda a, b: f'{a}.{b}' + ): + for finder in sys.meta_path: + module_spec = finder.find_spec(module_prefix, path) + if module_spec is not None: + path = module_spec.submodule_search_locations + break + assert module_spec is not None + module_path = module_spec.origin + if module_path is None: + raise TypeError(f'Cannot find path of module {module_prefix}') + # For now, assume the module's written in Python. + stub_path = pathlib.Path(module_path) + stub_path = stub_path.with_suffix('.cati') + return stub_path + + _module_namespaces: Dict[pathlib.Path, 'Environment'] = {} @@ -873,7 +899,8 @@ def __init__( self._type = args[1] self.children = [n for n in args if isinstance(n, TypeNode)] - # QUESTION: Should I have a separate space for the temporary associated names? + # QUESTION: Should I have a separate space for the temporary associated + # names? def to_type(self, env: Environment) -> Tuple[IndividualType, Environment]: if self._name is None: return self._type.to_type(env) @@ -1316,24 +1343,9 @@ def object_type_parser() -> Generator: def _generate_type_of_innermost_module( qualified_name: str, source_dir ) -> StackEffect: - # We resolve imports as if we are the source file. - sys.path, old_path = [source_dir, *sys.path], sys.path - try: - module = importlib.import_module(qualified_name) - except ModuleNotFoundError: - raise TypeError( - 'module {} not found during type checking'.format(qualified_name) - ) - finally: - sys.path = old_path - module_attributes = {} - for name in dir(module): - attribute_type = get_object_type() - if isinstance(getattr(module, name), int): - attribute_type = get_int_type() - elif callable(getattr(module, name)): - attribute_type = py_function_type - module_attributes[name] = attribute_type + stub_path = _find_stub_path(str.split('.')) + init_env = load_builtins_and_preamble() + module_attributes = _check_stub(stub_path, False, False, init_env) module_type_brand = get_module_type().unroll().brand # type: ignore brand = Brand( f'type({qualified_name})', IndividualKind, [module_type_brand] @@ -1367,12 +1379,14 @@ def _generate_module_type( effect = StackEffect( TypeSequence([_seq_var]), TypeSequence([_seq_var, module_t]) ) - return ObjectType({'__call__': effect,}, [_seq_var]) + return GenericType([_seq_var], ObjectType({'__call__': effect,})) else: innermost_type = _generate_type_of_innermost_module( _full_name, source_dir ) - return ObjectType({'__call__': innermost_type,}, [_seq_var]) + return GenericType( + [_seq_var], ObjectType({'__call__': innermost_type,}) + ) def _ensure_type( From 4ab282a97790950a771ced06d108071cf2f08961 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Sat, 7 Sep 2024 10:11:13 -0700 Subject: [PATCH 47/61] Update GitHub workflow after removing tox --- .github/workflows/python-app.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 389e5da..22d15cc 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -25,12 +25,14 @@ jobs: uses: actions/setup-python@v3 with: python-version: "3.7" - - name: Install development dependencies + - name: Install test dependencies run: | - pip install -e .[dev] + pip install -e .[test] - name: Test run: | - tox run + nose2 --pretty-assert concat.tests + - name: Collect coverage into one file + run: coverage combine - name: Report test coverage to DeepSource uses: deepsourcelabs/test-coverage-action@master with: From 66a6b94028848d41b2ff031bad6934741cd58572 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Sat, 7 Sep 2024 10:25:09 -0700 Subject: [PATCH 48/61] Make sure to generate coverage.xml and coverage.lcov --- .github/workflows/python-app.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 22d15cc..51d2740 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -32,7 +32,10 @@ jobs: run: | nose2 --pretty-assert concat.tests - name: Collect coverage into one file - run: coverage combine + run: + - coverage combine + - coverage xml + - coverage lcov - name: Report test coverage to DeepSource uses: deepsourcelabs/test-coverage-action@master with: From 8bdacaeedcb48eb16efba4d471e5dd018067f0d3 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Sat, 7 Sep 2024 10:28:25 -0700 Subject: [PATCH 49/61] Don't use sequence for multiple commands? --- .github/workflows/python-app.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 51d2740..c202027 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -32,10 +32,10 @@ jobs: run: | nose2 --pretty-assert concat.tests - name: Collect coverage into one file - run: - - coverage combine - - coverage xml - - coverage lcov + run: | + coverage combine + coverage xml + coverage lcov - name: Report test coverage to DeepSource uses: deepsourcelabs/test-coverage-action@master with: From 078a9e17a5e78c8d3b3ae2035526941edf890723 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Wed, 9 Oct 2024 00:18:38 -0700 Subject: [PATCH 50/61] Do some cleanup --- concat/__main__.py | 1 - concat/orderedset.py | 9 +++------ concat/tests/stdlib/test_repl.py | 1 - concat/tests/test_linked_list.py | 2 +- concat/tests/test_orderedset.py | 4 +--- concat/typecheck/__init__.py | 34 ++++++++++++++++---------------- concat/typecheck/types.py | 23 +++++++-------------- 7 files changed, 29 insertions(+), 45 deletions(-) diff --git a/concat/__main__.py b/concat/__main__.py index 26865ee..8541599 100644 --- a/concat/__main__.py +++ b/concat/__main__.py @@ -3,7 +3,6 @@ import argparse from concat.transpile import parse, transpile_ast, typecheck -import concat.astutils from concat.error_reporting import get_line_at, create_parsing_failure_message import concat.execute import concat.lex diff --git a/concat/orderedset.py b/concat/orderedset.py index 8195ffe..a12a437 100644 --- a/concat/orderedset.py +++ b/concat/orderedset.py @@ -1,15 +1,13 @@ -from concat.linked_list import LinkedList, empty_list +from concat.linked_list import LinkedList from typing import ( AbstractSet, Any, - Generic, Iterable, Iterator, Optional, Reversible, Tuple, TypeVar, - Union, ) _T = TypeVar('_T', covariant=True) @@ -471,9 +469,8 @@ def _delete_upwards_phase(self) -> '_Tree23': return self if self.is_2_node(): left, x, right = self._data - if self._is_2_node_terminal(): - if x is self.__hole: - return _Tree23((x, _leaf_23_tree)) + if self._is_2_node_terminal() and x is self.__hole: + return _Tree23((x, _leaf_23_tree)) if left._is_hole_node(): if right.is_2_node(): # 2-node parent, 2-node sibling, hole on left diff --git a/concat/tests/stdlib/test_repl.py b/concat/tests/stdlib/test_repl.py index 9b66bb9..e5cb5ea 100644 --- a/concat/tests/stdlib/test_repl.py +++ b/concat/tests/stdlib/test_repl.py @@ -2,7 +2,6 @@ import io import sys import contextlib -import concat.parse import concat.parser_combinators import concat.typecheck from concat.typecheck.types import SequenceVariable, StackEffect, TypeSequence diff --git a/concat/tests/test_linked_list.py b/concat/tests/test_linked_list.py index 8826d51..6a8c515 100644 --- a/concat/tests/test_linked_list.py +++ b/concat/tests/test_linked_list.py @@ -1,5 +1,5 @@ from concat.linked_list import LinkedList, empty_list -from hypothesis import example, given +from hypothesis import given import hypothesis.strategies as st from typing import Callable, List import unittest diff --git a/concat/tests/test_orderedset.py b/concat/tests/test_orderedset.py index 7c4f2ab..3e18093 100644 --- a/concat/tests/test_orderedset.py +++ b/concat/tests/test_orderedset.py @@ -15,9 +15,7 @@ def test_set_difference_preserves_order( self, original: Set[int], to_remove: Set[int] ) -> None: insertion_order_set = InsertionOrderedSet[int](list(original)) - expected_order = list( - x for x in insertion_order_set if x not in to_remove - ) + expected_order = [x for x in insertion_order_set if x not in to_remove] insertion_order_set -= to_remove actual_order = list(insertion_order_set) self.assertListEqual(expected_order, actual_order) diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index 4c9eec9..1c68503 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -61,6 +61,7 @@ def free_type_variables(self) -> 'InsertionOrderedSet[Variable]': class _Substitutable(Protocol[_Result]): def apply_substitution(self, sub: 'Substitutions') -> _Result: + # empty, abstract protocol method pass @@ -68,16 +69,17 @@ class Substitutions(Mapping['Variable', 'Type']): def __init__( self, sub: Union[ - Iterable[Tuple['Variable', 'Type']], Mapping['Variable', 'Type'] - ] = {}, + Iterable[Tuple['Variable', 'Type']], + Mapping['Variable', 'Type'], + None, + ] = None, ) -> None: - self._sub = dict(sub) - # See HACK KIND_POLY - # for variable, ty in self._sub.items(): - # if variable.kind != ty.kind: - # raise TypeError( - # f'{variable} is being substituted by {ty}, which has the wrong kind ({variable.kind} vs {ty.kind})' - # ) + self._sub = {} if sub is None else dict(sub) + for variable, ty in self._sub.items(): + if not (variable.kind >= ty.kind): + raise TypeError( + f'{variable} is being substituted by {ty}, which has the wrong kind ({variable.kind} vs {ty.kind})' + ) self._cache: Dict[int, 'Type'] = {} # innermost first self.subtyping_provenance: List[Any] = [] @@ -169,18 +171,14 @@ def __hash__(self) -> int: free_type_variables_of_mapping, get_int_type, get_list_type, - get_object_type, get_str_type, get_tuple_type, get_module_type, no_return_type, - py_function_type, ) import abc from concat.error_reporting import create_parsing_failure_message from concat.lex import Token -import importlib -import importlib.util import itertools import pathlib import sys @@ -210,7 +208,7 @@ def load_builtins_and_preamble() -> Environment: def check( environment: Environment, - program: concat.astutils.WordsOrStatements, + program: 'concat.astutils.WordsOrStatements', source_dir: str = '.', _should_check_bodies: bool = True, ) -> Environment: @@ -701,8 +699,7 @@ def infer( ) ty = Fix(self_type, ty) gamma[node.class_name] = ty - # elif isinstance(node, concat.parse.TypeAliasStatementNode): - # gamma[node.name], _ = node.type_node.to_type(gamma) + # TODO: Type aliases else: raise UnhandledNodeTypeError( "don't know how to handle '{}'".format(node) @@ -732,7 +729,9 @@ def _find_stub_path(module_parts: Sequence[str]) -> pathlib.Path: assert module_spec is not None module_path = module_spec.origin if module_path is None: - raise TypeError(f'Cannot find path of module {module_prefix}') + raise TypeError( + f'Cannot find path of module {".".join(module_parts)}' + ) # For now, assume the module's written in Python. stub_path = pathlib.Path(module_path) stub_path = stub_path.with_suffix('.cati') @@ -808,6 +807,7 @@ def _check_stub( class TypeNode(concat.parse.Node, abc.ABC): def __init__(self, location: concat.astutils.Location) -> None: + super().__init__() self.location = location self.children: Sequence[concat.parse.Node] diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 6cb201a..565d59b 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -982,13 +982,10 @@ def free_type_variables_of_mapping( def _contains_assumption( assumptions: Sequence[Tuple[Type, Type]], subtype: Type, supertype: Type ) -> bool: - for sub, sup in assumptions: - if ( - sub._type_id == subtype._type_id - and sup._type_id == supertype._type_id - ): - return True - return False + any( + sub._type_id == subtype._type_id and sup._type_id == supertype._type_id + for sub, sup in assumptions + ) # The representation of types of objects. @@ -1008,11 +1005,6 @@ def __init__( ) -> None: self._user_name = user_name self.kind = kind - # for t in superbrands: - # if t.kind != kind: - # raise ConcatTypeError( - # f'{t} must have kind {kind}, but has kind {t.kind}' - # ) self._superbrands = superbrands def __str__(self) -> str: @@ -1039,7 +1031,7 @@ def __init__(self, brand: Brand, ty: Type) -> None: self._brand = brand self._ty = ty - # assert brand.kind == ty.kind + # TODO: Make sure brands interact with generics properly def _free_type_variables(self) -> InsertionOrderedSet[Variable]: return self._ty.free_type_variables() @@ -1878,7 +1870,8 @@ def unroll(self) -> Type: self._unrolled_ty = self._apply(self) if self._internal_name is not None: self._unrolled_ty.set_internal_name(self._internal_name) - # self._unrolled_ty._type_id = self._type_id + # Do not make the type ids equal so that subtyping assumptions are + # useful return self._unrolled_ty def _free_type_variables(self) -> InsertionOrderedSet[Variable]: @@ -1913,8 +1906,6 @@ def constrain_and_bind_variables( if isinstance(supertype, Fix): unrolled = supertype.unroll() - # BUG: The unrolled types have the same type ids, so the assumption - # is used immediately, which is unsound. sub = self.unroll().constrain_and_bind_variables( unrolled, rigid_variables, From f11910206dc82d65647fc28ed3e10eaac652f536 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Wed, 9 Oct 2024 00:44:47 -0700 Subject: [PATCH 51/61] Build linked list in stack-safe way that doesn't require reversible iterable --- concat/linked_list.py | 27 +++++++++++++++++---------- concat/orderedset.py | 9 ++++++--- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/concat/linked_list.py b/concat/linked_list.py index ab3a9cc..887ef31 100644 --- a/concat/linked_list.py +++ b/concat/linked_list.py @@ -1,9 +1,9 @@ from typing import ( Callable, + Iterable, Iterator, List, Optional, - Reversible, Sequence, Tuple, TypeVar, @@ -21,19 +21,19 @@ def __init__( self, _val: Optional[Tuple[_T_co, 'LinkedList[_T_co]']] ) -> None: self._val = _val - if _val is None: - self._length = 0 - else: - self._length = 1 + len(_val[1]) + self._length: Optional[int] = None @classmethod - def from_iterable(cls, iterable: Reversible[_T_co]) -> 'LinkedList[_T_co]': + def from_iterable(cls, iterable: Iterable[_T_co]) -> 'LinkedList[_T_co]': if isinstance(iterable, cls): return iterable - l: LinkedList[_T_co] = empty_list - for el in reversed(iterable): - l = cls((el, l)) - return l + l: LinkedList[_T_co] = cls(None) + head = l + for el in iterable: + next_node = cls(None) + l._val = (el, next_node) + l = next_node + return head @overload def __getitem__(self, i: int) -> _T_co: @@ -57,6 +57,13 @@ def __getitem__( return self._val[0] def __len__(self) -> int: + if self._length is None: + node = self + length = 0 + while node._val is not None: + node = node._val[1] + length += 1 + self._length = length return self._length def __add__(self, other: 'LinkedList[_T_co]') -> 'LinkedList[_T_co]': diff --git a/concat/orderedset.py b/concat/orderedset.py index a12a437..3ffa270 100644 --- a/concat/orderedset.py +++ b/concat/orderedset.py @@ -5,7 +5,6 @@ Iterable, Iterator, Optional, - Reversible, Tuple, TypeVar, ) @@ -52,14 +51,18 @@ def __len__(self) -> int: class InsertionOrderedSet(AbstractSet[_T]): def __init__( self, - elements: 'Reversible[_T]', + elements: 'Iterable[_T]', _order: Optional[LinkedList[_T]] = None, ) -> None: super().__init__() - self._data = OrderedSet(elements) self._order = ( LinkedList.from_iterable(elements) if _order is None else _order ) + if isinstance(elements, OrderedSet): + self._data = elements + else: + # elements might be an iterator that gets exhausted + self._data = OrderedSet(self._order) def __sub__(self, other: object) -> 'InsertionOrderedSet[_T]': if not isinstance(other, AbstractSet): From 31c30c2d814212cf6bd5463433cd8f4503c3d2d2 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Wed, 9 Oct 2024 00:51:42 -0700 Subject: [PATCH 52/61] Make an additional effort to avoid substitutions on types --- concat/typecheck/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index 1c68503..31578c5 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -111,7 +111,10 @@ def __call__(self, arg: _Substitutable[_Result]) -> _Result: # nondeterministic Concat type errors from the type checker. if isinstance(arg, Type): if arg._type_id not in self._cache: - self._cache[arg._type_id] = arg.apply_substitution(self) + if not (self._dom() & arg.free_type_variables()): + self._cache[arg._type_id] = arg + else: + self._cache[arg._type_id] = arg.apply_substitution(self) result = self._cache[arg._type_id] if isinstance(arg, Environment): if arg.id not in self._cache: From 8995665913b3633a1d305fe89b2583bddf0c808d Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Wed, 9 Oct 2024 01:01:54 -0700 Subject: [PATCH 53/61] Simplify parameters of stub loading functions Since there's a global module cache, is_builtins and is_preamble are uneccessary. --- concat/typecheck/__init__.py | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index 31578c5..aafbe02 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -193,14 +193,10 @@ def __hash__(self) -> int: def load_builtins_and_preamble() -> Environment: + env = _check_stub(pathlib.Path(__file__).with_name('preamble0.cati'),) + env = _check_stub(_builtins_stub_path, initial_env=env) env = _check_stub( - pathlib.Path(__file__).with_name('preamble0.cati'), is_preamble=True, - ) - env = _check_stub(_builtins_stub_path, is_builtins=True, initial_env=env) - env = _check_stub( - pathlib.Path(__file__).with_name('preamble.cati'), - is_preamble=True, - initial_env=env, + pathlib.Path(__file__).with_name('preamble.cati'), initial_env=env, ) # pick up ModuleType _check_stub( @@ -745,10 +741,7 @@ def _find_stub_path(module_parts: Sequence[str]) -> pathlib.Path: def _check_stub_resolved_path( - path: pathlib.Path, - is_builtins: bool = False, - is_preamble: bool = False, - initial_env: Optional['Environment'] = None, + path: pathlib.Path, initial_env: Optional['Environment'] = None, ) -> 'Environment': if path in _module_namespaces: return _module_namespaces[path] @@ -794,15 +787,10 @@ def _check_stub_resolved_path( def _check_stub( - path: pathlib.Path, - is_builtins: bool = False, - is_preamble: bool = False, - initial_env: Optional['Environment'] = None, + path: pathlib.Path, initial_env: Optional['Environment'] = None, ) -> 'Environment': path = path.resolve() - return _check_stub_resolved_path( - path, is_builtins, is_preamble, initial_env - ) + return _check_stub_resolved_path(path, initial_env) # Parsing type annotations @@ -1348,7 +1336,7 @@ def _generate_type_of_innermost_module( ) -> StackEffect: stub_path = _find_stub_path(str.split('.')) init_env = load_builtins_and_preamble() - module_attributes = _check_stub(stub_path, False, False, init_env) + module_attributes = _check_stub(stub_path, init_env) module_type_brand = get_module_type().unroll().brand # type: ignore brand = Brand( f'type({qualified_name})', IndividualKind, [module_type_brand] From f98d2a64587cf6b7c3efa252f7bf4d0fba853db8 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Wed, 9 Oct 2024 01:04:43 -0700 Subject: [PATCH 54/61] Remove unused parameter of _generate_type_of_innermost_module --- concat/typecheck/__init__.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index aafbe02..9727bd2 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -479,7 +479,7 @@ def infer( if node.asname is not None: gamma[node.asname] = current_subs( _generate_type_of_innermost_module( - node.value, source_dir + node.value ).generalized_wrt(current_subs(gamma)) ) else: @@ -1331,9 +1331,7 @@ def object_type_parser() -> Generator: _seq_var = SequenceVariable() -def _generate_type_of_innermost_module( - qualified_name: str, source_dir -) -> StackEffect: +def _generate_type_of_innermost_module(qualified_name: str,) -> StackEffect: stub_path = _find_stub_path(str.split('.')) init_env = load_builtins_and_preamble() module_attributes = _check_stub(stub_path, init_env) @@ -1372,9 +1370,7 @@ def _generate_module_type( ) return GenericType([_seq_var], ObjectType({'__call__': effect,})) else: - innermost_type = _generate_type_of_innermost_module( - _full_name, source_dir - ) + innermost_type = _generate_type_of_innermost_module(_full_name,) return GenericType( [_seq_var], ObjectType({'__call__': innermost_type,}) ) From 67d5836339739eb297aab73250df9df0a23edd55 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Thu, 10 Oct 2024 23:22:19 -0700 Subject: [PATCH 55/61] Attach end location to type application syntax --- concat/typecheck/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index 9727bd2..bf7fe90 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -1267,8 +1267,11 @@ def generic_type_parser() -> Generator: type_arguments = yield parsers['type'].sep_by( concat.parse.token('COMMA'), min=1 ) - yield concat.parse.token('RSQB') - return _GenericTypeNode(type.location, type, type_arguments) + end_location = (yield concat.parse.token('RSQB')).end + # TODO: Add end_location to all type nodes + return _GenericTypeNode( + type.location, end_location, type, type_arguments + ) @concat.parser_combinators.generate def forall_type_parser() -> Generator: From 8af8a092cffb0b41a55be2264f96d80543d5eacf Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Thu, 10 Oct 2024 23:25:40 -0700 Subject: [PATCH 56/61] Return result from _contains_assumption --- concat/typecheck/types.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 565d59b..16c6e63 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -982,7 +982,7 @@ def free_type_variables_of_mapping( def _contains_assumption( assumptions: Sequence[Tuple[Type, Type]], subtype: Type, supertype: Type ) -> bool: - any( + return any( sub._type_id == subtype._type_id and sup._type_id == supertype._type_id for sub, sup in assumptions ) @@ -1046,6 +1046,12 @@ def attributes(self) -> Mapping[str, Type]: def constrain_and_bind_variables( self, supertype, rigid_variables, subtyping_assumptions ) -> 'Substitutions': + if ( + supertype._type_id == get_object_type()._type_id + or self._type_id == supertype._type_id + or _contains_assumption(subtyping_assumptions, self, supertype) + ): + return concat.typecheck.Substitutions() if isinstance(supertype, NominalType): if self._brand <= supertype._brand: return concat.typecheck.Substitutions() @@ -1244,8 +1250,6 @@ def constrain_and_bind_variables( if not isinstance(supertype, ObjectType): raise NotImplementedError(repr(supertype)) - subtyping_assumptions = subtyping_assumptions + [(self, supertype)] - # don't constrain the type arguments, constrain those based on # the attributes sub = Substitutions() From 7b68cdd8a9c891f289d8372896536b0ce3f98c1f Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Thu, 10 Oct 2024 23:28:31 -0700 Subject: [PATCH 57/61] Remove structural type check from ObjectType code --- concat/typecheck/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 16c6e63..0363466 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -1517,7 +1517,7 @@ def constrain_and_bind_variables( ) sub.add_subtyping_provenance((self, supertype)) return sub - if isinstance(supertype, ObjectType) and not supertype.nominal: + if isinstance(supertype, ObjectType): sub = Substitutions() for attr in supertype.attributes: self_attr_type = sub(self.get_type_of_attribute(attr)) From f48e00dc7026de71eadc42c43f0fc13366ac5e33 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Fri, 11 Oct 2024 00:24:37 -0700 Subject: [PATCH 58/61] Remove SubtypingExplanation and is_subtype_of --- concat/tests/test_typecheck.py | 67 +++++++++++++++++++++++----------- concat/typecheck/types.py | 67 ++++++++-------------------------- 2 files changed, 62 insertions(+), 72 deletions(-) diff --git a/concat/tests/test_typecheck.py b/concat/tests/test_typecheck.py index 1ed88c4..6b7114d 100644 --- a/concat/tests/test_typecheck.py +++ b/concat/tests/test_typecheck.py @@ -2,7 +2,8 @@ import concat.parse import concat.typecheck import concat.parse -from concat.typecheck import Environment +from concat.typecheck import Environment, Substitutions +from concat.typecheck.errors import TypeError as ConcatTypeError from concat.typecheck.types import ( BoundVariable, ClassType, @@ -304,9 +305,8 @@ def test_reflexive_equality(self, type): class TestSubtyping(unittest.TestCase): def test_int_not_subtype_of_float(self) -> None: """Differ from Reticulated Python: !(int <= float).""" - ex = get_int_type().is_subtype_of(float_type) - print(ex) - self.assertFalse(ex) + with self.assertRaises(ConcatTypeError): + get_int_type().constrain_and_bind_variables(float_type, set(), []) @given(from_type(IndividualType), from_type(IndividualType)) @settings(suppress_health_check=(HealthCheck.filter_too_much,)) @@ -315,20 +315,25 @@ def test_stack_effect_subtyping(self, type1, type2) -> None: fun2 = StackEffect( TypeSequence([no_return_type]), TypeSequence([get_object_type()]) ) - self.assertTrue(fun1.is_subtype_of(fun2)) + self.assertEqual( + fun1.constrain_and_bind_variables(fun2, set(), []), Substitutions() + ) @given(from_type(IndividualType)) @settings(suppress_health_check=(HealthCheck.filter_too_much,)) def test_no_return_is_bottom_type(self, type) -> None: - self.assertTrue(no_return_type.is_subtype_of(type)) + self.assertEqual( + no_return_type.constrain_and_bind_variables(type, set(), []), + Substitutions(), + ) @given(from_type(IndividualType)) @settings(suppress_health_check=(HealthCheck.filter_too_much,)) def test_object_is_top_type(self, type) -> None: - ex = type.is_subtype_of(get_object_type()) - note(repr(type)) - note(str(ex)) - self.assertTrue(ex) + self.assertEqual( + type.constrain_and_bind_variables(get_object_type(), set(), []), + Substitutions(), + ) __attributes_generator = dictionaries( text(max_size=25), from_type(IndividualType), max_size=5 # type: ignore @@ -346,7 +351,10 @@ def test_object_structural_subtyping( ) -> None: object1 = ObjectType({**other_attributes, **attributes}) object2 = ObjectType(attributes) - self.assertTrue(object1.is_subtype_of(object2)) + self.assertEqual( + object1.constrain_and_bind_variables(object2, set(), []), + Substitutions(), + ) @given(__attributes_generator, __attributes_generator) @settings(suppress_health_check=(HealthCheck.filter_too_much,)) @@ -357,15 +365,19 @@ def test_class_structural_subtyping( object2 = ClassType(attributes) note(repr(object1)) note(repr(object2)) - ex = object1.is_subtype_of(object2) - note(str(ex)) - self.assertTrue(ex) + self.assertEqual( + object1.constrain_and_bind_variables(object2, set(), []), + Substitutions(), + ) @given(from_type(StackEffect)) @settings(suppress_health_check=(HealthCheck.filter_too_much,)) def test_object_subtype_of_stack_effect(self, effect) -> None: object = ObjectType({'__call__': effect}) - self.assertTrue(object.is_subtype_of(effect)) + self.assertEqual( + object.constrain_and_bind_variables(effect, set(), []), + Substitutions(), + ) @given(from_type(IndividualType), from_type(IndividualType)) @settings( @@ -377,7 +389,10 @@ def test_object_subtype_of_stack_effect(self, effect) -> None: def test_object_subtype_of_py_function(self, type1, type2) -> None: py_function = py_function_type[TypeSequence([type1]), type2] object = ObjectType({'__call__': py_function}) - self.assertTrue(object.is_subtype_of(py_function)) + self.assertEqual( + object.constrain_and_bind_variables(py_function, set(), []), + Substitutions(), + ) @given(from_type(StackEffect)) def test_class_subtype_of_stack_effect(self, effect) -> None: @@ -387,7 +402,10 @@ def test_class_subtype_of_stack_effect(self, effect) -> None: TypeSequence([*effect.input, x]), effect.output ) cls = Fix(x, ClassType({'__init__': unbound_effect})) - self.assertTrue(cls.is_subtype_of(effect)) + self.assertEqual( + cls.constrain_and_bind_variables(effect, set(), []), + Substitutions(), + ) @given(from_type(IndividualType), from_type(IndividualType)) @settings( @@ -401,19 +419,26 @@ def test_class_subtype_of_py_function(self, type1, type2) -> None: py_function = py_function_type[TypeSequence([type1]), type2] unbound_py_function = py_function_type[TypeSequence([x, type1]), type2] cls = Fix(x, ClassType({'__init__': unbound_py_function})) - self.assertTrue(cls.is_subtype_of(py_function)) + self.assertEqual( + cls.constrain_and_bind_variables(py_function, set(), []), + Substitutions(), + ) @given(from_type(IndividualType)) def test_none_subtype_of_optional(self, ty: IndividualType) -> None: opt_ty = optional_type[ ty, ] - self.assertTrue(get_none_type().is_subtype_of(opt_ty)) + self.assertEqual( + get_none_type().constrain_and_bind_variables(opt_ty, set(), []), + Substitutions(), + ) @given(from_type(IndividualType)) def test_type_subtype_of_optional(self, ty: IndividualType) -> None: opt_ty = optional_type[ ty, ] - note(str(ty)) - self.assertTrue(ty.is_subtype_of(opt_ty)) + self.assertEqual( + ty.constrain_and_bind_variables(opt_ty, set(), []), Substitutions() + ) diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 0363466..c934613 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -32,32 +32,6 @@ from concat.typecheck import Environment, Substitutions -class SubtypeExplanation: - def __init__(self, data: Any) -> None: - self._data = data - - def __bool__(self) -> bool: - if isinstance(self._data, concat.typecheck.Substitutions): - return not self._data - return not isinstance(self._data, StaticAnalysisError) - - def __str__(self) -> str: - if isinstance(self._data, StaticAnalysisError): - e: Optional[BaseException] = self._data - string = '' - while e is not None: - string += '\n' + str(e) - e = e.__cause__ or e.__context__ - return string - if isinstance(self._data, concat.typecheck.Substitutions): - string = str(self._data) - string += '\n' + '\n'.join( - (map(str, self._data.subtyping_provenance)) - ) - return string - return str(self._data) - - class Type(abc.ABC): _next_type_id = 0 @@ -69,29 +43,30 @@ def __init__(self) -> None: self._type_id = Type._next_type_id Type._next_type_id += 1 - # QUESTION: Do I need this? - def is_subtype_of(self, supertype: 'Type') -> SubtypeExplanation: - from concat.typecheck import Substitutions - - try: - sub = self.constrain_and_bind_variables(supertype, set(), []) - except ConcatTypeError as e: - return SubtypeExplanation(e) - ftv = self.free_type_variables() | supertype.free_type_variables() - sub1 = Substitutions({v: t for v, t in sub.items() if v in ftv}) - sub1.subtyping_provenance = sub.subtyping_provenance - return SubtypeExplanation(sub1) - # No <= implementation using subtyping, because variables overload that for # sort by identity. def __eq__(self, other: object) -> bool: + from concat.typecheck import Substitutions + if self is other: return True if not isinstance(other, Type): return NotImplemented - # QUESTION: Define == separately from is_subtype_of? - return self.is_subtype_of(other) and other.is_subtype_of(self) # type: ignore + # QUESTION: Define == separately from subtyping code? + ftv = self.free_type_variables() | other.free_type_variables() + try: + subtype_sub = self.constrain_and_bind_variables(other, set(), []) + supertype_sub = other.constrain_and_bind_variables(self, set(), []) + except StaticAnalysisError: + return False + subtype_sub = Substitutions( + {v: t for v, t in subtype_sub.items() if v in ftv} + ) + supertype_sub = Substitutions( + {v: t for v, t in supertype_sub.items() if v in ftv} + ) + return not subtype_sub and not supertype_sub # NOTE: Avoid hashing types. I might I'm having correctness issues related # to hashing that I'd rather avoid entirely. Maybe one day I'll introduce @@ -143,16 +118,6 @@ def constrain_and_bind_variables( ) -> 'Substitutions': raise NotImplementedError - # QUESTION: Should I remove this? Should I not distinguish between subtype - # and supertype variables in the other two constraint methods? I should - # look bidirectional typing with polymorphism/generics. Maybe 'Complete and - # Easy'? - def constrain(self, supertype: 'Type') -> None: - if not self.is_subtype_of(supertype): - raise ConcatTypeError( - '{} is not a subtype of {}'.format(self, supertype) - ) - def instantiate(self) -> 'Type': return self From b651a94a0eeeb95076a41e09f397628958424e20 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Fri, 11 Oct 2024 14:34:06 -0700 Subject: [PATCH 59/61] Raise type error if end of Python function constraint algorithm is reached --- concat/typecheck/types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index c934613..0325fcf 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -1539,6 +1539,7 @@ def constrain_and_bind_variables( sub = Substitutions() sub.add_subtyping_provenance((self, supertype)) return sub + raise ConcatTypeError(f'{self} is not a subtype of {supertype}') class _PythonOverloadedType(Type): From d0b6271a7134196c9caad2d8bd3af5281465f1af Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Sat, 12 Oct 2024 01:48:08 -0700 Subject: [PATCH 60/61] Test typechecking of simple import forms --- concat/tests/fixtures/imported_module.cati | 0 concat/tests/typecheck/test_imports.py | 39 ++++++++++++++++++++++ concat/typecheck/__init__.py | 37 ++++++++++++-------- concat/typecheck/types.py | 10 ++++-- 4 files changed, 70 insertions(+), 16 deletions(-) create mode 100644 concat/tests/fixtures/imported_module.cati create mode 100644 concat/tests/typecheck/test_imports.py diff --git a/concat/tests/fixtures/imported_module.cati b/concat/tests/fixtures/imported_module.cati new file mode 100644 index 0000000..e69de29 diff --git a/concat/tests/typecheck/test_imports.py b/concat/tests/typecheck/test_imports.py new file mode 100644 index 0000000..821b9a5 --- /dev/null +++ b/concat/tests/typecheck/test_imports.py @@ -0,0 +1,39 @@ +from concat.parse import ImportStatementNode +from concat.typecheck import Environment, infer +from concat.typecheck.types import ( + BoundVariable, + GenericType, + SequenceKind, + StackEffect, + TypeSequence, + get_module_type, +) +import pathlib +from unittest import TestCase + + +class TestImports(TestCase): + def test_import_generates_module_type(self) -> None: + """Test that imports generate a module type for the right namespace.""" + test_module_path = ( + pathlib.Path(__file__) / '../../fixtures/' + ).resolve() + env = infer( + Environment(), + [ImportStatementNode('imported_module')], + is_top_level=True, + source_dir=test_module_path, + )[2] + ty = env['imported_module'] + seq_var = BoundVariable(kind=SequenceKind) + ty.constrain_and_bind_variables( + GenericType( + [seq_var], + StackEffect( + TypeSequence([seq_var]), + TypeSequence([seq_var, get_module_type()]), + ), + ), + set(), + [], + ) diff --git a/concat/typecheck/__init__.py b/concat/typecheck/__init__.py index bf7fe90..79bed40 100644 --- a/concat/typecheck/__init__.py +++ b/concat/typecheck/__init__.py @@ -479,7 +479,7 @@ def infer( if node.asname is not None: gamma[node.asname] = current_subs( _generate_type_of_innermost_module( - node.value + node.value, source_dir=pathlib.Path(source_dir) ).generalized_wrt(current_subs(gamma)) ) else: @@ -490,7 +490,7 @@ def infer( # should implement namespaces properly. gamma[components[0]] = current_subs( _generate_module_type( - components, source_dir=source_dir + components, source_dir=pathlib.Path(source_dir) ) ) elif isinstance(node, concat.parse.FuncdefStatementNode): @@ -709,14 +709,20 @@ def infer( return current_subs, current_effect, gamma -def _find_stub_path(module_parts: Sequence[str]) -> pathlib.Path: - module_spec = None - path = None +def _find_stub_path( + module_parts: Sequence[str], source_dir: pathlib.Path +) -> pathlib.Path: if module_parts[0] in sys.builtin_module_names: stub_path = pathlib.Path(__file__) / '../builtin_stubs' for part in module_parts: stub_path = stub_path / part else: + module_spec = None + path: Optional[List[str]] + if source_dir is not None: + path = [str(source_dir)] + sys.path + else: + path = sys.path for module_prefix in itertools.accumulate( module_parts, lambda a, b: f'{a}.{b}' ): @@ -725,7 +731,10 @@ def _find_stub_path(module_parts: Sequence[str]) -> pathlib.Path: if module_spec is not None: path = module_spec.submodule_search_locations break - assert module_spec is not None + if module_spec is None: + raise TypeError( + f'Cannot find module {".".join(module_parts)} from source dir {source_dir}' + ) module_path = module_spec.origin if module_path is None: raise TypeError( @@ -1334,8 +1343,10 @@ def object_type_parser() -> Generator: _seq_var = SequenceVariable() -def _generate_type_of_innermost_module(qualified_name: str,) -> StackEffect: - stub_path = _find_stub_path(str.split('.')) +def _generate_type_of_innermost_module( + qualified_name: str, source_dir: pathlib.Path +) -> StackEffect: + stub_path = _find_stub_path(qualified_name.split('.'), source_dir) init_env = load_builtins_and_preamble() module_attributes = _check_stub(stub_path, init_env) module_type_brand = get_module_type().unroll().brand # type: ignore @@ -1350,7 +1361,7 @@ def _generate_type_of_innermost_module(qualified_name: str,) -> StackEffect: def _generate_module_type( components: Sequence[str], _full_name: Optional[str] = None, source_dir='.' -) -> ObjectType: +) -> 'Type': if _full_name is None: _full_name = '.'.join(components) if len(components) > 1: @@ -1371,12 +1382,12 @@ def _generate_module_type( effect = StackEffect( TypeSequence([_seq_var]), TypeSequence([_seq_var, module_t]) ) - return GenericType([_seq_var], ObjectType({'__call__': effect,})) + return GenericType([_seq_var], effect) else: - innermost_type = _generate_type_of_innermost_module(_full_name,) - return GenericType( - [_seq_var], ObjectType({'__call__': innermost_type,}) + innermost_type = _generate_type_of_innermost_module( + _full_name, source_dir=pathlib.Path(source_dir) ) + return GenericType([_seq_var], innermost_type) def _ensure_type( diff --git a/concat/typecheck/types.py b/concat/typecheck/types.py index 0325fcf..d288671 100644 --- a/concat/typecheck/types.py +++ b/concat/typecheck/types.py @@ -597,15 +597,19 @@ def _free_type_variables(self) -> InsertionOrderedSet['Variable']: class TypeSequence(Type, Iterable[Type]): def __init__(self, sequence: Sequence[Type]) -> None: super().__init__() - self._rest: Optional[SequenceVariable] - if sequence and isinstance(sequence[0], SequenceVariable): + self._rest: Optional[Variable] + if ( + sequence + and sequence[0].kind is SequenceKind + and isinstance(sequence[0], Variable) + ): self._rest = sequence[0] self._individual_types = sequence[1:] else: self._rest = None self._individual_types = sequence for ty in self._individual_types: - if ty.kind == SequenceKind: + if ty.kind is SequenceKind: raise ConcatTypeError(f'{ty} cannot be a sequence type') def as_sequence(self) -> Sequence[Type]: From 55706b5950eb64e9e8d074801d66a09d2e6f0146 Mon Sep 17 00:00:00 2001 From: Jason Manuel Date: Sat, 12 Oct 2024 11:25:48 -0700 Subject: [PATCH 61/61] Run nose2 under coverage --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index c202027..fdf6626 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -30,7 +30,7 @@ jobs: pip install -e .[test] - name: Test run: | - nose2 --pretty-assert concat.tests + coverage run -m nose2 --pretty-assert concat.tests - name: Collect coverage into one file run: | coverage combine