more mypy (#125)

* start checking mypy and work on some hints

(cherry picked from commit 60e7433)

* remove unused imports

(cherry picked from commit 9741f9c)

* some more mypy

(cherry picked from commit 65b0fb8)

* more

(cherry picked from commit f2a0be4)

* fixup

* little

* catch up on test hints

* more

* fixup

* more

* py.typed

* ignores...

* stuff

* just ignore for now

* note

* tidy

* fix

* fix

* flake8

* typing_externsions.TypeGuard

* Apply suggestions from code review

Co-authored-by: Richard Kiss <him@richardkiss.com>

* add newly needed ignore

* fix NestedTupleOfBytes

* `CLVMObjectLike` -> `CLVMStorage`

* tidy hinting in `core_ops`

* unused assignment in tests

* typing_extensions.Never

* tidy

* tidy

* more tidy

* oops

* update note

* shift around

* more

* undo

* tidy

* tidy

* tidy

* prepare for more strict mypy

* enable 'free' mypy constraints

* no_implicit_reexport

* some untyped generics

* almost, but not quite

* ToCLVMStorage

* break circular import

* touchup

* future

* less future

* `SExp.to(Any)`

* remove ignores

* use `PythonReturnType` for as python return type

* drop any

* hmm

* Revert "hmm"

This reverts commit bfe9e84.

* Revert "drop any"

This reverts commit 1f3f89b.

* Revert "use `PythonReturnType` for as python return type"

This reverts commit f806e68.

---------

Co-authored-by: Richard Kiss <him@richardkiss.com>

.github/workflows/main.yml

@@ -40,6 +40,8 @@ jobs:
           python -m pip install flake8
       - name: flake8
         run: flake8 clvm tests --max-line-length=120
+      - name: mypy
+        run: mypy
       - name: Test with pytest
         run: |
           pytest tests

clvm/CLVMObject.py

@@ -1,6 +1,22 @@
+from __future__ import annotations
+
 import typing
 
 
+class CLVMStorage(typing.Protocol):
+    # It's not clear if it is possible to express the exclusivity without maybe
+    # restructuring all the classes, such as having a separate instance for each
+    # of the atom and pair cases and hinting a union of a protocol of each type.
+    atom: typing.Optional[bytes]
+    pair: typing.Optional[PairType]
+
+
+PairType = typing.Tuple[CLVMStorage, CLVMStorage]
+
+
+_T_CLVMObject = typing.TypeVar("_T_CLVMObject", bound="CLVMObject")
+
+
 class CLVMObject:
     """
     This class implements the CLVM Object protocol in the simplest possible way,
@@ -11,19 +27,26 @@ class CLVMObject:
 
     # this is always a 2-tuple of an object implementing the CLVM object
     # protocol.
-    pair: typing.Optional[typing.Tuple[typing.Any, typing.Any]]
+    pair: typing.Optional[PairType]
     __slots__ = ["atom", "pair"]
 
-    def __new__(class_, v):
-        if isinstance(v, CLVMObject):
+    @staticmethod
+    def __new__(
+        class_: typing.Type[_T_CLVMObject],
+        v: typing.Union[_T_CLVMObject, bytes, PairType],
+    ) -> _T_CLVMObject:
+        if isinstance(v, class_):
             return v
+        # mypy does not realize that the isinstance check is type narrowing like this
+        narrowed_v: typing.Union[bytes, PairType] = v  # type: ignore[assignment]
+
         self = super(CLVMObject, class_).__new__(class_)
-        if isinstance(v, tuple):
-            if len(v) != 2:
-                raise ValueError("tuples must be of size 2, cannot create CLVMObject from: %s" % str(v))
-            self.pair = v
+        if isinstance(narrowed_v, tuple):
+            if len(narrowed_v) != 2:
+                raise ValueError("tuples must be of size 2, cannot create CLVMObject from: %s" % str(narrowed_v))
+            self.pair = narrowed_v
             self.atom = None
         else:
-            self.atom = v
+            self.atom = narrowed_v
             self.pair = None
         return self

clvm/EvalError.py

@@ -1,4 +1,12 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from clvm.SExp import SExp
+
+
 class EvalError(Exception):
-    def __init__(self, message: str, sexp):
+    def __init__(self, message: str, sexp: SExp) -> None:
         super().__init__(message)
         self._sexp = sexp

clvm/SExp.py

@@ -1,9 +1,12 @@
+from __future__ import annotations
+
 import io
 import typing
 
+import typing_extensions
 
 from .as_python import as_python
-from .CLVMObject import CLVMObject
+from .CLVMObject import CLVMObject, CLVMStorage
 
 from .EvalError import EvalError
 
@@ -16,27 +19,28 @@
 
 CastableType = typing.Union[
     "SExp",
-    "CLVMObject",
+    CLVMStorage,
+    typing.SupportsBytes,
     bytes,
     str,
     int,
     None,
-    list,
-    typing.Tuple[typing.Any, typing.Any],
+    typing.Sequence["CastableType"],
+    typing.Tuple["CastableType", "CastableType"],
 ]
 
 
 NULL = b""
 
 
-def looks_like_clvm_object(o: typing.Any) -> bool:
+def looks_like_clvm_object(o: typing.Any) -> typing_extensions.TypeGuard[CLVMStorage]:
     d = dir(o)
     return "atom" in d and "pair" in d
 
 
 # this function recognizes some common types and turns them into plain bytes,
 def convert_atom_to_bytes(
-    v: typing.Union[bytes, str, int, None, list],
+    v: typing.Union[bytes, str, int, None, typing.List[typing_extensions.Never], typing.SupportsBytes],
 ) -> bytes:
 
     if isinstance(v, bytes):
@@ -55,10 +59,15 @@ def convert_atom_to_bytes(
     raise ValueError("can't cast %s (%s) to bytes" % (type(v), v))
 
 
+ValType = typing.Union["SExp", CastableType]
+StackType = typing.List[ValType]
+
+
 # returns a clvm-object like object
+@typing.no_type_check
 def to_sexp_type(
     v: CastableType,
-):
+) -> CLVMStorage:
     stack = [v]
     ops = [(0, None)]  # convert
 
@@ -115,6 +124,9 @@ def to_sexp_type(
     return stack[0]
 
 
+_T_SExp = typing.TypeVar("_T_SExp", bound="SExp")
+
+
 class SExp:
     """
     SExp provides higher level API on top of any object implementing the CLVM
@@ -129,86 +141,89 @@ class SExp:
        elements implementing the CLVM object protocol.
     Exactly one of "atom" and "pair" must be None.
     """
-    true: "SExp"
-    false: "SExp"
-    __null__: "SExp"
+    true: typing.ClassVar[SExp]
+    false: typing.ClassVar[SExp]
+    __null__: typing.ClassVar[SExp]
 
     # the underlying object implementing the clvm object protocol
     atom: typing.Optional[bytes]
 
     # this is a tuple of the otherlying CLVMObject-like objects. i.e. not
     # SExp objects with higher level functions, or None
-    pair: typing.Optional[typing.Tuple[typing.Any, typing.Any]]
+    pair: typing.Optional[typing.Tuple[CLVMStorage, CLVMStorage]]
 
-    def __init__(self, obj):
+    def __init__(self, obj: CLVMStorage) -> None:
         self.atom = obj.atom
         self.pair = obj.pair
 
     # this returns a tuple of two SExp objects, or None
-    def as_pair(self) -> typing.Tuple["SExp", "SExp"]:
+    def as_pair(self) -> typing.Optional[typing.Tuple[SExp, SExp]]:
         pair = self.pair
         if pair is None:
             return pair
         return (self.__class__(pair[0]), self.__class__(pair[1]))
 
     # TODO: deprecate this. Same as .atom property
-    def as_atom(self):
+    def as_atom(self) -> typing.Optional[bytes]:
         return self.atom
 
-    def listp(self):
+    def listp(self) -> bool:
         return self.pair is not None
 
-    def nullp(self):
+    def nullp(self) -> bool:
         v = self.atom
         return v is not None and len(v) == 0
 
-    def as_int(self):
+    def as_int(self) -> int:
+        if self.atom is None:
+            raise TypeError("Unable to convert a pair to an int")
         return int_from_bytes(self.atom)
 
-    def as_bin(self):
+    def as_bin(self) -> bytes:
         f = io.BytesIO()
         sexp_to_stream(self, f)
         return f.getvalue()
 
+    # TODO: should be `v: CastableType`
     @classmethod
-    def to(class_, v: CastableType) -> "SExp":
-        if isinstance(v, class_):
+    def to(cls: typing.Type[_T_SExp], v: typing.Any) -> _T_SExp:
+        if isinstance(v, cls):
             return v
 
         if looks_like_clvm_object(v):
-            return class_(v)
+            return cls(v)
 
         # this will lazily convert elements
-        return class_(to_sexp_type(v))
+        return cls(to_sexp_type(v))
 
-    def cons(self, right):
+    def cons(self: _T_SExp, right: _T_SExp) -> _T_SExp:
         return self.to((self, right))
 
-    def first(self):
+    def first(self: _T_SExp) -> _T_SExp:
         pair = self.pair
         if pair:
             return self.__class__(pair[0])
         raise EvalError("first of non-cons", self)
 
-    def rest(self):
+    def rest(self: _T_SExp) -> _T_SExp:
         pair = self.pair
         if pair:
             return self.__class__(pair[1])
         raise EvalError("rest of non-cons", self)
 
     @classmethod
-    def null(class_):
+    def null(class_) -> SExp:
         return class_.__null__
 
-    def as_iter(self):
+    def as_iter(self: _T_SExp) -> typing.Iterator[_T_SExp]:
         v = self
         while not v.nullp():
             yield v.first()
             v = v.rest()
 
-    def __eq__(self, other: CastableType):
+    def __eq__(self, other: object) -> bool:
         try:
-            other = self.to(other)
+            other = self.to(typing.cast(CastableType, other))
             to_compare_stack = [(self, other)]
             while to_compare_stack:
                 s1, s2 = to_compare_stack.pop()
@@ -226,21 +241,21 @@ def __eq__(self, other: CastableType):
         except ValueError:
             return False
 
-    def list_len(self):
+    def list_len(self) -> int:
         v = self
         size = 0
         while v.listp():
             size += 1
             v = v.rest()
         return size
 
-    def as_python(self):
+    def as_python(self) -> typing.Any:
         return as_python(self)
 
-    def __str__(self):
+    def __str__(self) -> str:
         return self.as_bin().hex()
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return "%s(%s)" % (self.__class__.__name__, str(self))
 
 

clvm/as_python.py

@@ -1,37 +1,61 @@
-def as_python(sexp):
-    def _roll(op_stack, val_stack):
-        v1 = val_stack.pop()
-        v2 = val_stack.pop()
-        val_stack.append(v1)
-        val_stack.append(v2)
-
-    def _make_tuple(op_stack, val_stack):
-        left = val_stack.pop()
-        right = val_stack.pop()
-        if right == b"":
-            val_stack.append([left])
-        elif isinstance(right, list):
-            v = [left] + right
-            val_stack.append(v)
-        else:
-            val_stack.append((left, right))
-
-    def _as_python(op_stack, val_stack):
-        t = val_stack.pop()
-        pair = t.as_pair()
-        if pair:
-            left, right = pair
-            op_stack.append(_make_tuple)
-            op_stack.append(_as_python)
-            op_stack.append(_roll)
-            op_stack.append(_as_python)
-            val_stack.append(left)
-            val_stack.append(right)
-        else:
-            val_stack.append(t.as_atom())
-
-    op_stack = [_as_python]
-    val_stack = [sexp]
+from __future__ import annotations
+
+from typing import Any, Callable, List, Tuple, TYPE_CHECKING, Union
+
+if TYPE_CHECKING:
+    from clvm.SExp import SExp
+
+OpCallable = Callable[["OpStackType", "ValStackType"], None]
+
+PythonReturnType = Union[bytes, Tuple["PythonReturnType", "PythonReturnType"], List["PythonReturnType"]]
+
+ValType = Union["SExp", PythonReturnType]
+ValStackType = List[ValType]
+
+OpStackType = List[OpCallable]
+
+
+def _roll(op_stack: OpStackType, val_stack: ValStackType) -> None:
+    v1 = val_stack.pop()
+    v2 = val_stack.pop()
+    val_stack.append(v1)
+    val_stack.append(v2)
+
+
+MakeTupleValStackType = List[Union[bytes, Tuple[object, object], "MakeTupleValStackType"]]
+
+
+def _make_tuple(op_stack: OpStackType, val_stack: ValStackType) -> None:
+    left: PythonReturnType = val_stack.pop()  # type: ignore[assignment]
+    right: PythonReturnType = val_stack.pop()  # type: ignore[assignment]
+    if right == b"":
+        val_stack.append([left])
+    elif isinstance(right, list):
+        v = [left] + right
+        val_stack.append(v)
+    else:
+        val_stack.append((left, right))
+
+
+def _as_python(op_stack: OpStackType, val_stack: ValStackType) -> None:
+    t: SExp = val_stack.pop()  # type: ignore[assignment]
+    pair = t.as_pair()
+    if pair:
+        left, right = pair
+        op_stack.append(_make_tuple)
+        op_stack.append(_as_python)
+        op_stack.append(_roll)
+        op_stack.append(_as_python)
+        val_stack.append(left)
+        val_stack.append(right)
+    else:
+        # we know that t.atom is not None here because the pair is None
+        val_stack.append(t.atom)  # type:ignore[arg-type]
+
+
+def as_python(sexp: SExp) -> Any:
+    op_stack: OpStackType = [_as_python]
+    val_stack: ValStackType = [sexp]
     while op_stack:
         op_f = op_stack.pop()
         op_f(op_stack, val_stack)