clvm mypie (#16898)

* clvm mypie

* update exclusions

* wip

* some

* undo for now

* more

* more

* drop the program usage in datalayer tests

* ignore tree hash errors

* ignore enough in _sexp_replace...

* again

* almost

* Fix test_singleton.py

* dl fixup

* did fixup

* touchup

* nft fixup

* fixup tests

* Revert "fixup tests"

This reverts commit 5df1a70.

* or this i guess

* blah

* Fix get_owner_did

* tidy

* blah

* `.as_atom()`

* `.as_int()`

* Update chia/wallet/nft_wallet/nft_puzzles.py

* `rrrff`

* stop using `CastableType`

* undo stuff around using `CastableType`

* `announcement_assertions: List[Program]`

* Update chia/types/blockchain_format/program.py

* more

* clvm==0.9.10

---------

Co-authored-by: Amine Khaldi <amine.khaldi@reactos.org>
Co-authored-by: Matt <quexington@gmail.com>

benchmarks/mempool-long-lived.py

@@ -11,7 +11,7 @@
 from chia.consensus.default_constants import DEFAULT_CONSTANTS
 from chia.full_node.mempool_manager import MempoolManager
 from chia.types.blockchain_format.coin import Coin
-from chia.types.blockchain_format.program import Program
+from chia.types.blockchain_format.serialized_program import SerializedProgram
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.types.coin_record import CoinRecord
 from chia.types.coin_spend import CoinSpend
@@ -44,7 +44,7 @@ def is_transaction_block(self) -> bool:
         return self.timestamp is not None
 
 
-IDENTITY_PUZZLE = Program.to(1)
+IDENTITY_PUZZLE = SerializedProgram.to(1)
 IDENTITY_PUZZLE_HASH = IDENTITY_PUZZLE.get_tree_hash()
 
 
@@ -62,7 +62,7 @@ def make_spend_bundle(coin: Coin, height: int) -> SpendBundle:
             int_to_bytes(coin.amount // 2 - height * 10),
         ],
     ]
-    spend = CoinSpend(coin, IDENTITY_PUZZLE, Program.to(conditions))
+    spend = CoinSpend(coin, IDENTITY_PUZZLE, SerializedProgram.to(conditions))
     return SpendBundle([spend], G2Element())
 
 

chia/_tests/core/data_layer/test_data_store.py

@@ -259,7 +259,7 @@ async def test_build_a_tree(
     example = await create_example(data_store, store_id)
 
     await _debug_dump(db=data_store.db_wrapper, description="final")
-    actual = await data_store.get_tree_as_program(store_id=store_id)
+    actual = await data_store.get_tree_as_nodes(store_id=store_id)
     # print("actual  ", actual.as_python())
     # print("expected", example.expected.as_python())
     assert actual == example.expected
@@ -745,30 +745,28 @@ async def test_autoinsert_balances_gaps(data_store: DataStore, store_id: bytes32
 async def test_delete_from_left_both_terminal(data_store: DataStore, store_id: bytes32) -> None:
     await add_01234567_example(data_store=data_store, store_id=store_id)
 
-    expected = Program.to(
-        (
-            (
-                (
-                    (b"\x00", b"\x10\x00"),
-                    (b"\x01", b"\x11\x01"),
-                ),
-                (
-                    (b"\x02", b"\x12\x02"),
-                    (b"\x03", b"\x13\x03"),
-                ),
+    expected = InternalNode.from_child_nodes(
+        left=InternalNode.from_child_nodes(
+            left=InternalNode.from_child_nodes(
+                left=TerminalNode.from_key_value(key=b"\x00", value=b"\x10\x00"),
+                right=TerminalNode.from_key_value(key=b"\x01", value=b"\x11\x01"),
             ),
-            (
-                (b"\x05", b"\x15\x05"),
-                (
-                    (b"\x06", b"\x16\x06"),
-                    (b"\x07", b"\x17\x07"),
-                ),
+            right=InternalNode.from_child_nodes(
+                left=TerminalNode.from_key_value(key=b"\x02", value=b"\x12\x02"),
+                right=TerminalNode.from_key_value(key=b"\x03", value=b"\x13\x03"),
+            ),
+        ),
+        right=InternalNode.from_child_nodes(
+            left=TerminalNode.from_key_value(key=b"\x05", value=b"\x15\x05"),
+            right=InternalNode.from_child_nodes(
+                left=TerminalNode.from_key_value(key=b"\x06", value=b"\x16\x06"),
+                right=TerminalNode.from_key_value(key=b"\x07", value=b"\x17\x07"),
             ),
         ),
     )
 
     await data_store.delete(key=b"\x04", store_id=store_id, status=Status.COMMITTED)
-    result = await data_store.get_tree_as_program(store_id=store_id)
+    result = await data_store.get_tree_as_nodes(store_id=store_id)
 
     assert result == expected
 
@@ -777,28 +775,26 @@ async def test_delete_from_left_both_terminal(data_store: DataStore, store_id: b
 async def test_delete_from_left_other_not_terminal(data_store: DataStore, store_id: bytes32) -> None:
     await add_01234567_example(data_store=data_store, store_id=store_id)
 
-    expected = Program.to(
-        (
-            (
-                (
-                    (b"\x00", b"\x10\x00"),
-                    (b"\x01", b"\x11\x01"),
-                ),
-                (
-                    (b"\x02", b"\x12\x02"),
-                    (b"\x03", b"\x13\x03"),
-                ),
+    expected = InternalNode.from_child_nodes(
+        left=InternalNode.from_child_nodes(
+            left=InternalNode.from_child_nodes(
+                left=TerminalNode.from_key_value(key=b"\x00", value=b"\x10\x00"),
+                right=TerminalNode.from_key_value(key=b"\x01", value=b"\x11\x01"),
             ),
-            (
-                (b"\x06", b"\x16\x06"),
-                (b"\x07", b"\x17\x07"),
+            right=InternalNode.from_child_nodes(
+                left=TerminalNode.from_key_value(key=b"\x02", value=b"\x12\x02"),
+                right=TerminalNode.from_key_value(key=b"\x03", value=b"\x13\x03"),
             ),
         ),
+        right=InternalNode.from_child_nodes(
+            left=TerminalNode.from_key_value(key=b"\x06", value=b"\x16\x06"),
+            right=TerminalNode.from_key_value(key=b"\x07", value=b"\x17\x07"),
+        ),
     )
 
     await data_store.delete(key=b"\x04", store_id=store_id, status=Status.COMMITTED)
     await data_store.delete(key=b"\x05", store_id=store_id, status=Status.COMMITTED)
-    result = await data_store.get_tree_as_program(store_id=store_id)
+    result = await data_store.get_tree_as_nodes(store_id=store_id)
 
     assert result == expected
 
@@ -807,30 +803,28 @@ async def test_delete_from_left_other_not_terminal(data_store: DataStore, store_
 async def test_delete_from_right_both_terminal(data_store: DataStore, store_id: bytes32) -> None:
     await add_01234567_example(data_store=data_store, store_id=store_id)
 
-    expected = Program.to(
-        (
-            (
-                (
-                    (b"\x00", b"\x10\x00"),
-                    (b"\x01", b"\x11\x01"),
-                ),
-                (b"\x02", b"\x12\x02"),
+    expected = InternalNode.from_child_nodes(
+        left=InternalNode.from_child_nodes(
+            left=InternalNode.from_child_nodes(
+                left=TerminalNode.from_key_value(key=b"\x00", value=b"\x10\x00"),
+                right=TerminalNode.from_key_value(key=b"\x01", value=b"\x11\x01"),
             ),
-            (
-                (
-                    (b"\x04", b"\x14\x04"),
-                    (b"\x05", b"\x15\x05"),
-                ),
-                (
-                    (b"\x06", b"\x16\x06"),
-                    (b"\x07", b"\x17\x07"),
-                ),
+            right=TerminalNode.from_key_value(key=b"\x02", value=b"\x12\x02"),
+        ),
+        right=InternalNode.from_child_nodes(
+            left=InternalNode.from_child_nodes(
+                left=TerminalNode.from_key_value(key=b"\x04", value=b"\x14\x04"),
+                right=TerminalNode.from_key_value(key=b"\x05", value=b"\x15\x05"),
+            ),
+            right=InternalNode.from_child_nodes(
+                left=TerminalNode.from_key_value(key=b"\x06", value=b"\x16\x06"),
+                right=TerminalNode.from_key_value(key=b"\x07", value=b"\x17\x07"),
             ),
         ),
     )
 
     await data_store.delete(key=b"\x03", store_id=store_id, status=Status.COMMITTED)
-    result = await data_store.get_tree_as_program(store_id=store_id)
+    result = await data_store.get_tree_as_nodes(store_id=store_id)
 
     assert result == expected
 
@@ -839,28 +833,26 @@ async def test_delete_from_right_both_terminal(data_store: DataStore, store_id:
 async def test_delete_from_right_other_not_terminal(data_store: DataStore, store_id: bytes32) -> None:
     await add_01234567_example(data_store=data_store, store_id=store_id)
 
-    expected = Program.to(
-        (
-            (
-                (b"\x00", b"\x10\x00"),
-                (b"\x01", b"\x11\x01"),
+    expected = InternalNode.from_child_nodes(
+        left=InternalNode.from_child_nodes(
+            left=TerminalNode.from_key_value(key=b"\x00", value=b"\x10\x00"),
+            right=TerminalNode.from_key_value(key=b"\x01", value=b"\x11\x01"),
+        ),
+        right=InternalNode.from_child_nodes(
+            left=InternalNode.from_child_nodes(
+                left=TerminalNode.from_key_value(key=b"\x04", value=b"\x14\x04"),
+                right=TerminalNode.from_key_value(key=b"\x05", value=b"\x15\x05"),
             ),
-            (
-                (
-                    (b"\x04", b"\x14\x04"),
-                    (b"\x05", b"\x15\x05"),
-                ),
-                (
-                    (b"\x06", b"\x16\x06"),
-                    (b"\x07", b"\x17\x07"),
-                ),
+            right=InternalNode.from_child_nodes(
+                left=TerminalNode.from_key_value(key=b"\x06", value=b"\x16\x06"),
+                right=TerminalNode.from_key_value(key=b"\x07", value=b"\x17\x07"),
             ),
         ),
     )
 
     await data_store.delete(key=b"\x03", store_id=store_id, status=Status.COMMITTED)
     await data_store.delete(key=b"\x02", store_id=store_id, status=Status.COMMITTED)
-    result = await data_store.get_tree_as_program(store_id=store_id)
+    result = await data_store.get_tree_as_nodes(store_id=store_id)
 
     assert result == expected
 

chia/_tests/core/data_layer/util.py

@@ -8,7 +8,7 @@
 from dataclasses import dataclass
 from typing import IO, TYPE_CHECKING, Any, Dict, Iterator, List, Literal, Optional, Union, overload
 
-from chia.data_layer.data_layer_util import NodeType, Side, Status
+from chia.data_layer.data_layer_util import InternalNode, Node, NodeType, Side, Status, TerminalNode
 from chia.data_layer.data_store import DataStore
 from chia.types.blockchain_format.program import Program
 from chia.types.blockchain_format.sized_bytes import bytes32
@@ -47,21 +47,19 @@ async def general_insert(
 
 @dataclass(frozen=True)
 class Example:
-    expected: Program
+    expected: Node
     terminal_nodes: List[bytes32]
 
 
 async def add_0123_example(data_store: DataStore, store_id: bytes32) -> Example:
-    expected = Program.to(
-        (
-            (
-                (b"\x00", b"\x10\x00"),
-                (b"\x01", b"\x11\x01"),
-            ),
-            (
-                (b"\x02", b"\x12\x02"),
-                (b"\x03", b"\x13\x03"),
-            ),
+    expected = InternalNode.from_child_nodes(
+        left=InternalNode.from_child_nodes(
+            left=TerminalNode.from_key_value(key=b"\x00", value=b"\x10\x00"),
+            right=TerminalNode.from_key_value(key=b"\x01", value=b"\x11\x01"),
+        ),
+        right=InternalNode.from_child_nodes(
+            left=TerminalNode.from_key_value(key=b"\x02", value=b"\x12\x02"),
+            right=TerminalNode.from_key_value(key=b"\x03", value=b"\x13\x03"),
         ),
     )
 
@@ -76,27 +74,25 @@ async def add_0123_example(data_store: DataStore, store_id: bytes32) -> Example:
 
 
 async def add_01234567_example(data_store: DataStore, store_id: bytes32) -> Example:
-    expected = Program.to(
-        (
-            (
-                (
-                    (b"\x00", b"\x10\x00"),
-                    (b"\x01", b"\x11\x01"),
-                ),
-                (
-                    (b"\x02", b"\x12\x02"),
-                    (b"\x03", b"\x13\x03"),
-                ),
+    expected = InternalNode.from_child_nodes(
+        left=InternalNode.from_child_nodes(
+            InternalNode.from_child_nodes(
+                left=TerminalNode.from_key_value(key=b"\x00", value=b"\x10\x00"),
+                right=TerminalNode.from_key_value(key=b"\x01", value=b"\x11\x01"),
+            ),
+            InternalNode.from_child_nodes(
+                left=TerminalNode.from_key_value(key=b"\x02", value=b"\x12\x02"),
+                right=TerminalNode.from_key_value(key=b"\x03", value=b"\x13\x03"),
+            ),
+        ),
+        right=InternalNode.from_child_nodes(
+            InternalNode.from_child_nodes(
+                left=TerminalNode.from_key_value(key=b"\x04", value=b"\x14\x04"),
+                right=TerminalNode.from_key_value(key=b"\x05", value=b"\x15\x05"),
             ),
-            (
-                (
-                    (b"\x04", b"\x14\x04"),
-                    (b"\x05", b"\x15\x05"),
-                ),
-                (
-                    (b"\x06", b"\x16\x06"),
-                    (b"\x07", b"\x17\x07"),
-                ),
+            InternalNode.from_child_nodes(
+                left=TerminalNode.from_key_value(key=b"\x06", value=b"\x16\x06"),
+                right=TerminalNode.from_key_value(key=b"\x07", value=b"\x17\x07"),
             ),
         ),
     )

chia/_tests/generator/test_compression.py

@@ -7,8 +7,8 @@
 
 import pytest
 from chia_rs import ALLOW_BACKREFS
-from clvm import SExp
 from clvm.serialize import sexp_from_stream
+from clvm.SExp import SExp
 from clvm_tools import binutils
 
 from chia._tests.core.make_block_generator import make_spend_bundle

chia/_tests/generator/test_rom.py

@@ -2,6 +2,7 @@
 
 from typing import List, Tuple
 
+from clvm.CLVMObject import CLVMStorage
 from clvm_tools import binutils
 from clvm_tools.clvmc import compile_clvm_text
 
@@ -84,7 +85,7 @@ def run_generator(self: BlockGenerator) -> Tuple[int, Program]:
     return GENERATOR_MOD.run_with_cost(MAX_COST, [self.program, args])
 
 
-def as_atom_list(prg: Program) -> List[bytes]:
+def as_atom_list(prg: CLVMStorage) -> List[bytes]:
     """
     Pretend `prg` is a list of atoms. Return the corresponding
     python list of atoms.

chia/_tests/wallet/cat_wallet/test_offer_lifecycle.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any, Dict, List, Optional, cast
+from typing import Any, Dict, List, Optional
 
 import pytest
 from chia_rs import G2Element
@@ -32,8 +32,7 @@
 
 # Some methods mapping strings to CATs
 def str_to_tail(tail_str: str) -> Program:
-    # TODO: Remove cast when we improve typing
-    return cast(Program, Program.to([3, [], [1, tail_str], []]))
+    return Program.to([3, [], [1, tail_str], []])
 
 
 def str_to_tail_hash(tail_str: str) -> bytes32:
@@ -118,10 +117,10 @@ def generate_secure_bundle(
     offered_amount: uint64,
     tail_str: Optional[str] = None,
 ) -> SpendBundle:
-    announcement_assertions = [a.to_program() for a in announcements]
+    announcement_assertions: List[Program] = [a.to_program() for a in announcements]
     selected_coin_amount = sum([c.amount for c in selected_coins])
     non_primaries = [] if len(selected_coins) < 2 else selected_coins[1:]
-    inner_solution: List[List[Any]] = [
+    inner_solution: List[Any] = [
         [51, Offer.ph(), offered_amount],  # Offered coin
         [51, acs_ph, uint64(selected_coin_amount - offered_amount)],  # Change
         *announcement_assertions,

chia/_tests/wallet/dao_wallet/test_dao_clvm.py

@@ -316,11 +316,11 @@ def test_proposal() -> None:
         treasury_inner,
     )
     treasury_puzhash = treasury.get_tree_hash()
-    apa_msg = singleton_id
+    apa_msg_bytes = singleton_id
 
     timer_apa = std_hash(timer_puzhash + singleton_id)
     assert conds[ConditionOpcode.ASSERT_PUZZLE_ANNOUNCEMENT][0].vars[0] == timer_apa
-    assert conds[ConditionOpcode.ASSERT_PUZZLE_ANNOUNCEMENT][1].vars[0] == std_hash(treasury_puzhash + apa_msg)
+    assert conds[ConditionOpcode.ASSERT_PUZZLE_ANNOUNCEMENT][1].vars[0] == std_hash(treasury_puzhash + apa_msg_bytes)
 
     # close a failed proposal
     full_proposal = proposal_curry_one.curry(
@@ -347,8 +347,8 @@ def test_proposal() -> None:
         ]
     )
     conds = conditions_dict_for_solution(full_proposal, solution, INFINITE_COST)
-    apa_msg = int_to_bytes(0)
-    assert conds[ConditionOpcode.ASSERT_PUZZLE_ANNOUNCEMENT][1].vars[0] == std_hash(treasury_puzhash + apa_msg)
+    apa_msg_int = int_to_bytes(0)
+    assert conds[ConditionOpcode.ASSERT_PUZZLE_ANNOUNCEMENT][1].vars[0] == std_hash(treasury_puzhash + apa_msg_int)
     assert conds[ConditionOpcode.ASSERT_PUZZLE_ANNOUNCEMENT][0].vars[0] == timer_apa
 
     finished_puz = DAO_FINISHED_STATE.curry(singleton_struct, DAO_FINISHED_STATE_HASH)
@@ -373,7 +373,7 @@ def test_proposal() -> None:
         ]
     )
     conds = conditions_dict_for_solution(full_proposal, solution, INFINITE_COST)
-    assert conds[ConditionOpcode.ASSERT_PUZZLE_ANNOUNCEMENT][0].vars[0] == std_hash(treasury_puzhash + apa_msg)
+    assert conds[ConditionOpcode.ASSERT_PUZZLE_ANNOUNCEMENT][0].vars[0] == std_hash(treasury_puzhash + apa_msg_int)
     assert conds[ConditionOpcode.CREATE_COIN][0].vars[0] == finished_puz.get_tree_hash()