Further refactoring including address- and txlist to wallet package

src/cryptoadvance/specter/liquid/addresslist.py

@@ -1,4 +1,4 @@
-from ..addresslist import *
+from ..wallet.addresslist import *
 from embit.liquid.addresses import addr_decode, to_unconfidential
 
 

src/cryptoadvance/specter/liquid/txlist.py

@@ -1,4 +1,4 @@
-from ..txlist import *
+from ..wallet.txlist import *
 from embit.liquid.transaction import LTransaction, TxOutWitness, unblind
 from embit.liquid.pset import PSET
 from embit.liquid import slip77

src/cryptoadvance/specter/liquid/wallet.py

@@ -8,12 +8,12 @@
 from embit.liquid.pset import PSET
 from embit.liquid.transaction import LTransaction
 
-from ..addresslist import Address
 from ..specter_error import SpecterError
 from ..wallet import *
 from .addresslist import LAddressList
 from .txlist import LTxList
 from .util.pset import SpecterPSET
+from .addresslist import Address
 
 
 class LWallet(Wallet):

src/cryptoadvance/specter/server_endpoints/wallets/wallets_api.py

@@ -21,7 +21,7 @@
 from flask_login import current_user, login_required
 from werkzeug.wrappers import Response
 
-from cryptoadvance.specter.txlist import WalletAwareTxItem
+from cryptoadvance.specter.wallet.txlist import WalletAwareTxItem
 
 from ...commands.psbt_creator import PsbtCreator
 from ...helpers import bcur2base64

src/cryptoadvance/specter/services/service.py

@@ -13,7 +13,6 @@
 from .service_encrypted_storage import ServiceEncryptedStorageManager
 from .service_annotations_storage import ServiceAnnotationsStorage
 
-from cryptoadvance.specter.addresslist import Address
 from cryptoadvance.specter.services import callbacks
 
 

src/cryptoadvance/specter/wallet/__init__.py

@@ -1 +1,3 @@
 from .wallet import Wallet, purposes
+from .addresslist import Address
+from .txlist import WalletAwareTxItem

src/cryptoadvance/specter/wallet/abstract_wallet.py

@@ -0,0 +1,27 @@
+from .txlist import TxList
+import os
+from abc import ABC, abstractmethod
+
+
+class AbstractWallet:
+    @property
+    def transactions(self) -> TxList:
+        if hasattr(self, "_transactions"):
+            return self._transactions
+        else:
+            return None
+
+    @property
+    def addresses(self) -> TxList:
+        if hasattr(self, "_addresses"):
+            return self._addresses
+        else:
+            return None
+
+    @property
+    @abstractmethod
+    def rpc(self):
+        """Cache RPC instance. Reuse if manager's RPC instance hasn't changed. Create new RPC instance otherwise.
+        This RPC instance is also used by objects created by the wallet, such as TxList or TxItem
+        """
+        pass

src/cryptoadvance/specter/addresslist.py → src/cryptoadvance/specter/wallet/addresslist.py

@@ -2,7 +2,7 @@
 Manages the list of addresses for the wallet, including labels and derivation paths
 """
 import os
-from .persistence import write_csv, read_csv
+from ..persistence import write_csv, read_csv
 import logging
 
 logger = logging.getLogger(__name__)

src/cryptoadvance/specter/wallet/tx_fetcher.py

@@ -1,5 +1,5 @@
 import logging
-from .wallet import Wallet
+from .abstract_wallet import AbstractWallet
 
 logger = logging.getLogger(__name__)
 
@@ -9,9 +9,94 @@ class TxFetcher:
 
     LISTTRANSACTIONS_BATCH_SIZE = 1000
 
-    def __init__(self, wallet: Wallet):
+    def __init__(self, wallet: AbstractWallet):
         self.wallet = wallet
 
+    def _fetch_transactions(self):
+
+        # unconfirmed_selftransfers needed since Bitcoin Core does not properly list `selftransfer` txs in `listtransactions` command
+        # Until v0.21, it listed there consolidations to a receive address, but not change address
+        # Since v0.21, it does not list there consolidations at all
+        # Therefore we need to check here if a transaction might got confirmed
+        # NOTE: This might be a problem in case of re-org...
+        # More details: https://github.com/cryptoadvance/specter-desktop/issues/996
+
+        arr = [
+            tx["result"]
+            for tx in self.unconfirmed_selftransfers_txs
+            if tx.get("result")
+        ]
+        arr.extend(self.interesting_txs())
+        txs = self.transform_to_dict_with_txid_as_key(
+            arr
+        )  # and gettransaction as value
+
+        # fix for core versions < v0.20 (add blockheight if not there)
+        self.fill_blockheight_if_necessary(txs)
+
+        if self.wallet.use_descriptors:
+            # Get all used addresses that belong to the wallet
+
+            addresses_info = self.extract_addresses(txs)
+
+            # representing the highest index of the addresses from the wallet and
+            # the passed addresses
+            (
+                max_used_receiving,
+                max_used_change,
+            ) = self.calculate_max_used_from_addresses(addresses_info)
+
+            # If max receiving address bigger than current max receiving index minus the gap limit - self._addresses.max_index(change=False)
+            if (
+                max_used_receiving + self.wallet.GAP_LIMIT
+                > self.wallet._addresses.max_index(change=False)
+            ):
+                addresses = [
+                    dict(
+                        address=self.wallet.get_address(
+                            idx, change=False, check_keypool=False
+                        ),
+                        index=idx,
+                        change=False,
+                    )
+                    for idx in range(
+                        self.wallet.addresses.max_index(change=False),
+                        max_used_receiving + self.wallet.GAP_LIMIT,
+                    )
+                ]
+                self.wallet.addresses.add(addresses, check_rpc=False)
+
+            # If max change address bigger than current max change index minus the gap limit  - wallet.addresses.max_index(change=True)
+            if (
+                max_used_change + self.wallet.GAP_LIMIT
+                > self.wallet.addresses.max_index(change=True)
+            ):
+                # Add change addresses until the new max address plus the GAP_LIMIT
+                change_addresses = [
+                    dict(
+                        address=self.wallet.get_address(
+                            idx, change=True, check_keypool=False
+                        ),
+                        index=idx,
+                        change=True,
+                    )
+                    for idx in range(
+                        self.wallet.addresses.max_index(change=True),
+                        max_used_change + self.wallet.GAP_LIMIT,
+                    )
+                ]
+                self.wallet.addresses.add(change_addresses, check_rpc=False)
+
+        # only delete with confirmed txs
+        self.wallet.delete_spent_pending_psbts(
+            [
+                tx["hex"]
+                for tx in txs.values()
+                if tx.get("confirmations", 0) > 0 or tx.get("blockheight")
+            ]
+        )
+        self.wallet.transactions.add(txs)
+
     def is_interesting_tx(self, tx: dict):
         """transactions that we don't know about,
         # or that it has a different blockhash (reorg / confirmed)
@@ -114,134 +199,67 @@ def unconfirmed_selftransfers_txs(self):
                 self._unconfirmed_selftransfers_txs = []
         return self._unconfirmed_selftransfers_txs
 
-    def _fetch_transactions(self):
-
-        # unconfirmed_selftransfers needed since Bitcoin Core does not properly list `selftransfer` txs in `listtransactions` command
-        # Until v0.21, it listed there consolidations to a receive address, but not change address
-        # Since v0.21, it does not list there consolidations at all
-        # Therefore we need to check here if a transaction might got confirmed
-        # NOTE: This might be a problem in case of re-org...
-        # More details: https://github.com/cryptoadvance/specter-desktop/issues/996
-
-        arr = [
-            tx["result"]
-            for tx in self.unconfirmed_selftransfers_txs
-            if tx.get("result")
+    def extract_addresses(self, txs):
+        """Takes txs (dict with txid as key and the result of gettransaction as value )
+        and extracts all the addresses which
+        * belongs to the wallet
+        * are not yet in self.addresses
+        """
+        potential_relevant_txs = [
+            tx
+            for tx in txs.values()
+            if tx
+            and tx.get("details")
+            and (
+                tx.get("details")[0].get("category") != "send"
+                and tx["details"][0].get("address") not in self.wallet.addresses
+            )
         ]
-        arr.extend(self.interesting_txs())
-        txs = self.transform_to_dict_with_txid_as_key(
-            arr
-        )  # and gettransaction as value
 
-        # fix for core versions < v0.20 (add blockheight if not there)
-        self.fill_blockheight_if_necessary(txs)
-
-        if self.wallet.use_descriptors:
-            # Get all used addresses that belong to the wallet
-
-            potential_relevant_txs = [
-                tx
-                for tx in txs.values()
-                if tx
-                and tx.get("details")
-                and (
-                    tx.get("details")[0].get("category")
-                    != "send"
-                    # and tx["details"][0].get("address") not in self.wallet._addresses
-                )
-            ]
-
-            addresses_info_multi = self.wallet.rpc.multi(
-                [
-                    ("getaddressinfo", address)
-                    for address in [
-                        tx["details"][0].get("address") for tx in potential_relevant_txs
-                    ]
-                    if address
+        addresses_info_multi = self.wallet.rpc.multi(
+            [
+                ("getaddressinfo", address)
+                for address in [
+                    tx["details"][0].get("address") for tx in potential_relevant_txs
                 ]
-            )
-
-            addresses_info = [
-                r["result"]
-                for r in addresses_info_multi
-                if r["result"].get("ismine", False)
+                if address
             ]
+        )
 
-            # Gets max index used receiving and change addresses
-            max_used_receiving = self.wallet._addresses.max_used_index(change=False)
-            max_used_change = self.wallet._addresses.max_used_index(change=True)
-
-            for address in addresses_info:
-                desc = self.wallet.DescriptorCls.from_string(address["desc"])
-                indexes = [
-                    {
-                        "idx": k.origin.derivation[-1],
-                        "change": k.origin.derivation[-2],
-                    }
-                    for k in desc.keys
-                ]
-                for idx in indexes:
-                    if int(idx["change"]) == 0:
-                        max_used_receiving = max(max_used_receiving, int(idx["idx"]))
-                    elif int(idx["change"]) == 1:
-                        max_used_change = max(max_used_change, int(idx["idx"]))
-
-            # If max receiving address bigger than current max receiving index minus the gap limit - self._addresses.max_index(change=False)
-            if (
-                max_used_receiving + self.wallet.GAP_LIMIT
-                > self.wallet._addresses.max_index(change=False)
-            ):
-                print(
-                    "------------+++++++++++++++++++++++++++++++++Add receiving addresses until the new max address plus the GAP_LIMIT"
-                )
-                addresses = [
-                    dict(
-                        address=self.wallet.get_address(
-                            idx, change=False, check_keypool=False
-                        ),
-                        index=idx,
-                        change=False,
-                    )
-                    for idx in range(
-                        self.wallet._addresses.max_index(change=False),
-                        max_used_receiving + self.wallet.GAP_LIMIT,
-                    )
-                ]
-                self.wallet._addresses.add(addresses, check_rpc=False)
+        addresses_info = [
+            r["result"]
+            for r in addresses_info_multi
+            if r["result"].get("ismine", False)
+        ]
+        return addresses_info
 
-            # If max change address bigger than current max change index minus the gap limit  - wallet._addresses.max_index(change=True)
-            if (
-                max_used_change + self.wallet.GAP_LIMIT
-                > self.wallet._addresses.max_index(change=True)
-            ):
-                # Add change addresses until the new max address plus the GAP_LIMIT
-                change_addresses = [
-                    dict(
-                        address=self.wallet.get_address(
-                            idx, change=True, check_keypool=False
-                        ),
-                        index=idx,
-                        change=True,
-                    )
-                    for idx in range(
-                        self.wallet._addresses.max_index(change=True),
-                        max_used_change + self.wallet.GAP_LIMIT,
-                    )
-                ]
-                self.wallet._addresses.add(change_addresses, check_rpc=False)
+    def calculate_max_used_from_addresses(self, addresses_info):
+        """Return a tuple of max_used_receiving and max_used_change
+        representing the highest index of the addresses from the wallet and
+        the passed addresses
+        """
+        # Gets max index used receiving and change addresses
+        max_used_receiving = self.wallet.addresses.max_used_index(change=False)
+        max_used_change = self.wallet.addresses.max_used_index(change=True)
 
-        # only delete with confirmed txs
-        self.wallet.delete_spent_pending_psbts(
-            [
-                tx["hex"]
-                for tx in txs.values()
-                if tx.get("confirmations", 0) > 0 or tx.get("blockheight")
+        for address in addresses_info:
+            desc = self.wallet.DescriptorCls.from_string(address["desc"])
+            indexes = [
+                {
+                    "idx": k.origin.derivation[-1],
+                    "change": k.origin.derivation[-2],
+                }
+                for k in desc.keys
             ]
-        )
-        self.wallet._transactions.add(txs)
+            for idx in indexes:
+                if int(idx["change"]) == 0:
+                    max_used_receiving = max(max_used_receiving, int(idx["idx"]))
+                elif int(idx["change"]) == 1:
+                    max_used_change = max(max_used_change, int(idx["idx"]))
+        return max_used_receiving, max_used_change
 
     @classmethod
-    def fetch_transactions(cls, wallet: Wallet):
+    def fetch_transactions(cls, wallet: AbstractWallet):
         """Loads new transactions from Bitcoin Core. A quite confusing method which mainly tries to figure out which transactions are new
         and need to be added to the local TxList wallet._transactions and adding them.
         So the method doesn't return anything but has these side_effects:

src/cryptoadvance/specter/txlist.py → src/cryptoadvance/specter/wallet/txlist.py

@@ -11,20 +11,20 @@
 from embit.liquid.networks import get_network
 from embit.transaction import Transaction
 
-from .helpers import get_address_from_dict
-from .persistence import delete_file, read_csv, write_csv
-from .specter_error import SpecterError, SpecterInternalException
+from ..helpers import get_address_from_dict
+from ..persistence import delete_file, read_csv, write_csv
+from ..specter_error import SpecterError, SpecterInternalException
 from embit.descriptor import Descriptor
 from embit.liquid.descriptor import LDescriptor
-from .util.common import str2bool
-from .util.psbt import (
+from ..util.common import str2bool
+from ..util.psbt import (
     AbstractTxContext,
     SpecterInputScope,
     SpecterOutputScope,
     SpecterPSBT,
     SpecterTx,
 )
-from .util.tx import decoderawtransaction
+from ..util.tx import decoderawtransaction
 from threading import RLock
 
 logger = logging.getLogger(__name__)