New AddressIndexer leveraging CoinView

src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs

@@ -42,8 +42,8 @@ public CoinviewTests()
             this.loggerFactory = new ExtendedLoggerFactory();
             this.nodeStats = new NodeStats(this.dateTimeProvider, NodeSettings.Default(this.network), new Mock<IVersionProvider>().Object);
 
-            this.coindb = new Coindb<DBreezeDbWithCoinDbNames>(this.network, this.dataFolder, this.dateTimeProvider, this.nodeStats, new DBreezeSerializer(this.network.Consensus.ConsensusFactory));
-            this.coindb.Initialize();
+            this.coindb = new Coindb<DBreezeDbWithCoinDbNames>(this.network, this.dataFolder, this.dateTimeProvider, this.nodeStats, new DBreezeSerializer(this.network.Consensus.ConsensusFactory), new ScriptAddressReader());
+            this.coindb.Initialize(false);
 
             this.chainIndexer = new ChainIndexer(this.network);
             this.stakeChainStore = new StakeChainStore(this.network, this.chainIndexer, (IStakedb)this.coindb, this.loggerFactory);

src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs

@@ -10,6 +10,7 @@
 using Stratis.Bitcoin.Consensus;
 using Stratis.Bitcoin.Features.Consensus.ProvenBlockHeaders;
 using Stratis.Bitcoin.Features.Consensus.Rules.CommonRules;
+using Stratis.Bitcoin.Interfaces;
 using Stratis.Bitcoin.Primitives;
 using Stratis.Bitcoin.Utilities;
 using Stratis.Bitcoin.Utilities.Extensions;
@@ -115,6 +116,10 @@ public long GetScriptSize
         /// <remarks>All access to this object has to be protected by <see cref="lockobj"/>.</remarks>
         private readonly Dictionary<OutPoint, CacheItem> cachedUtxoItems;
 
+        /// <summary>Tracks pending balance updates for dirty cache entries.</summary>
+        /// <remarks>All access to this object has to be protected by <see cref="lockobj"/>.</remarks>
+        private readonly Dictionary<TxDestination, Dictionary<uint, long>> cacheBalancesByDestination;
+
         /// <summary>Number of items in the cache.</summary>
         /// <remarks>The getter violates the lock contract on <see cref="cachedUtxoItems"/>, but the lock here is unnecessary as the <see cref="cachedUtxoItems"/> is marked as readonly.</remarks>
         private int cacheCount => this.cachedUtxoItems.Count;
@@ -133,12 +138,14 @@ public long GetScriptSize
         private IConsensusManager consensusManager;
         private readonly ConsensusSettings consensusSettings;
         private readonly ChainIndexer chainIndexer;
+        private readonly bool addressIndexingEnabled;
         private CachePerformanceSnapshot latestPerformanceSnapShot;
+        private IScriptAddressReader scriptAddressReader;
 
         private readonly Random random;
 
         public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, ChainIndexer chainIndexer,
-            StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null, INodeLifetime nodeLifetime = null, NodeSettings nodeSettings = null)
+            StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null, IScriptAddressReader scriptAddressReader = null, INodeLifetime nodeLifetime = null, NodeSettings nodeSettings = null)
         {
             Guard.NotNull(coindb, nameof(CachedCoinView.coindb));
 
@@ -153,9 +160,12 @@ public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTim
             this.cancellationToken = (nodeLifetime == null) ? new CancellationTokenSource() : CancellationTokenSource.CreateLinkedTokenSource(nodeLifetime.ApplicationStopping);
             this.lockobj = new object();
             this.cachedUtxoItems = new Dictionary<OutPoint, CacheItem>();
+            this.cacheBalancesByDestination = new Dictionary<TxDestination, Dictionary<uint, long>>();
             this.performanceCounter = new CachePerformanceCounter(this.dateTimeProvider);
             this.lastCacheFlushTime = this.dateTimeProvider.GetUtcNow();
             this.cachedRewindData = new Dictionary<int, RewindData>();
+            this.scriptAddressReader = scriptAddressReader;
+            this.addressIndexingEnabled = nodeSettings?.ConfigReader.GetOrDefault("addressindex", false) ?? false;
             this.random = new Random();
 
             this.MaxCacheSizeBytes = consensusSettings.MaxCoindbCacheInMB * 1024 * 1024;
@@ -269,7 +279,7 @@ public void Initialize(IConsensusManager consensusManager)
         {
             this.consensusManager = consensusManager;
 
-            this.coindb.Initialize();
+            this.coindb.Initialize(this.addressIndexingEnabled);
 
             Sync();
 
@@ -490,10 +500,11 @@ public void Flush(bool force = true)
 
                 this.logger.LogDebug("Flushing {0} items.", modify.Count);
 
-                this.coindb.SaveChanges(modify, this.innerBlockHash, this.blockHash, this.cachedRewindData.Select(c => c.Value).ToList());
+                this.coindb.SaveChanges(modify, this.cacheBalancesByDestination, this.innerBlockHash, this.blockHash, this.cachedRewindData.Select(c => c.Value).ToList());
 
                 // All the cached utxos are now on disk so we can clear the cached entry list.
                 this.cachedUtxoItems.Clear();
+                this.cacheBalancesByDestination.Clear();
                 this.cacheSizeBytes = 0;
 
                 this.cachedRewindData.Clear();
@@ -584,6 +595,10 @@ public void SaveChanges(IList<UnspentOutput> outputs, HashHeightPair oldBlockHas
                     {
                         // DELETE COINS
 
+                        // Record the UTXO as having been spent at this height.
+                        if (cacheItem.Coins != null)
+                            this.RecordBalanceChange(cacheItem.Coins.TxOut.ScriptPubKey, -cacheItem.Coins.TxOut.Value, (uint)nextBlockHash.Height);
+
                         // In cases of an output spent in the same block 
                         // it wont exist in cash or in disk so its safe to remove it
                         if (cacheItem.Coins == null)
@@ -632,6 +647,9 @@ public void SaveChanges(IList<UnspentOutput> outputs, HashHeightPair oldBlockHas
                     {
                         // ADD COINS
 
+                        // Update the balance.
+                        this.RecordBalanceChange(output.Coins.TxOut.ScriptPubKey, output.Coins.TxOut.Value, output.Coins.Height);
+
                         if (cacheItem.Coins != null)
                         {
                             // Allow overrides.
@@ -713,6 +731,7 @@ public HashHeightPair Rewind(HashHeightPair target = null)
 
                 // All the cached utxos are now on disk so we can clear the cached entry list.
                 this.cachedUtxoItems.Clear();
+                this.cacheBalancesByDestination.Clear();
                 this.cacheSizeBytes = 0;
                 this.dirtyCacheCount = 0;
 
@@ -772,5 +791,62 @@ private void AddBenchStats(StringBuilder log)
 
             this.latestPerformanceSnapShot = snapShot;
         }
+
+        private void RecordBalanceChange(Script scriptPubKey, long satoshis, uint height)
+        {
+            if (!this.coindb.BalanceIndexingEnabled || scriptPubKey.Length == 0 || satoshis == 0)
+                return;
+
+            foreach (TxDestination txDestination in this.scriptAddressReader.GetDestinationFromScriptPubKey(this.network, scriptPubKey))
+            {
+                if (!this.cacheBalancesByDestination.TryGetValue(txDestination, out Dictionary<uint, long> value))
+                {
+                    value = new Dictionary<uint, long>();
+                    this.cacheBalancesByDestination[txDestination] = value;
+                }
+
+                if (!value.TryGetValue(height, out long balance))
+                    balance = 0;
+
+                balance += satoshis;
+
+                value[height] = balance;
+            }
+        }
+
+        public IEnumerable<(uint, long)> GetBalance(TxDestination txDestination)
+        {
+            IEnumerable<(uint, long)> CachedBalances()
+            {
+                if (this.cacheBalancesByDestination.TryGetValue(txDestination, out Dictionary<uint, long> itemsByHeight))
+                {
+                    long balance = 0;
+
+                    foreach (uint height in itemsByHeight.Keys.OrderBy(k => k))
+                    {
+                        balance += itemsByHeight[height];
+                        yield return (height, balance);
+                    }
+                }
+            }
+
+            bool first = true;
+            foreach ((uint height, long satoshis) in this.coindb.GetBalance(txDestination))
+            {
+                if (first)
+                {
+                    first = false;
+
+                    foreach ((uint height2, long satoshis2) in CachedBalances().Reverse())
+                        yield return (height2, satoshis2 + satoshis);
+                }
+
+                yield return (height, satoshis);
+            }
+
+            if (first)
+                foreach ((uint height2, long satoshis2) in CachedBalances().Reverse())
+                    yield return (height2, satoshis2);
+        }
     }
 }

src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs

@@ -82,5 +82,15 @@ public interface ICoinView
         /// </summary>
         /// <param name="height">The height of the block.</param>
         RewindData GetRewindData(int height);
+
+        /// <summary>
+        /// Returns a combination of (height, satoshis) values with the cumulative balance up to the corresponding height.
+        /// </summary>
+        /// <param name="txDestination">The destination value derived from the address being queried.</param>
+        /// <returns>A combination of (height, satoshis) values with the cumulative balance up to the corresponding height.</returns>
+        /// <remarks>Balance updates (even when nett 0) are delivered for every height at which transactions for the address 
+        /// had been recorded and as such the returned heights can be used in conjunction with the block store to discover 
+        /// all related transactions.</remarks>
+        IEnumerable<(uint height, long satoshis)> GetBalance(TxDestination txDestination);
     }
 }