diff --git a/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerOutpointsRepositoryTests.cs b/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerOutpointsRepositoryTests.cs deleted file mode 100644 index 47939abe0d..0000000000 --- a/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerOutpointsRepositoryTests.cs +++ /dev/null @@ -1,94 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.InteropServices; -using LiteDB; -using NBitcoin; -using Stratis.Bitcoin.Features.BlockStore.AddressIndexing; -using Xunit; - -namespace Stratis.Bitcoin.Features.BlockStore.Tests -{ - public class AddressIndexerOutpointsRepositoryTests - { - private readonly AddressIndexerOutpointsRepository repository; - - private readonly Random random = new Random(); - - private readonly int maxItems = 10; - - public AddressIndexerOutpointsRepositoryTests() - { - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - var db = new LiteDatabase(new ConnectionString() { Filename = this.RandomString(20) + ".litedb", Mode = fileMode }); - - this.repository = new AddressIndexerOutpointsRepository(db, this.maxItems); - } - - [Fact] - public void LoadPercentageCalculatedCorrectly() - { - for (int i = 0; i < this.maxItems / 2; i++) - this.repository.AddOutPointData(new OutPointData() { Outpoint = this.RandomString(20) }); - - Assert.Equal(50, this.repository.GetLoadPercentage()); - } - - [Fact] - public void CanAddAndRemoveOutpointData() - { - var outPoint = new OutPoint(new uint256(RandomUtils.GetUInt64()), 1); - - var data = new OutPointData() { Outpoint = outPoint.ToString(), Money = 1, ScriptPubKeyBytes = RandomUtils.GetBytes(20) }; - this.repository.AddOutPointData(data); - - // Add more to trigger eviction. - for (int i = 0; i < this.maxItems * 2; i++) - this.repository.AddOutPointData(new OutPointData() { Outpoint = this.RandomString(20) }); - - Assert.True(this.repository.TryGetOutPointData(outPoint, out OutPointData dataOut)); - Assert.True(data.ScriptPubKeyBytes.SequenceEqual(dataOut.ScriptPubKeyBytes)); - } - - [Fact] - public void CanRewind() - { - var rewindDataBlockHash = new uint256(RandomUtils.GetUInt64()); - - var outPoint = new OutPoint(new uint256(RandomUtils.GetUInt64()), 1); - var data = new OutPointData() { Outpoint = outPoint.ToString(), Money = 1, ScriptPubKeyBytes = RandomUtils.GetBytes(20) }; - - var rewindData = new AddressIndexerRewindData() - { - BlockHash = rewindDataBlockHash.ToString(), - BlockHeight = 100, - SpentOutputs = new List() { data } - }; - - this.repository.RecordRewindData(rewindData); - - Assert.False(this.repository.TryGetOutPointData(outPoint, out OutPointData dataOut)); - - this.repository.RewindDataAboveHeight(rewindData.BlockHeight - 1); - - Assert.True(this.repository.TryGetOutPointData(outPoint, out dataOut)); - - // Now record and purge rewind data. - this.repository.RecordRewindData(rewindData); - - this.repository.RemoveOutPointData(outPoint); - Assert.False(this.repository.TryGetOutPointData(outPoint, out dataOut)); - - this.repository.PurgeOldRewindData(rewindData.BlockHeight + 1); - - Assert.False(this.repository.TryGetOutPointData(outPoint, out dataOut)); - } - - private string RandomString(int length) - { - const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - return new string(Enumerable.Repeat(chars, length) - .Select(s => s[this.random.Next(s.Length)]).ToArray()); - } - } -} diff --git a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs index 192c0a3085..f1474bf915 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs @@ -1,27 +1,13 @@ using System; using System.Collections.Generic; -using System.Diagnostics; -using System.IO; using System.Linq; -using System.Runtime.InteropServices; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using LiteDB; -using Microsoft.Extensions.Logging; using NBitcoin; -using Stratis.Bitcoin.AsyncWork; using Stratis.Bitcoin.Builder.Feature; -using Stratis.Bitcoin.Configuration; -using Stratis.Bitcoin.Configuration.Logging; using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Controllers.Models; using Stratis.Bitcoin.Features.BlockStore.Models; +using Stratis.Bitcoin.Features.Consensus.CoinViews; using Stratis.Bitcoin.Interfaces; -using Stratis.Bitcoin.Primitives; -using Stratis.Bitcoin.Utilities; -using FileMode = LiteDB.FileMode; -using Script = NBitcoin.Script; namespace Stratis.Bitcoin.Features.BlockStore.AddressIndexing { @@ -50,123 +36,30 @@ public interface IAddressIndexer : IDisposable public class AddressIndexer : IAddressIndexer { - public ChainedHeader IndexerTip { get; private set; } - - private readonly StoreSettings storeSettings; - private readonly Network network; - - private readonly INodeStats nodeStats; - - private readonly ILogger logger; - - private readonly DataFolder dataFolder; - - private readonly IConsensusManager consensusManager; - - private readonly IAsyncProvider asyncProvider; - + private readonly ICoinView coinView; + private readonly ChainIndexer chainIndexer; private readonly IScriptAddressReader scriptAddressReader; - private readonly TimeSpan flushChangesInterval; - - private const string DbTipDataKey = "AddrTipData"; + public ChainedHeader IndexerTip => GetTip(); - private const string AddressIndexerDatabaseFilename = "addressindex.litedb"; + public IFullNodeFeature InitializingFeature { set; private get; } /// Max supported reorganization length for networks without max reorg property. public const int FallBackMaxReorg = 200; - /// - /// Time to wait before attempting to index the next block. - /// Waiting happens after a failure to get next block to index. - /// - private const int DelayTimeMs = 2000; - - private int compactionThreshold; - - private int compactionAmount => this.compactionThreshold / 2; - - /// Max distance between consensus and indexer tip to consider indexer synced. - private const int ConsiderSyncedMaxDistance = 10; - - private LiteDatabase db; - - private LiteCollection tipDataStore; - - /// A mapping between addresses and their balance changes. - /// All access should be protected by . - private AddressIndexRepository addressIndexRepository; - - /// Script pub keys and amounts mapped by outpoints. - /// All access should be protected by . - private AddressIndexerOutpointsRepository outpointsRepository; - - /// Protects access to and . - private readonly object lockObject; - - private readonly CancellationTokenSource cancellation; - - private readonly ChainIndexer chainIndexer; - - private readonly AverageCalculator averageTimePerBlock; - - private readonly IDateTimeProvider dateTimeProvider; - - private readonly IUtxoIndexer utxoIndexer; - - private Task indexingTask; - - private DateTime lastFlushTime; - - private const int PurgeIntervalSeconds = 60; - - /// Last time rewind data was purged. - private DateTime lastPurgeTime; - - private Task prefetchingTask; - - /// Indexer height at the last save. - /// Should be protected by . - private int lastSavedHeight; - - /// Distance in blocks from consensus tip at which compaction should start. - /// It can't be lower than maxReorg since compacted data can't be converted back to uncompacted state for partial reversion. - private readonly int compactionTriggerDistance; - /// /// This is a window of some blocks that is needed to reduce the consequences of nodes having different view of consensus chain. /// We assume that nodes usually don't have view that is different from other nodes by that constant of blocks. /// public const int SyncBuffer = 50; - public IFullNodeFeature InitializingFeature { get; set; } - - public AddressIndexer(StoreSettings storeSettings, DataFolder dataFolder, Network network, INodeStats nodeStats, - IConsensusManager consensusManager, IAsyncProvider asyncProvider, ChainIndexer chainIndexer, IDateTimeProvider dateTimeProvider, IUtxoIndexer utxoIndexer) + public AddressIndexer(Network network, ChainIndexer chainIndexer, IScriptAddressReader scriptAddressReader, ICoinView coinView) { - this.storeSettings = storeSettings; this.network = network; - this.nodeStats = nodeStats; - this.dataFolder = dataFolder; - this.consensusManager = consensusManager; - this.asyncProvider = asyncProvider; - this.dateTimeProvider = dateTimeProvider; - this.utxoIndexer = utxoIndexer; - this.scriptAddressReader = new ScriptAddressReader(); - - this.lockObject = new object(); - this.flushChangesInterval = TimeSpan.FromMinutes(2); - this.lastFlushTime = this.dateTimeProvider.GetUtcNow(); - this.cancellation = new CancellationTokenSource(); + this.coinView = coinView; this.chainIndexer = chainIndexer; - this.logger = LogManager.GetCurrentClassLogger(); - - this.compactionThreshold = storeSettings.AddressIndexerCompactionThreshold; - this.averageTimePerBlock = new AverageCalculator(200); - int maxReorgLength = GetMaxReorgOrFallbackMaxReorg(this.network); - - this.compactionTriggerDistance = maxReorgLength * 2 + SyncBuffer + 1000; + this.scriptAddressReader = scriptAddressReader; } /// Gets the maxReorg of in case maxReorg is 0. @@ -179,588 +72,81 @@ public static int GetMaxReorgOrFallbackMaxReorg(Network network) return maxReorgLength; } - public void Initialize() + private ChainedHeader GetTip() { - // The transaction index is needed in the event of a reorg. - if (!this.storeSettings.AddressIndex) - { - this.logger.LogTrace("(-)[DISABLED]"); - return; - } - - string dbPath = Path.Combine(this.dataFolder.RootPath, AddressIndexerDatabaseFilename); - - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - this.db = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode }); - - this.addressIndexRepository = new AddressIndexRepository(this.db); - - this.logger.LogDebug("Address indexing is enabled."); - - this.tipDataStore = this.db.GetCollection(DbTipDataKey); - - lock (this.lockObject) - { - AddressIndexerTipData tipData = this.tipDataStore.FindAll().FirstOrDefault(); + this.coinView.Sync(); - this.logger.LogDebug("Tip data: '{0}'.", tipData == null ? "null" : tipData.ToString()); - - this.IndexerTip = tipData == null ? this.chainIndexer.Genesis : this.consensusManager.Tip.FindAncestorOrSelf(new uint256(tipData.TipHashBytes)); - - if (this.IndexerTip == null) - { - // This can happen if block hash from tip data is no longer a part of the consensus chain and node was killed in the middle of a reorg. - int rewindAmount = this.compactionTriggerDistance / 2; - - if (rewindAmount > this.consensusManager.Tip.Height) - this.IndexerTip = this.chainIndexer.Genesis; - else - this.IndexerTip = this.consensusManager.Tip.GetAncestor(this.consensusManager.Tip.Height - rewindAmount); - } - } - - this.outpointsRepository = new AddressIndexerOutpointsRepository(this.db); - - this.RewindAndSave(this.IndexerTip); - - this.logger.LogDebug("Indexer initialized at '{0}'.", this.IndexerTip); - - this.indexingTask = Task.Run(async () => await this.IndexAddressesContinuouslyAsync().ConfigureAwait(false)); - - this.asyncProvider.RegisterTask($"{nameof(AddressIndexer)}.{nameof(this.indexingTask)}", this.indexingTask); - - this.nodeStats.RegisterStats(this.AddInlineStats, StatsType.Inline, this.GetType().Name, 400); + return this.chainIndexer[this.coinView.GetTipHash().Hash]; } - private async Task IndexAddressesContinuouslyAsync() + public void Initialize() { - var watch = Stopwatch.StartNew(); - - while (!this.cancellation.IsCancellationRequested) - { - if (this.dateTimeProvider.GetUtcNow() - this.lastFlushTime > this.flushChangesInterval) - { - this.logger.LogDebug("Flushing changes."); - - this.SaveAll(); - - this.lastFlushTime = this.dateTimeProvider.GetUtcNow(); - - this.logger.LogDebug("Flush completed."); - } - - if (this.cancellation.IsCancellationRequested) - { - this.logger.LogDebug("Cancelled loop."); - break; - } - - ChainedHeader nextHeader = this.consensusManager.Tip.GetAncestor(this.IndexerTip.Height + 1); - - if (nextHeader == null) - { - this.logger.LogDebug("Next header wasn't found. Waiting."); - - try - { - await Task.Delay(DelayTimeMs, this.cancellation.Token).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - } - - continue; - } - - if (nextHeader.Previous.HashBlock != this.IndexerTip.HashBlock) - { - ChainedHeader lastCommonHeader = nextHeader.FindFork(this.IndexerTip); - - this.logger.LogDebug("Reorganization detected. Rewinding till '{0}'.", lastCommonHeader); - - this.RewindAndSave(lastCommonHeader); - - continue; - } - - // First try to see if it's prefetched. - ChainedHeaderBlock prefetchedBlock = this.prefetchingTask == null ? null : await this.prefetchingTask.ConfigureAwait(false); - - Block blockToProcess; - - if (prefetchedBlock != null && prefetchedBlock.ChainedHeader == nextHeader) - blockToProcess = prefetchedBlock.Block; - else - blockToProcess = this.consensusManager.GetBlockData(nextHeader.HashBlock)?.Block; - - if (blockToProcess == null) - { - this.logger.LogDebug("Next block wasn't found. Waiting."); - - try - { - await Task.Delay(DelayTimeMs, this.cancellation.Token).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - } - - continue; - } - - // Schedule prefetching of the next block; - ChainedHeader headerToPrefetch = this.consensusManager.Tip.GetAncestor(nextHeader.Height + 1); - - if (headerToPrefetch != null) - this.prefetchingTask = Task.Run(() => this.consensusManager.GetBlockData(headerToPrefetch.HashBlock)); - - watch.Restart(); - - bool success = this.ProcessBlock(blockToProcess, nextHeader); - - watch.Stop(); - this.averageTimePerBlock.AddSample(watch.Elapsed.TotalMilliseconds); - - if (!success) - { - this.logger.LogDebug("Failed to process next block. Waiting."); - - try - { - await Task.Delay(DelayTimeMs, this.cancellation.Token).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - } - - continue; - } - - this.IndexerTip = nextHeader; - } - - this.SaveAll(); } - private void RewindAndSave(ChainedHeader rewindToHeader) + private TxDestination AddressToDestination(string address) { - lock (this.lockObject) - { - // The cache doesn't really lend itself to handling a reorg very well. - // Therefore, we leverage LiteDb's indexing capabilities to tell us - // which records are for the affected blocks. - - List affectedAddresses = this.addressIndexRepository.GetAddressesHigherThanHeight(rewindToHeader.Height); - - foreach (string address in affectedAddresses) - { - AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address); - indexData.BalanceChanges.RemoveAll(x => x.BalanceChangedHeight > rewindToHeader.Height); - } - - this.logger.LogDebug("Rewinding changes for {0} addresses.", affectedAddresses.Count); - - // Rewind all the way back to the fork point. - this.outpointsRepository.RewindDataAboveHeight(rewindToHeader.Height); - - this.IndexerTip = rewindToHeader; - - this.SaveAll(); - } + var bitcoinAddress = BitcoinAddress.Create(address, this.network); + return this.scriptAddressReader.GetDestinationFromScriptPubKey(this.network, bitcoinAddress.ScriptPubKey).Single(); } - private void SaveAll() + public AddressBalancesResult GetAddressBalances(string[] addresses, int minConfirmations = 0) { - this.logger.LogDebug("Saving address indexer."); - - lock (this.lockObject) + return new AddressBalancesResult() { - this.logger.LogDebug("Saving addr indexer repo."); - this.addressIndexRepository.SaveAllItems(); - - this.logger.LogDebug("Saving outpoints repo."); - this.outpointsRepository.SaveAllItems(); - - AddressIndexerTipData tipData = this.tipDataStore.FindAll().FirstOrDefault(); - - if (tipData == null) - tipData = new AddressIndexerTipData(); - - tipData.Height = this.IndexerTip.Height; - tipData.TipHashBytes = this.IndexerTip.HashBlock.ToBytes(); - - this.logger.LogDebug("Saving tip data."); - - this.tipDataStore.Upsert(tipData); - this.lastSavedHeight = this.IndexerTip.Height; - } - - this.logger.LogDebug("Address indexer saved."); - } - - private void AddInlineStats(StringBuilder benchLog) - { - benchLog.AppendLine("AddressIndexer Height".PadRight(LoggingConfiguration.ColumnLength) + $": {this.IndexerTip.Height}".PadRight(9) + - " AddressCache%: " + this.addressIndexRepository.GetLoadPercentage().ToString().PadRight(8) + - "OutPointCache%: " + this.outpointsRepository.GetLoadPercentage().ToString().PadRight(8) + - $"Ms/block: {Math.Round(this.averageTimePerBlock.Average, 2)}"); - } - - /// Processes a block that was added or removed from the consensus chain. - /// The block to process. - /// The chained header associated to the block being processed. - /// true if block was sucessfully processed. - private bool ProcessBlock(Block block, ChainedHeader header) - { - this.logger.LogTrace("Processing block " + header.ToString()); - - lock (this.lockObject) - { - // Record outpoints. - foreach (Transaction tx in block.Transactions) - { - for (int i = 0; i < tx.Outputs.Count; i++) - { - // OP_RETURN outputs and empty outputs cannot be spent and therefore do not need to be put into the cache. - if (tx.Outputs[i].IsEmpty || tx.Outputs[i].ScriptPubKey.IsUnspendable) - continue; - - var outPoint = new OutPoint(tx, i); - - var outPointData = new OutPointData() - { - Outpoint = outPoint.ToString(), - ScriptPubKeyBytes = tx.Outputs[i].ScriptPubKey.ToBytes(), - Money = tx.Outputs[i].Value - }; - - // TODO: When the outpoint cache is full, adding outpoints singly causes overhead writing evicted entries out to the repository - this.outpointsRepository.AddOutPointData(outPointData); - } - } - } - - // Process inputs. - var inputs = new List(); - - // Collect all inputs. - foreach (TxInList inputsCollection in block.Transactions.Select(x => x.Inputs)) - inputs.AddRange(inputsCollection); - - lock (this.lockObject) - { - var rewindData = new AddressIndexerRewindData() { BlockHash = header.HashBlock.ToString(), BlockHeight = header.Height, SpentOutputs = new List() }; - - foreach (TxIn input in inputs) - { - OutPoint consumedOutput = input.PrevOut; - - // Ignore coinbase. - if (consumedOutput.Hash == uint256.Zero) - continue; - - if (!this.outpointsRepository.TryGetOutPointData(consumedOutput, out OutPointData consumedOutputData)) - { - this.logger.LogError("Missing outpoint data for {0}.", consumedOutput); - this.logger.LogTrace("(-)[MISSING_OUTPOINTS_DATA]"); - throw new Exception($"Missing outpoint data for {consumedOutput}"); - } - - Money amountSpent = consumedOutputData.Money; - - rewindData.SpentOutputs.Add(consumedOutputData); - - // Transactions that don't actually change the balance just bloat the database. - if (amountSpent == 0) - continue; - - string address = this.scriptAddressReader.GetAddressFromScriptPubKey(this.network, new Script(consumedOutputData.ScriptPubKeyBytes)); - - if (string.IsNullOrEmpty(address)) + Balances = addresses + .Select(address => (address, destination: AddressToDestination(address))) + .Select(t => new AddressBalanceResult() { - // This condition need not be logged, as the address reader should be aware of all possible address formats already. - continue; - } - - this.ProcessBalanceChangeLocked(header.Height, address, amountSpent, false); - } - - // Process outputs. - foreach (Transaction tx in block.Transactions) - { - foreach (TxOut txOut in tx.Outputs) - { - Money amountReceived = txOut.Value; - - // Transactions that don't actually change the balance just bloat the database. - if (amountReceived == 0 || txOut.IsEmpty || txOut.ScriptPubKey.IsUnspendable) - continue; - - string address = this.scriptAddressReader.GetAddressFromScriptPubKey(this.network, txOut.ScriptPubKey); - - if (string.IsNullOrEmpty(address)) - { - // This condition need not be logged, as the address reader should be aware of all - // possible address formats already. - continue; - } + Address = t.address, + Balance = (t.destination == null) ? 0 : new Money(this.coinView.GetBalance(t.destination).First(x => x.height <= (this.chainIndexer.Tip.Height - minConfirmations)).satoshis), - this.ProcessBalanceChangeLocked(header.Height, address, amountReceived, true); - } - } - - this.outpointsRepository.RecordRewindData(rewindData); - - int purgeRewindDataThreshold = Math.Min(this.consensusManager.Tip.Height - this.compactionTriggerDistance, this.lastSavedHeight); - - if ((this.dateTimeProvider.GetUtcNow() - this.lastPurgeTime).TotalSeconds > PurgeIntervalSeconds) - { - this.outpointsRepository.PurgeOldRewindData(purgeRewindDataThreshold); - this.lastPurgeTime = this.dateTimeProvider.GetUtcNow(); - } - - // Remove outpoints that were consumed. - foreach (OutPoint consumedOutPoint in inputs.Where(x => x.PrevOut.Hash != uint256.Zero).Select(x => x.PrevOut)) - this.outpointsRepository.RemoveOutPointData(consumedOutPoint); - } - - this.logger.LogTrace("Block processed."); - return true; - } - - /// Adds a new balance change entry to to the . - /// The height of the block this being processed. - /// The address receiving the funds. - /// The amount being received. - /// false if this is an output being spent, true otherwise. - /// Should be protected by . - private void ProcessBalanceChangeLocked(int height, string address, Money amount, bool deposited) - { - AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address); - - // Record new balance change into the address index data. - indexData.BalanceChanges.Add(new AddressBalanceChange() - { - BalanceChangedHeight = height, - Satoshi = amount.Satoshi, - Deposited = deposited - }); - - // Anything less than that should be compacted. - int heightThreshold = this.consensusManager.Tip.Height - this.compactionTriggerDistance; - - bool compact = (indexData.BalanceChanges.Count > this.compactionThreshold) && (indexData.BalanceChanges[1].BalanceChangedHeight < heightThreshold); - - if (!compact) - { - this.addressIndexRepository.AddOrUpdate(indexData.Address, indexData, indexData.BalanceChanges.Count + 1); - - this.logger.LogTrace("(-)[TOO_FEW_CHANGE_RECORDS]"); - return; - } - - var compacted = new List(this.compactionThreshold / 2) - { - new AddressBalanceChange() - { - BalanceChangedHeight = 0, - Satoshi = 0, - Deposited = true - } + }).ToList() }; - - for (int i = 0; i < indexData.BalanceChanges.Count; i++) - { - AddressBalanceChange change = indexData.BalanceChanges[i]; - - if ((change.BalanceChangedHeight) < heightThreshold && i < this.compactionAmount) - { - this.logger.LogDebug("Balance change: {0} was selected for compaction. Compacted balance now: {1}.", change, compacted[0].Satoshi); - - if (change.Deposited) - compacted[0].Satoshi += change.Satoshi; - else - compacted[0].Satoshi -= change.Satoshi; - - this.logger.LogDebug("New compacted balance: {0}.", compacted[0].Satoshi); - } - else - compacted.Add(change); - } - - indexData.BalanceChanges = compacted; - this.addressIndexRepository.AddOrUpdate(indexData.Address, indexData, indexData.BalanceChanges.Count + 1); } - private bool IsSynced() + public LastBalanceDecreaseTransactionModel GetLastBalanceDecreaseTransaction(string address) { - lock (this.lockObject) - { - return this.consensusManager.Tip.Height - this.IndexerTip.Height <= ConsiderSyncedMaxDistance; - } + throw new NotImplementedException(); } - /// - /// This is currently not in use but will be required for exchange integration. - public AddressBalancesResult GetAddressBalances(string[] addresses, int minConfirmations = 1) + private IEnumerable ToDiff(List addressBalanceChanges) { - var (isQueryable, reason) = this.IsQueryable(); - - if (!isQueryable) - return AddressBalancesResult.RequestFailed(reason); - - var result = new AddressBalancesResult(); - - lock (this.lockObject) + for (int i = addressBalanceChanges.Count - 1; i > 0; i--) { - foreach (var address in addresses) + yield return new AddressBalanceChange() { - AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address); - - int maxAllowedHeight = this.consensusManager.Tip.Height - minConfirmations + 1; - - long balance = indexData.BalanceChanges.Where(x => x.BalanceChangedHeight <= maxAllowedHeight).CalculateBalance(); - - this.logger.LogDebug("Address: {0}, balance: {1}.", address, balance); - result.Balances.Add(new AddressBalanceResult(address, new Money(balance))); - } - - return result; + BalanceChangedHeight = addressBalanceChanges[i - 1].BalanceChangedHeight, + Deposited = addressBalanceChanges[i - 1].Satoshi >= addressBalanceChanges[i].Satoshi, + Satoshi = Math.Abs(addressBalanceChanges[i - 1].Satoshi - addressBalanceChanges[i].Satoshi) + }; } } - /// + /// public VerboseAddressBalancesResult GetAddressIndexerState(string[] addresses) { // If the containing feature is not initialized then wait a bit. this.InitializingFeature?.WaitInitialized(); - var result = new VerboseAddressBalancesResult(this.consensusManager.Tip.Height); - - if (addresses.Length == 0) - return result; - - if (!this.storeSettings.AddressIndex) - throw new NotSupportedException("Address indexing is not enabled."); - - (bool isQueryable, string reason) = this.IsQueryable(); - - if (!isQueryable) - return VerboseAddressBalancesResult.RequestFailed(reason); - - lock (this.lockObject) - { - foreach (var address in addresses) - { - AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address); - - var copy = new AddressIndexerData() - { - Address = indexData.Address, - BalanceChanges = new List(indexData.BalanceChanges) - }; - - result.BalancesData.Add(copy); - } - } - - return result; - } - - public LastBalanceDecreaseTransactionModel GetLastBalanceDecreaseTransaction(string address) - { - if (address == null) - return null; - - (bool isQueryable, string reason) = this.IsQueryable(); - - if (!isQueryable) - return null; - - int lastBalanceHeight; - - lock (this.lockObject) - { - AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address); - - AddressBalanceChange lastBalanceUpdate = indexData.BalanceChanges.Where(a => !a.Deposited).OrderByDescending(b => b.BalanceChangedHeight).FirstOrDefault(); - - if (lastBalanceUpdate == null) - return null; - - lastBalanceHeight = lastBalanceUpdate.BalanceChangedHeight; - } - - // Height 0 is used as a placeholder height for compacted address balance records, so ignore them if they are the only record. - if (lastBalanceHeight == 0) - return null; - - ChainedHeader header = this.chainIndexer.GetHeader(lastBalanceHeight); - - if (header == null) - return null; - - Block block = this.consensusManager.GetBlockData(header.HashBlock).Block; - - if (block == null) - return null; - - // Get the UTXO snapshot as of one block lower than the last balance change, so that we are definitely able to look up the inputs of each transaction in the next block. - ReconstructedCoinviewContext utxos = this.utxoIndexer.GetCoinviewAtHeight(lastBalanceHeight - 1); - - Transaction foundTransaction = null; - - foreach (Transaction transaction in block.Transactions) + return new VerboseAddressBalancesResult(this.IndexerTip.Height) { - if (transaction.IsCoinBase) - continue; - - foreach (TxIn txIn in transaction.Inputs) - { - Transaction prevTx = utxos.Transactions[txIn.PrevOut.Hash]; - - foreach (TxOut txOut in prevTx.Outputs) + BalancesData = addresses + .Select(address => (address, destination: AddressToDestination(address))) + .Select(t => new AddressIndexerData() { - if (this.scriptAddressReader.GetAddressFromScriptPubKey(this.network, txOut.ScriptPubKey) == address) + Address = t.address, + BalanceChanges = (t.destination == null) ? new List() : ToDiff(this.coinView.GetBalance(t.destination).Select(b => new AddressBalanceChange() { - foundTransaction = transaction; - } - } - } - } - - return foundTransaction == null ? null : new LastBalanceDecreaseTransactionModel() { BlockHeight = lastBalanceHeight, Transaction = new TransactionVerboseModel(foundTransaction, this.network) }; - } - - private (bool isQueryable, string reason) IsQueryable() - { - if (this.addressIndexRepository == null) - { - this.logger.LogTrace("(-)[NOT_INITIALIZED]"); - return (false, "Address indexer is not initialized."); - } - - if (!this.IsSynced()) - { - this.logger.LogTrace("(-)[NOT_SYNCED]"); - return (false, "Address indexer is not synced."); - } - - return (true, string.Empty); + BalanceChangedHeight = (int)b.height, + Deposited = b.satoshis >= 0, + Satoshi = Math.Abs(b.satoshis) + }).ToList()).ToList() + }).ToList() + }; } - /// public void Dispose() { - this.logger.LogDebug("Disposing."); - - this.cancellation.Cancel(); - - this.indexingTask?.GetAwaiter().GetResult(); - - this.db?.Dispose(); - - this.logger.LogDebug("Disposed."); } } -} +} \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs b/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs index 02a7e88324..1abaac1189 100644 --- a/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs +++ b/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs @@ -42,8 +42,8 @@ public CoinviewTests() this.loggerFactory = new ExtendedLoggerFactory(); this.nodeStats = new NodeStats(this.dateTimeProvider, NodeSettings.Default(this.network), new Mock().Object); - this.coindb = new Coindb(this.network, this.dataFolder, this.dateTimeProvider, this.nodeStats, new DBreezeSerializer(this.network.Consensus.ConsensusFactory)); - this.coindb.Initialize(); + this.coindb = new Coindb(this.network, this.dataFolder, this.dateTimeProvider, this.nodeStats, new DBreezeSerializer(this.network.Consensus.ConsensusFactory), new ScriptAddressReader()); + this.coindb.Initialize(false); this.chainIndexer = new ChainIndexer(this.network); this.stakeChainStore = new StakeChainStore(this.network, this.chainIndexer, (IStakedb)this.coindb, this.loggerFactory); diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs index d61572e039..6f30b42fab 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs @@ -10,6 +10,7 @@ using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Features.Consensus.ProvenBlockHeaders; using Stratis.Bitcoin.Features.Consensus.Rules.CommonRules; +using Stratis.Bitcoin.Interfaces; using Stratis.Bitcoin.Primitives; using Stratis.Bitcoin.Utilities; using Stratis.Bitcoin.Utilities.Extensions; @@ -115,6 +116,10 @@ public long GetScriptSize /// All access to this object has to be protected by . private readonly Dictionary cachedUtxoItems; + /// Tracks pending balance updates for dirty cache entries. + /// All access to this object has to be protected by . + private readonly Dictionary> cacheBalancesByDestination; + /// Number of items in the cache. /// The getter violates the lock contract on , but the lock here is unnecessary as the is marked as readonly. private int cacheCount => this.cachedUtxoItems.Count; @@ -133,12 +138,14 @@ public long GetScriptSize private IConsensusManager consensusManager; private readonly ConsensusSettings consensusSettings; private readonly ChainIndexer chainIndexer; + private readonly bool addressIndexingEnabled; private CachePerformanceSnapshot latestPerformanceSnapShot; + private IScriptAddressReader scriptAddressReader; private readonly Random random; public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, ChainIndexer chainIndexer, - StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null, INodeLifetime nodeLifetime = null, NodeSettings nodeSettings = null) + StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null, IScriptAddressReader scriptAddressReader = null, INodeLifetime nodeLifetime = null, NodeSettings nodeSettings = null) { Guard.NotNull(coindb, nameof(CachedCoinView.coindb)); @@ -153,9 +160,12 @@ public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTim this.cancellationToken = (nodeLifetime == null) ? new CancellationTokenSource() : CancellationTokenSource.CreateLinkedTokenSource(nodeLifetime.ApplicationStopping); this.lockobj = new object(); this.cachedUtxoItems = new Dictionary(); + this.cacheBalancesByDestination = new Dictionary>(); this.performanceCounter = new CachePerformanceCounter(this.dateTimeProvider); this.lastCacheFlushTime = this.dateTimeProvider.GetUtcNow(); this.cachedRewindData = new Dictionary(); + this.scriptAddressReader = scriptAddressReader; + this.addressIndexingEnabled = nodeSettings?.ConfigReader.GetOrDefault("addressindex", false) ?? false; this.random = new Random(); this.MaxCacheSizeBytes = consensusSettings.MaxCoindbCacheInMB * 1024 * 1024; @@ -269,7 +279,7 @@ public void Initialize(IConsensusManager consensusManager) { this.consensusManager = consensusManager; - this.coindb.Initialize(); + this.coindb.Initialize(this.addressIndexingEnabled); Sync(); @@ -490,10 +500,11 @@ public void Flush(bool force = true) this.logger.LogDebug("Flushing {0} items.", modify.Count); - this.coindb.SaveChanges(modify, this.innerBlockHash, this.blockHash, this.cachedRewindData.Select(c => c.Value).ToList()); + this.coindb.SaveChanges(modify, this.cacheBalancesByDestination, this.innerBlockHash, this.blockHash, this.cachedRewindData.Select(c => c.Value).ToList()); // All the cached utxos are now on disk so we can clear the cached entry list. this.cachedUtxoItems.Clear(); + this.cacheBalancesByDestination.Clear(); this.cacheSizeBytes = 0; this.cachedRewindData.Clear(); @@ -584,6 +595,10 @@ public void SaveChanges(IList outputs, HashHeightPair oldBlockHas { // DELETE COINS + // Record the UTXO as having been spent at this height. + if (cacheItem.Coins != null) + this.RecordBalanceChange(cacheItem.Coins.TxOut.ScriptPubKey, -cacheItem.Coins.TxOut.Value, (uint)nextBlockHash.Height); + // In cases of an output spent in the same block // it wont exist in cash or in disk so its safe to remove it if (cacheItem.Coins == null) @@ -632,6 +647,9 @@ public void SaveChanges(IList outputs, HashHeightPair oldBlockHas { // ADD COINS + // Update the balance. + this.RecordBalanceChange(output.Coins.TxOut.ScriptPubKey, output.Coins.TxOut.Value, output.Coins.Height); + if (cacheItem.Coins != null) { // Allow overrides. @@ -713,6 +731,7 @@ public HashHeightPair Rewind(HashHeightPair target = null) // All the cached utxos are now on disk so we can clear the cached entry list. this.cachedUtxoItems.Clear(); + this.cacheBalancesByDestination.Clear(); this.cacheSizeBytes = 0; this.dirtyCacheCount = 0; @@ -772,5 +791,62 @@ private void AddBenchStats(StringBuilder log) this.latestPerformanceSnapShot = snapShot; } + + private void RecordBalanceChange(Script scriptPubKey, long satoshis, uint height) + { + if (!this.coindb.BalanceIndexingEnabled || scriptPubKey.Length == 0 || satoshis == 0) + return; + + foreach (TxDestination txDestination in this.scriptAddressReader.GetDestinationFromScriptPubKey(this.network, scriptPubKey)) + { + if (!this.cacheBalancesByDestination.TryGetValue(txDestination, out Dictionary value)) + { + value = new Dictionary(); + this.cacheBalancesByDestination[txDestination] = value; + } + + if (!value.TryGetValue(height, out long balance)) + balance = 0; + + balance += satoshis; + + value[height] = balance; + } + } + + public IEnumerable<(uint, long)> GetBalance(TxDestination txDestination) + { + IEnumerable<(uint, long)> CachedBalances() + { + if (this.cacheBalancesByDestination.TryGetValue(txDestination, out Dictionary itemsByHeight)) + { + long balance = 0; + + foreach (uint height in itemsByHeight.Keys.OrderBy(k => k)) + { + balance += itemsByHeight[height]; + yield return (height, balance); + } + } + } + + bool first = true; + foreach ((uint height, long satoshis) in this.coindb.GetBalance(txDestination)) + { + if (first) + { + first = false; + + foreach ((uint height2, long satoshis2) in CachedBalances().Reverse()) + yield return (height2, satoshis2 + satoshis); + } + + yield return (height, satoshis); + } + + if (first) + foreach ((uint height2, long satoshis2) in CachedBalances().Reverse()) + yield return (height2, satoshis2); + } } -} +} \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs index 8b77dae5fe..593770e2dc 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs @@ -43,7 +43,6 @@ public interface ICoinView /// /// Brings the coinview back on-chain if a re-org occurred. /// - /// The current consensus chain. void Sync(); /// @@ -82,5 +81,15 @@ public interface ICoinView /// /// The height of the block. RewindData GetRewindData(int height); + + /// + /// Returns a combination of (height, satoshis) values with the cumulative balance up to the corresponding height. + /// + /// The destination value derived from the address being queried. + /// A combination of (height, satoshis) values with the cumulative balance up to the corresponding height. + /// Balance updates (even when nett 0) are delivered for every height at which transactions for the address + /// had been recorded and as such the returned heights can be used in conjunction with the block store to discover + /// all related transactions. + IEnumerable<(uint height, long satoshis)> GetBalance(TxDestination txDestination); } } diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs index cb703fb8af..4bee90169b 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs @@ -3,10 +3,12 @@ using System.Linq; using System.Text; using Microsoft.Extensions.Logging; +using Stratis.Bitcoin.Consensus; using NBitcoin; using Stratis.Bitcoin.Configuration; using Stratis.Bitcoin.Configuration.Logging; using Stratis.Bitcoin.Database; +using Stratis.Bitcoin.Interfaces; using Stratis.Bitcoin.Utilities; namespace Stratis.Bitcoin.Features.Consensus.CoinViews @@ -24,6 +26,11 @@ namespace Stratis.Bitcoin.Features.Consensus.CoinViews private static readonly byte blockTable = 2; private static readonly byte rewindTable = 3; private static readonly byte stakeTable = 4; + private static readonly byte balanceTable = 5; + private static readonly byte balanceAdjustmentTable = 6; + + /// Database key under which the block hash of the coin view's last indexed tip is stored. + private static readonly byte[] blockIndexedHashKey = new byte[1]; private readonly string dataFolder; @@ -33,9 +40,15 @@ namespace Stratis.Bitcoin.Features.Consensus.CoinViews /// Specification of the network the node runs on - regtest/testnet/mainnet. private readonly Network network; + /// Indicates whether balance indexing is enabled. + public bool BalanceIndexingEnabled { get; private set; } + /// Hash of the block which is currently the tip of the coinview. private HashHeightPair persistedCoinviewTip; + /// The script address reader. + private readonly IScriptAddressReader scriptAddressReader; + /// Performance counter to measure performance of the database insert and query operations. private readonly BackendPerformanceCounter performanceCounter; @@ -49,7 +62,7 @@ namespace Stratis.Bitcoin.Features.Consensus.CoinViews private const int MaxRewindBatchSize = 10000; public Coindb(Network network, DataFolder dataFolder, IDateTimeProvider dateTimeProvider, - INodeStats nodeStats, DBreezeSerializer dBreezeSerializer) + INodeStats nodeStats, DBreezeSerializer dBreezeSerializer, IScriptAddressReader scriptAddressReader) { Guard.NotNull(network, nameof(network)); Guard.NotNull(dataFolder, nameof(dataFolder)); @@ -59,18 +72,21 @@ public Coindb(Network network, DataFolder dataFolder, IDateTimeProvider dateTime this.logger = LogManager.GetCurrentClassLogger(); this.network = network; this.performanceCounter = new BackendPerformanceCounter(dateTimeProvider); + this.scriptAddressReader = scriptAddressReader; if (nodeStats.DisplayBenchStats) nodeStats.RegisterStats(this.AddBenchStats, StatsType.Benchmark, this.GetType().Name, 400); } /// - public void Initialize() + public void Initialize(bool balanceIndexingEnabled) { // Open a connection to a new DB and create if not found this.coinDb = new T(); this.coinDb.Open(this.dataFolder); + this.BalanceIndexingEnabled = balanceIndexingEnabled; + EnsureCoinDatabaseIntegrity(); Block genesis = this.network.GetGenesis(); @@ -158,13 +174,43 @@ private void EnsureCoinDatabaseIntegrity() return; } + HashHeightPair maxHeight = new HashHeightPair(this.persistedCoinviewTip.Hash, this.persistedCoinviewTip.Height); + + // If the balance table is empty then rebuild the coin db. + if (this.BalanceIndexingEnabled) + { + HashHeightPair indexedTipHash = this.GetIndexedTipHash(); + if (indexedTipHash == null) + { + this.logger.LogInformation($"Rebuilding coin database to include balance information."); + this.coinDb.Clear(); + return; + } + + if (indexedTipHash.Height < maxHeight.Height) + { + this.logger.LogInformation($"Rewinding the coin database to include missing balance information."); + maxHeight = indexedTipHash; + } + } + + for (int height = this.persistedCoinviewTip.Height; height > maxHeight.Height;) + { + this.logger.LogInformation($"Fixing coin database, deleting rewind data at height {height} above tip '{maxHeight}'."); + + // Do a batch of rewinding. + height = RewindInternal(height, maxHeight).Height; + } + this.logger.LogInformation("Coin database integrity good."); } - private void SetBlockHash(IDbBatch batch, HashHeightPair nextBlockHash) + private void SetBlockHash(IDbBatch batch, HashHeightPair nextBlockHash, bool forceUpdateIndexedHeight = false) { this.persistedCoinviewTip = nextBlockHash; batch.Put(blockTable, blockHashKey, nextBlockHash.ToBytes()); + if (this.BalanceIndexingEnabled || forceUpdateIndexedHeight) + batch.Put(blockTable, blockIndexedHashKey, nextBlockHash.ToBytes()); } public HashHeightPair GetTipHash() @@ -182,6 +228,19 @@ public HashHeightPair GetTipHash() return this.persistedCoinviewTip; } + private HashHeightPair GetIndexedTipHash() + { + var row = this.coinDb.Get(blockTable, blockIndexedHashKey); + if (row != null) + { + var tip = new HashHeightPair(); + tip.FromBytes(row); + return tip; + } + + return null; + } + public FetchCoinsResponse FetchCoins(OutPoint[] utxos) { FetchCoinsResponse res = new FetchCoinsResponse(); @@ -204,7 +263,7 @@ public FetchCoinsResponse FetchCoins(OutPoint[] utxos) return res; } - public void SaveChanges(IList unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList) + public void SaveChanges(IList unspentOutputs, Dictionary> balanceUpdates, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList) { int insertedEntities = 0; @@ -313,8 +372,12 @@ private HashHeightPair RewindInternal(int startHeight, HashHeightPair target) { HashHeightPair res = null; + int indexedHeight = this.GetIndexedTipHash()?.Height ?? -1; + using (var batch = this.coinDb.GetReadWriteBatch(coinsTable, rewindTable, blockTable)) { + var balanceAdjustments = new Dictionary>(); + for (int height = startHeight; height > (target?.Height ?? (startHeight - 1)) && height > (startHeight - MaxRewindBatchSize); height--) { byte[] rowKey = BitConverter.GetBytes(height).Reverse().ToArray(); @@ -333,6 +396,10 @@ private HashHeightPair RewindInternal(int startHeight, HashHeightPair target) if (this.TryGetCoins(batch, key, out Coins coins)) { this.logger.LogDebug("Outputs of outpoint '{0}' will be removed.", outPoint); + + if (height <= indexedHeight) + Update(balanceAdjustments, coins.TxOut.ScriptPubKey, coins.Height, -coins.TxOut.Value); + batch.Delete(coinsTable, key); } else @@ -345,12 +412,17 @@ private HashHeightPair RewindInternal(int startHeight, HashHeightPair target) { this.logger.LogDebug("Outputs of outpoint '{0}' will be restored.", rewindDataOutput.OutPoint); batch.Put(coinsTable, rewindDataOutput.OutPoint.ToBytes(), this.dBreezeSerializer.Serialize(rewindDataOutput.Coins)); + + if (height <= indexedHeight) + Update(balanceAdjustments, rewindDataOutput.Coins.TxOut.ScriptPubKey, (uint)height, rewindDataOutput.Coins.TxOut.Value); } res = rewindData.PreviousBlockHash; } - this.SetBlockHash(batch, res); + AdjustBalance(batch, balanceAdjustments); + + this.SetBlockHash(batch, res, res.Height < indexedHeight); batch.Write(); } @@ -409,6 +481,79 @@ private void AddBenchStats(StringBuilder log) this.latestPerformanceSnapShot = snapShot; } + private void AdjustBalance(ReadWriteBatch batch, Dictionary> balanceUpdates) + { + foreach ((TxDestination txDestination, Dictionary balanceAdjustments) in balanceUpdates) + { + long totalAdjustment = 0; + + foreach (uint height in balanceAdjustments.Keys.OrderBy(k => k)) + { + var key = txDestination.ToBytes().Concat(BitConverter.GetBytes(height).Reverse()).ToArray(); + byte[] row = batch.Get(balanceAdjustmentTable, key); + long adjustment = balanceAdjustments[height]; + long balance = ((row == null) ? 0 : BitConverter.ToInt64(row)) + adjustment; + batch.Put(balanceAdjustmentTable, key, BitConverter.GetBytes(balance)); + + totalAdjustment += adjustment; + } + + { + var key = txDestination.ToBytes(); + byte[] row = batch.Get(balanceTable, key); + long balance = ((row == null) ? 0 : BitConverter.ToInt64(row)) + totalAdjustment; + batch.Put(balanceTable, key, BitConverter.GetBytes(balance)); + } + } + } + + private void Update(Dictionary> balanceAdjustments, Script scriptPubKey, uint height, long change) + { + if (scriptPubKey.Length == 0 || change == 0) + return; + + foreach (TxDestination txDestination in this.scriptAddressReader.GetDestinationFromScriptPubKey(this.network, scriptPubKey)) + { + if (!balanceAdjustments.TryGetValue(txDestination, out Dictionary value)) + { + value = new Dictionary(); + balanceAdjustments[txDestination] = value; + } + + if (!value.TryGetValue(height, out long balance)) + balance = change; + else + balance += change; + + value[height] = balance; + } + } + + /// + public IEnumerable<(uint height, long satoshis)> GetBalance(TxDestination txDestination) + { + long balance; + { + byte[] row = this.coinDb.Get(balanceTable, txDestination.ToBytes()); + balance = (row == null) ? 0 : BitConverter.ToInt64(row); + } + + using (var iterator = this.coinDb.GetIterator(balanceAdjustmentTable)) + { + foreach ((uint height, long adjustment) in iterator.GetAll(ascending: false, + lastKey: txDestination.ToBytes().Concat(BitConverter.GetBytes(this.persistedCoinviewTip.Height + 1).Reverse()).ToArray(), + includeLastKey: false, + firstKey: txDestination.ToBytes(), + includeFirstKey: false).Select(x => (height: BitConverter.ToUInt32(x.Item1.Reverse().ToArray()), adjustment: BitConverter.ToInt64(x.Item2)))) + { + yield return (height, balance); + balance -= adjustment; + } + } + + yield return (0, balance); + } + /// public void Dispose() { diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs index 01cd11ace5..e30f1ced55 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs @@ -10,7 +10,10 @@ namespace Stratis.Bitcoin.Features.Consensus.CoinViews public interface ICoindb { /// Initialize the coin database. - void Initialize(); + /// Indicates whether to enable balance indexing. + void Initialize(bool balanceIndexingEnabled); + + bool BalanceIndexingEnabled { get; } /// /// Retrieves the block hash of the current tip of the coinview. @@ -30,10 +33,11 @@ public interface ICoindb /// /// Information about the changes between the old block and the new block. An item in this list represents a list of all outputs /// for a specific transaction. If a specific output was spent, the output is null. + /// Non-cumulative balance updates at each height. /// Block hash of the current tip of the coinview. /// Block hash of the tip of the coinview after the change is applied. /// List of rewind data items to be persisted. - void SaveChanges(IList unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList); + void SaveChanges(IList unspentOutputs, Dictionary> balanceUpdates, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList); /// /// Obtains information about unspent outputs. @@ -68,13 +72,15 @@ public interface ICoindb /// See . RewindData GetRewindData(int height); - /// Gets the minimum rewind height. - /// - /// - /// The minimum rewind height or -1 if rewind is not possible. - /// - /// - int GetMinRewindHeight(); + /// + /// Returns a combination of (height, satoshis) values with the cumulative balance up to the corresponding height. + /// + /// The destination value derived from the address being queried. + /// A combination of (height, satoshis) values with the cumulative balance up to the corresponding height. + /// Balance updates (even when nett 0) are delivered for every height at which transactions for the address + /// had been recorded and as such the returned heights can be used in conjunction with the block store to discover + /// all related transactions. + IEnumerable<(uint height, long satoshis)> GetBalance(TxDestination txDestination); } public interface IStakedb : ICoindb diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs index 4df310e6ef..954fab33a8 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs @@ -24,6 +24,8 @@ public class InMemoryCoinView : ICoinView, ICoindb /// All access to this object has to be protected by . private HashHeightPair tipHash; + public bool BalanceIndexingEnabled => false; + /// /// Initializes an instance of the object. /// @@ -39,6 +41,12 @@ public void Initialize(IConsensusManager consensusManager) throw new NotImplementedException(); } + /// + public void Initialize(bool balanceIndexingEnabled) + { + throw new NotImplementedException(); + } + /// public void Sync() { @@ -55,6 +63,11 @@ public void CacheCoins(OutPoint[] utxos) throw new NotImplementedException(); } + public IEnumerable<(uint height, long satoshis)> GetBalance(TxDestination txDestination) + { + throw new NotImplementedException(); + } + /// public FetchCoinsResponse FetchCoins(OutPoint[] txIds) { @@ -107,6 +120,11 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB } } + public void SaveChanges(IList unspentOutputs, Dictionary> balanceUpdates, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList = null) + { + this.SaveChanges(unspentOutputs, oldBlockHash, nextBlockHash, rewindDataList); + } + public int GetMinRewindHeight() { throw new NotImplementedException(); @@ -121,9 +139,5 @@ public RewindData GetRewindData(int height) { throw new NotImplementedException(); } - - public void Initialize() - { - } } } diff --git a/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs b/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs index 460f1014c9..27eccb56ea 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs @@ -30,6 +30,7 @@ public static IFullNodeBuilder UsePowConsensus(this IFullNodeBuilder fullNodeBui { ConfigureCoinDatabaseImplementation(services, coindbType); + services.Replace((p, old) => old ?? new ScriptAddressReader(), ServiceLifetime.Singleton); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs b/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs index f4862e5e46..5cbee4760d 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs @@ -60,6 +60,11 @@ public void Sync() { } + public IEnumerable<(uint height, long satoshis)> GetBalance(TxDestination txDestination) + { + throw new NotImplementedException(); + } + /// /// Gets the unspent transaction output set. /// diff --git a/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs b/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs index 28fade3589..7c092081ce 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs @@ -47,14 +47,14 @@ public void TestDatabaseSerialization() Block genesis = ctx.Network.GetGenesis(); var genesisChainedHeader = new ChainedHeader(genesis.Header, ctx.Network.GenesisHash, 0); ChainedHeader chained = this.MakeNext(genesisChainedHeader, ctx.Network); - ctx.Coindb.SaveChanges(new UnspentOutput[] { new UnspentOutput(new OutPoint(genesis.Transactions[0], 0), new Coins(0, genesis.Transactions[0].Outputs.First(), true)) }, new HashHeightPair(genesisChainedHeader), new HashHeightPair(chained), new List()); + ctx.Coindb.SaveChanges(new UnspentOutput[] { new UnspentOutput(new OutPoint(genesis.Transactions[0], 0), new Coins(0, genesis.Transactions[0].Outputs.First(), true)) }, new Dictionary>(), new HashHeightPair(genesisChainedHeader), new HashHeightPair(chained), new List()); Assert.NotNull(ctx.Coindb.FetchCoins(new[] { new OutPoint(genesis.Transactions[0], 0) }).UnspentOutputs.Values.FirstOrDefault().Coins); Assert.Null(ctx.Coindb.FetchCoins(new[] { new OutPoint() }).UnspentOutputs.Values.FirstOrDefault().Coins); ChainedHeader previous = chained; chained = this.MakeNext(this.MakeNext(genesisChainedHeader, ctx.Network), ctx.Network); chained = this.MakeNext(this.MakeNext(genesisChainedHeader, ctx.Network), ctx.Network); - ctx.Coindb.SaveChanges(new List(), new HashHeightPair(previous), new HashHeightPair(chained), new List()); + ctx.Coindb.SaveChanges(new List(), new Dictionary>(), new HashHeightPair(previous), new HashHeightPair(chained), new List()); Assert.Equal(chained.HashBlock, ctx.Coindb.GetTipHash().Hash); ctx.ReloadPersistentCoinView(); Assert.Equal(chained.HashBlock, ctx.Coindb.GetTipHash().Hash); diff --git a/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs b/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs index 58f39ef1eb..4792ef81ce 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs @@ -5,6 +5,7 @@ using Moq; using NBitcoin; using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Features.Consensus.CoinViews; using Stratis.Bitcoin.Interfaces; @@ -28,8 +29,8 @@ public NodeContext(object caller, string name, Network network) this.FolderName = TestBase.CreateTestDir(caller, name); var dateTimeProvider = new DateTimeProvider(); var serializer = new DBreezeSerializer(this.Network.Consensus.ConsensusFactory); - this.Coindb = new Coindb(network, new DataFolder(this.FolderName), dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(network), new Mock().Object), serializer); - this.Coindb.Initialize(); + this.Coindb = new Coindb(network, new DataFolder(this.FolderName), dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(network), new Mock().Object), serializer, new ScriptAddressReader()); + this.Coindb.Initialize(false); this.cleanList = new List { (IDisposable)this.Coindb }; } @@ -66,9 +67,9 @@ public void ReloadPersistentCoinView() this.cleanList.Remove((IDisposable)this.Coindb); var dateTimeProvider = new DateTimeProvider(); var serializer = new DBreezeSerializer(this.Network.Consensus.ConsensusFactory); - this.Coindb = new Coindb(this.Network, new DataFolder(this.FolderName), dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(this.Network), new Mock().Object), serializer); + this.Coindb = new Coindb(this.Network, new DataFolder(this.FolderName), dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(this.Network), new Mock().Object), serializer, new ScriptAddressReader()); - this.Coindb.Initialize(); + this.Coindb.Initialize(false); this.cleanList.Add((IDisposable)this.Coindb); } } diff --git a/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs b/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs index d8ca898d1d..1500101506 100644 --- a/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs +++ b/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs @@ -75,6 +75,11 @@ public FetchCoinsResponse FetchCoins(OutPoint[] txIds) } } + public IEnumerable<(uint, long)> GetBalance(TxDestination txDestination) + { + throw new NotImplementedException(); + } + /// public void SaveChanges(IList unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList = null) {