Skip to content

Commit

Permalink
Fix endianness of delete on rewind table (stratisproject#1080)
Browse files Browse the repository at this point in the history 
* Fix endianness of delete on rewind table

* Fix duplicate call

* Fix consistency

* Update mock

(cherry picked from commit 81d3ad7)
  • Loading branch information
@quantumagi
quantumagi committed Jan 31, 2023
1 parent 5440972 commit 4170264
Show file tree
Hide file tree
Showing 8 changed files with 276 additions and 60 deletions.
71 changes: 61 additions & 10 deletions src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerTests.cs
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,24 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading;
using LiteDB;
using Microsoft.Extensions.DependencyInjection;
using Moq;
using NBitcoin;
using Stratis.Bitcoin.Configuration;
using Stratis.Bitcoin.Consensus;
using Stratis.Bitcoin.Consensus.Rules;
using Stratis.Bitcoin.Controllers.Models;
using Stratis.Bitcoin.Database;
using Stratis.Bitcoin.Features.BlockStore.AddressIndexing;
using Stratis.Bitcoin.Features.BlockStore.Repositories;
using Stratis.Bitcoin.Features.Consensus.CoinViews;
using Stratis.Bitcoin.Features.Consensus.Rules;
using Stratis.Bitcoin.Features.Consensus.Rules.CommonRules;
using Stratis.Bitcoin.Interfaces;
using Stratis.Bitcoin.Networks;
using Stratis.Bitcoin.Primitives;
using Stratis.Bitcoin.Tests.Common;
Expand All @@ -26,30 +35,49 @@ public class AddressIndexerTests

private readonly Mock<IConsensusManager> consensusManagerMock;

private readonly ChainIndexer chainIndexer;

private readonly Network network;

private readonly IConsensusRuleEngine consensusRuleEngine;

private readonly ChainedHeader genesisHeader;

public AddressIndexerTests()
{
this.network = new StraxMain();
this.chainIndexer = new ChainIndexer(this.network);
var nodeSettings = new NodeSettings(this.network, args: new[] { "-addressindex", "-txindex" });

var mockingServices = new ServiceCollection()
.AddSingleton(this.network)
.AddSingleton(new StoreSettings(NodeSettings.Default(this.network))
{
AddressIndex = true,
TxIndex = true
})
.AddSingleton(nodeSettings)
.AddSingleton(nodeSettings.LoggerFactory)
.AddSingleton(new DataFolder(TestBase.CreateTestDir(this)))
.AddSingleton(new ChainIndexer(this.network))
.AddSingleton<IScriptAddressReader, ScriptAddressReader>()
.AddSingleton<ConsensusRulesContainer>()
.AddSingleton<IConsensusRuleEngine, PosConsensusRuleEngine>()
.AddSingleton<IBlockRepository>(typeof(BlockRepository<LevelDb>).GetConstructors().First(c => c.GetParameters().Any(p => p.ParameterType == typeof(DataFolder))))
.AddSingleton<IBlockStore, BlockStoreQueue>()
.AddSingleton<ICoindb>(typeof(Coindb<LevelDb>).GetConstructors().First(c => c.GetParameters().Any(p => p.ParameterType == typeof(DataFolder))))
.AddSingleton<ICoinView, CachedCoinView>()
.AddSingleton(this.chainIndexer)
.AddSingleton<IDateTimeProvider, DateTimeProvider>()
.AddSingleton<IAddressIndexer, AddressIndexer>();

var mockingContext = new MockingContext(mockingServices);

this.addressIndexer = mockingContext.GetService<IAddressIndexer>();
this.genesisHeader = mockingContext.GetService<ChainIndexer>().GetHeader(0);

var rulesContainer = mockingContext.GetService<ConsensusRulesContainer>();
rulesContainer.FullValidationRules.Add(Activator.CreateInstance(typeof(LoadCoinviewRule)) as FullValidationConsensusRule);
rulesContainer.FullValidationRules.Add(Activator.CreateInstance(typeof(SaveCoinviewRule)) as FullValidationConsensusRule);
rulesContainer.FullValidationRules.Add(Activator.CreateInstance(typeof(StraxCoinviewRule)) as FullValidationConsensusRule);
rulesContainer.FullValidationRules.Add(Activator.CreateInstance(typeof(SetActivationDeploymentsFullValidationRule)) as FullValidationConsensusRule);

this.consensusManagerMock = mockingContext.GetService<Mock<IConsensusManager>>();
this.consensusRuleEngine = mockingContext.GetService<IConsensusRuleEngine>();
}

[Fact]
Expand All @@ -64,7 +92,7 @@ public void CanInitializeAndDispose()
[Fact]
public void CanIndexAddresses()
{
List<ChainedHeader> headers = ChainedHeadersHelper.CreateConsecutiveHeaders(100, null, false, null, this.network);
List<ChainedHeader> headers = ChainedHeadersHelper.CreateConsecutiveHeaders(100, null, false, null, this.network, chainIndexer: this.chainIndexer);
this.consensusManagerMock.Setup(x => x.Tip).Returns(() => headers.Last());

Script p2pk1 = this.GetRandomP2PKScript(out string address1);
Expand Down Expand Up @@ -106,7 +134,7 @@ public void CanIndexAddresses()
tx.Inputs.Add(new TxIn(new OutPoint(block5.Transactions.First().GetHash(), 0)));
var block10 = new Block() { Transactions = new List<Transaction>() { tx } };

this.consensusManagerMock.Setup(x => x.GetBlockData(It.IsAny<uint256>())).Returns((uint256 hash) =>
ChainedHeaderBlock GetChainedHeaderBlock(uint256 hash)
{
ChainedHeader header = headers.SingleOrDefault(x => x.HashBlock == hash);

Expand All @@ -123,8 +151,26 @@ public void CanIndexAddresses()
}

return new ChainedHeaderBlock(new Block(), header);
}

this.consensusManagerMock.Setup(x => x.GetBlockData(It.IsAny<uint256>())).Returns((uint256 hash) =>
{
return GetChainedHeaderBlock(hash);
});

this.consensusManagerMock.Setup(x => x.GetBlockData(It.IsAny<List<uint256>>())).Returns((List<uint256> hashes) =>
{
return hashes.Select(h => GetChainedHeaderBlock(h)).ToArray();
});

this.consensusManagerMock.Setup(x => x.GetBlocksAfterBlock(It.IsAny<ChainedHeader>(), It.IsAny<int>(), It.IsAny<CancellationTokenSource>())).Returns((ChainedHeader header, int size, CancellationTokenSource token) =>
{
return headers.Where(h => h.Height > header.Height).Select(h => GetChainedHeaderBlock(h.HashBlock)).ToArray();
});

this.consensusManagerMock.Setup(x => x.ConsensusRules).Returns(this.consensusRuleEngine);

this.consensusRuleEngine.Initialize(headers.Last(), this.consensusManagerMock.Object);
this.addressIndexer.Initialize();

TestBase.WaitLoop(() => this.addressIndexer.IndexerTip == headers.Last());
Expand All @@ -137,7 +183,7 @@ public void CanIndexAddresses()
// Now trigger rewind to see if indexer can handle reorgs.
ChainedHeader forkPoint = headers.Single(x => x.Height == 8);

List<ChainedHeader> headersFork = ChainedHeadersHelper.CreateConsecutiveHeaders(100, forkPoint, false, null, this.network);
List<ChainedHeader> headersFork = ChainedHeadersHelper.CreateConsecutiveHeaders(100, forkPoint, false, null, this.network, chainIndexer: this.chainIndexer);

this.consensusManagerMock.Setup(x => x.GetBlockData(It.IsAny<uint256>())).Returns((uint256 hash) =>
{
Expand All @@ -146,6 +192,11 @@ public void CanIndexAddresses()
return new ChainedHeaderBlock(new Block(), headerFork);
});

this.consensusManagerMock.Setup(x => x.GetBlocksAfterBlock(It.IsAny<ChainedHeader>(), It.IsAny<int>(), It.IsAny<CancellationTokenSource>())).Returns((ChainedHeader header, int size, CancellationTokenSource token) =>
{
return headersFork.Where(h => h.Height > header.Height).Select(h => new ChainedHeaderBlock(new Block(), h)).ToArray();
});

this.consensusManagerMock.Setup(x => x.Tip).Returns(() => headersFork.Last());
TestBase.WaitLoop(() => this.addressIndexer.IndexerTip == headersFork.Last());

Expand Down
2 changes: 1 addition & 1 deletion src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ private async Task IndexAddressesContinuouslyAsync()
if (prefetchedBlock != null && prefetchedBlock.ChainedHeader == nextHeader)
blockToProcess = prefetchedBlock.Block;
else
blockToProcess = this.consensusManager.GetBlockData(nextHeader.HashBlock).Block;
blockToProcess = this.consensusManager.GetBlockData(nextHeader.HashBlock)?.Block;

if (blockToProcess == null)
{
Expand Down
72 changes: 43 additions & 29 deletions src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@
using System.Threading;
using Microsoft.Extensions.Logging;
using NBitcoin;
using Stratis.Bitcoin.Configuration;
using Stratis.Bitcoin.Configuration.Settings;
using Stratis.Bitcoin.Consensus;
using Stratis.Bitcoin.Features.Consensus.ProvenBlockHeaders;
using Stratis.Bitcoin.Features.Consensus.Rules.CommonRules;
using Stratis.Bitcoin.Interfaces;
using Stratis.Bitcoin.Primitives;
using Stratis.Bitcoin.Utilities;
using Stratis.Bitcoin.Utilities.Extensions;
using TracerAttributes;
Expand Down Expand Up @@ -128,16 +129,16 @@ public long GetScriptSize
private DateTime lastCacheFlushTime;
private readonly Network network;
private readonly IDateTimeProvider dateTimeProvider;
private readonly IBlockStore blockStore;
private readonly CancellationTokenSource cancellationToken;
private IConsensusManager consensusManager;
private readonly ConsensusSettings consensusSettings;
private readonly ChainIndexer chainIndexer;
private CachePerformanceSnapshot latestPerformanceSnapShot;

private readonly Random random;

public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, ChainIndexer chainIndexer,
StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null, IBlockStore blockStore = null, INodeLifetime nodeLifetime = null)
public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, ChainIndexer chainIndexer,
StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null, INodeLifetime nodeLifetime = null, NodeSettings nodeSettings = null)
{
Guard.NotNull(coindb, nameof(CachedCoinView.coindb));

Expand All @@ -149,7 +150,6 @@ public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTim
this.chainIndexer = chainIndexer;
this.stakeChainStore = stakeChainStore;
this.rewindDataIndexCache = rewindDataIndexCache;
this.blockStore = blockStore;
this.cancellationToken = (nodeLifetime == null) ? new CancellationTokenSource() : CancellationTokenSource.CreateLinkedTokenSource(nodeLifetime.ApplicationStopping);
this.lockobj = new object();
this.cachedUtxoItems = new Dictionary<OutPoint, CacheItem>();
Expand Down Expand Up @@ -178,50 +178,55 @@ public void Sync()

Flush();

ChainedHeader fork = this.chainIndexer[coinViewTip.Hash];
if (fork == null)
if (coinViewTip.Height > this.chainIndexer.Height || this.chainIndexer[coinViewTip.Hash] == null)
{
// Determine the last usable height.
int height = BinarySearch.BinaryFindFirst(h => (h > this.chainIndexer.Height) || this.GetRewindData(h).PreviousBlockHash.Hash != this.chainIndexer[h].Previous.HashBlock, 0, coinViewTip.Height + 1) - 1;
fork = this.chainIndexer[(height < 0) ? coinViewTip.Height : height];
}
// The coinview tip is above the chain height or on a fork.
// Determine the first unusable height by finding the first rewind data that is not on the consensus chain.
int unusableHeight = BinarySearch.BinaryFindFirst(h => (h > this.chainIndexer.Height) || (this.GetRewindData(h)?.PreviousBlockHash.Hash != this.chainIndexer[h].Previous.HashBlock), 2, coinViewTip.Height - 1);
ChainedHeader fork = this.chainIndexer[unusableHeight - 2];

while (coinViewTip.Height != fork.Height)
{
if ((coinViewTip.Height % 100) == 0)
this.logger.LogInformation("Rewinding coin view from '{0}' to {1}.", coinViewTip, fork);
while (coinViewTip.Height != fork.Height)
{
if ((coinViewTip.Height % 100) == 0)
this.logger.LogInformation("Rewinding coin view from '{0}' to {1}.", coinViewTip, fork);

// If the block store was initialized behind the coin view's tip, rewind it to on or before it's tip.
// The node will complete loading before connecting to peers so the chain will never know that a reorg happened.
coinViewTip = this.coindb.Rewind(new HashHeightPair(fork));
};

this.blockHash = coinViewTip;
this.innerBlockHash = this.blockHash;
}

// If the block store was initialized behind the coin view's tip, rewind it to on or before it's tip.
// The node will complete loading before connecting to peers so the chain will never know that a reorg happened.
coinViewTip = this.coindb.Rewind(new HashHeightPair(fork));
};
CatchUp();
}
}

public void Initialize(IConsensusManager consensusManager)
private void CatchUp()
{
ChainedHeader chainTip = this.chainIndexer.Tip;

this.coindb.Initialize();

Sync();

HashHeightPair coinViewTip = this.coindb.GetTipHash();

// If the coin view is behind the block store then catch up from the block store.
if (coinViewTip.Height < chainTip.Height)
{
try
{
IConsensusRuleEngine consensusRuleEngine = consensusManager.ConsensusRules;
IConsensusRuleEngine consensusRuleEngine = this.consensusManager.ConsensusRules;

var loadCoinViewRule = consensusRuleEngine.GetRule<LoadCoinviewRule>();
var saveCoinViewRule = consensusRuleEngine.GetRule<SaveCoinviewRule>();
var coinViewRule = consensusRuleEngine.GetRule<CoinViewRule>();
var deploymentsRule = consensusRuleEngine.GetRule<SetActivationDeploymentsFullValidationRule>();

foreach ((ChainedHeader chainedHeader, Block block) in this.blockStore.BatchBlocksFrom(this.chainIndexer[coinViewTip.Hash], this.chainIndexer, this.cancellationToken, batchSize: 1000))
foreach (ChainedHeaderBlock chb in this.consensusManager.GetBlocksAfterBlock(this.chainIndexer[coinViewTip.Hash], 1000, this.cancellationToken))
{
ChainedHeader chainedHeader = chb?.ChainedHeader;
if (chainedHeader == null)
break;

Block block = chb.Block;
if (block == null)
break;

Expand All @@ -244,7 +249,7 @@ public void Initialize(IConsensusManager consensusManager)
coinViewRule.RunAsync(utxoRuleContext).ConfigureAwait(false).GetAwaiter().GetResult();

// Saves the changes to the coinview.
saveCoinViewRule.RunAsync(utxoRuleContext).ConfigureAwait(false).GetAwaiter().GetResult();
saveCoinViewRule.RunAsync(utxoRuleContext).ConfigureAwait(false).GetAwaiter().GetResult();
}
}
finally
Expand All @@ -258,6 +263,15 @@ public void Initialize(IConsensusManager consensusManager)
}
}
}
}

public void Initialize(IConsensusManager consensusManager)
{
this.consensusManager = consensusManager;

this.coindb.Initialize();

Sync();

this.logger.LogInformation("Coin view initialized at '{0}'.", this.coindb.GetTipHash());
}
Expand Down Expand Up @@ -485,7 +499,7 @@ public void Flush(bool force = true)
this.cachedRewindData.Clear();
this.rewindDataSizeBytes = 0;
this.dirtyCacheCount = 0;
this.innerBlockHash = this.blockHash;
this.innerBlockHash = this.blockHash;
this.lastCacheFlushTime = this.dateTimeProvider.GetUtcNow();
}
}
Expand Down
Loading

0 comments on commit 4170264

Please sign in to comment.