From adf8cea09e2d72230b15038525e5cc37d615af29 Mon Sep 17 00:00:00 2001 From: Alessio Treglia Date: Thu, 11 Oct 2018 20:37:21 -0700 Subject: [PATCH 01/55] Pass nil to NewValidatorSet() when genesis file's Validators field is nil (#2617) Closes: #2616 --- CHANGELOG_PENDING.md | 1 + state/state.go | 32 ++++++++++++++++++++------------ state/state_test.go | 13 +++++++++++++ 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 81c7a3a2932..d8ed9a94c7a 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -15,3 +15,4 @@ FEATURES: IMPROVEMENTS: BUG FIXES: +- [state] \#2616 Pass nil to NewValidatorSet() when genesis file's Validators field is nil diff --git a/state/state.go b/state/state.go index 26510816b69..1f60fd6534e 100644 --- a/state/state.go +++ b/state/state.go @@ -198,17 +198,25 @@ func MakeGenesisState(genDoc *types.GenesisDoc) (State, error) { } // Make validators slice - validators := make([]*types.Validator, len(genDoc.Validators)) - for i, val := range genDoc.Validators { - pubKey := val.PubKey - address := pubKey.Address() - - // Make validator - validators[i] = &types.Validator{ - Address: address, - PubKey: pubKey, - VotingPower: val.Power, + var validatorSet, nextValidatorSet *types.ValidatorSet + if genDoc.Validators == nil { + validatorSet = types.NewValidatorSet(nil) + nextValidatorSet = types.NewValidatorSet(nil) + } else { + validators := make([]*types.Validator, len(genDoc.Validators)) + for i, val := range genDoc.Validators { + pubKey := val.PubKey + address := pubKey.Address() + + // Make validator + validators[i] = &types.Validator{ + Address: address, + PubKey: pubKey, + VotingPower: val.Power, + } } + validatorSet = types.NewValidatorSet(validators) + nextValidatorSet = types.NewValidatorSet(validators).CopyIncrementAccum(1) } return State{ @@ -219,8 +227,8 @@ func MakeGenesisState(genDoc *types.GenesisDoc) (State, error) { LastBlockID: types.BlockID{}, LastBlockTime: genDoc.GenesisTime, - NextValidators: types.NewValidatorSet(validators).CopyIncrementAccum(1), - Validators: types.NewValidatorSet(validators), + NextValidators: nextValidatorSet, + Validators: validatorSet, LastValidators: types.NewValidatorSet(nil), LastHeightValidatorsChanged: 1, diff --git a/state/state_test.go b/state/state_test.go index 1ab470b0241..2c777307aec 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -48,6 +48,19 @@ func TestStateCopy(t *testing.T) { %v`, state)) } +//TestMakeGenesisStateNilValidators tests state's consistency when genesis file's validators field is nil. +func TestMakeGenesisStateNilValidators(t *testing.T) { + doc := types.GenesisDoc{ + ChainID: "dummy", + Validators: nil, + } + require.Nil(t, doc.ValidateAndComplete()) + state, err := MakeGenesisState(&doc) + require.Nil(t, err) + require.Equal(t, 0, len(state.Validators.Validators)) + require.Equal(t, 0, len(state.NextValidators.Validators)) +} + // TestStateSaveLoad tests saving and loading State from a db. func TestStateSaveLoad(t *testing.T) { tearDown, stateDB, state := setupTestCase(t) From 72201676b281038d2a398ce1258a4ee9c9deb6e3 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 15 Oct 2018 06:28:41 +0400 Subject: [PATCH 02/55] set next validators along with validators while replay (#2637) Closes #2634 --- consensus/replay.go | 1 + 1 file changed, 1 insertion(+) diff --git a/consensus/replay.go b/consensus/replay.go index c92654f2ce9..af6369c3bae 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -287,6 +287,7 @@ func (h *Handshaker) ReplayBlocks(state sm.State, appHash []byte, appBlockHeight return nil, err } state.Validators = types.NewValidatorSet(vals) + state.NextValidators = types.NewValidatorSet(vals) } if res.ConsensusParams != nil { state.ConsensusParams = types.PB2TM.ConsensusParams(res.ConsensusParams) From 1ad379aa874f6b81817384baa5d49ee87339081d Mon Sep 17 00:00:00 2001 From: Alessio Treglia Date: Wed, 17 Oct 2018 18:55:23 -0700 Subject: [PATCH 03/55] Update CHANGELOG_PENDING.md --- CHANGELOG_PENDING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index d8ed9a94c7a..1453630f315 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -15,4 +15,5 @@ FEATURES: IMPROVEMENTS: BUG FIXES: -- [state] \#2616 Pass nil to NewValidatorSet() when genesis file's Validators field is nil +- \#2616 Pass nil to NewValidatorSet() when genesis file's Validators field is nil +- \#2634 Set next validators along with validators while replay From 90eda9bfb6e6daeed1c8015df41cb36772d91778 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 17 Oct 2018 22:09:48 -0400 Subject: [PATCH 04/55] changelog and version --- CHANGELOG.md | 13 +++++++++++-- CHANGELOG_PENDING.md | 2 -- version/version.go | 4 ++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6032fc20491..4d4d787890a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## v0.25.1 + +*October 17, 2018* + +BUG FIXES: + +- [state] [\#2616](https://github.com/tendermint/tendermint/issues/2616) Fix panic when genesis file's `validators` field is nil +- [consensus] [\#2634](https://github.com/tendermint/tendermint/issues/2634) Set `NextValidators` during replay + ## v0.25.0 *September 22, 2018* @@ -164,8 +173,8 @@ BUG FIXES: *August 22nd, 2018* BUG FIXES: -- [libs/autofile] \#2261 Fix log rotation so it actually happens. - - Fixes issues with consensus WAL growing unbounded ala \#2259 +- [libs/autofile] [\#2261](https://github.com/tendermint/tendermint/issues/2261) Fix log rotation so it actually happens. + - Fixes issues with consensus WAL growing unbounded ala [\#2259](https://github.com/tendermint/tendermint/issues/2259) ## 0.23.0 diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 1453630f315..81c7a3a2932 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -15,5 +15,3 @@ FEATURES: IMPROVEMENTS: BUG FIXES: -- \#2616 Pass nil to NewValidatorSet() when genesis file's Validators field is nil -- \#2634 Set next validators along with validators while replay diff --git a/version/version.go b/version/version.go index d8bab5772ca..5c09ce36e60 100644 --- a/version/version.go +++ b/version/version.go @@ -4,13 +4,13 @@ package version const ( Maj = "0" Min = "25" - Fix = "0" + Fix = "1" ) var ( // Version is the current version of Tendermint // Must be a string because scripts like dist.sh read this file. - Version = "0.25.0" + Version = "0.25.1" // GitCommit is the current HEAD set using ldflags. GitCommit string From c1a7c784d8c1e515124139b57d79946852657582 Mon Sep 17 00:00:00 2001 From: Raymond Date: Mon, 22 Oct 2018 16:39:20 +0800 Subject: [PATCH 05/55] upgrade to version 0.25.1-rc0 --- state/execution.go | 8 +++++++- state/state.go | 3 +++ state/store.go | 16 +++++++++++++++- version/version.go | 2 +- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/state/execution.go b/state/execution.go index c6d5ce0a11d..fae23436b2c 100644 --- a/state/execution.go +++ b/state/execution.go @@ -86,7 +86,10 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b fail.Fail() // XXX - // Update the state with the block and responses. + // update the state with the block and responses + //////////////////// iris/tendermint begin /////////////////////////// + preState := state.Copy() + //////////////////// iris/tendermint end /////////////////////////// state, err = updateState(state, blockID, &block.Header, abciResponses) if err != nil { return state, fmt.Errorf("Commit failed for application: %v", err) @@ -106,6 +109,9 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b // Update the app hash and save the state. state.AppHash = appHash SaveState(blockExec.db, state) + //////////////////// iris/tendermint begin /////////////////////////// + SavePreState(blockExec.db, preState) + //////////////////// iris/tendermint end /////////////////////////// fail.Fail() // XXX diff --git a/state/state.go b/state/state.go index 1f60fd6534e..6ed906acaa8 100644 --- a/state/state.go +++ b/state/state.go @@ -13,6 +13,9 @@ import ( // database keys var ( stateKey = []byte("stateKey") + //////////////////// iris/tendermint begin /////////////////////////// + statePreKey = []byte("statePreKey") + //////////////////// iris/tendermint end /////////////////////////// ) //----------------------------------------------------------------------------- diff --git a/state/store.go b/state/store.go index 2f90c747ea3..c859d59e357 100644 --- a/state/store.go +++ b/state/store.go @@ -62,6 +62,12 @@ func LoadState(db dbm.DB) State { return loadState(db, stateKey) } +//////////////////// iris/tendermint begin /////////////////////////// +func LoadPreState(db dbm.DB) State { + return loadState(db, statePreKey) +} +//////////////////// iris/tendermint end /////////////////////////// + func loadState(db dbm.DB, key []byte) (state State) { buf := db.Get(key) if len(buf) == 0 { @@ -85,6 +91,12 @@ func SaveState(db dbm.DB, state State) { saveState(db, state, stateKey) } +//////////////////// iris/tendermint begin /////////////////////////// +func SavePreState(db dbm.DB, state State) { + saveState(db, state, statePreKey) +} +//////////////////// iris/tendermint end /////////////////////////// + func saveState(db dbm.DB, state State, key []byte) { nextHeight := state.LastBlockHeight + 1 // If first block, save validators for block 1. @@ -96,7 +108,9 @@ func saveState(db dbm.DB, state State, key []byte) { saveValidatorsInfo(db, nextHeight+1, state.LastHeightValidatorsChanged, state.NextValidators) // Save next consensus params. saveConsensusParamsInfo(db, nextHeight, state.LastHeightConsensusParamsChanged, state.ConsensusParams) - db.SetSync(stateKey, state.Bytes()) + //////////////////// iris/tendermint begin /////////////////////////// + db.SetSync(key, state.Bytes()) + //////////////////// iris/tendermint end /////////////////////////// } //------------------------------------------------------------------------ diff --git a/version/version.go b/version/version.go index 5c09ce36e60..887bb45666e 100644 --- a/version/version.go +++ b/version/version.go @@ -10,7 +10,7 @@ const ( var ( // Version is the current version of Tendermint // Must be a string because scripts like dist.sh read this file. - Version = "0.25.1" + Version = "0.25.1-rc0-iris" // GitCommit is the current HEAD set using ldflags. GitCommit string From 58574b7372cc9dd224ef04aaa438578f72e69a77 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Fri, 9 Nov 2018 15:45:50 -0800 Subject: [PATCH 06/55] Require addressbook to only store addresses with valid ID --- p2p/netaddress.go | 12 ++++++++++++ p2p/pex/addrbook.go | 4 ++++ p2p/pex/errors.go | 8 ++++++++ 3 files changed, 24 insertions(+) diff --git a/p2p/netaddress.go b/p2p/netaddress.go index ec9a0ea7c0f..298ebf0d82e 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -218,10 +218,22 @@ func (na *NetAddress) Routable() bool { // For IPv4 these are either a 0 or all bits set address. For IPv6 a zero // address or one that matches the RFC3849 documentation address format. func (na *NetAddress) Valid() bool { + if string(na.ID) != "" { + data, err := hex.DecodeString(string(na.ID)) + if err != nil || len(data) != IDByteLength { + return false + } + } return na.IP != nil && !(na.IP.IsUnspecified() || na.RFC3849() || na.IP.Equal(net.IPv4bcast)) } +// HasID returns true if the address has an ID. +// NOTE: It does not check whether the ID is valid or not. +func (na *NetAddress) HasID() bool { + return string(na.ID) != "" +} + // Local returns true if it is a local address. func (na *NetAddress) Local() bool { return na.IP.IsLoopback() || zero4.Contains(na.IP) diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index 61710bbf221..405a4628b65 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -652,6 +652,10 @@ func (a *addrBook) addAddress(addr, src *p2p.NetAddress) error { return ErrAddrBookInvalidAddr{addr} } + if !addr.HasID() { + return ErrAddrBookInvalidAddrNoID{addr} + } + // TODO: we should track ourAddrs by ID and by IP:PORT and refuse both. if _, ok := a.ourAddrs[addr.String()]; ok { return ErrAddrBookSelf{addr} diff --git a/p2p/pex/errors.go b/p2p/pex/errors.go index fbee748ac66..1f44ceee7e0 100644 --- a/p2p/pex/errors.go +++ b/p2p/pex/errors.go @@ -54,3 +54,11 @@ type ErrAddrBookInvalidAddr struct { func (err ErrAddrBookInvalidAddr) Error() string { return fmt.Sprintf("Cannot add invalid address %v", err.Addr) } + +type ErrAddrBookInvalidAddrNoID struct { + Addr *p2p.NetAddress +} + +func (err ErrAddrBookInvalidAddrNoID) Error() string { + return fmt.Sprintf("Cannot add address with no ID %v", err.Addr) +} From 48ab899923c564bbf2fa2f1244c11cb930e28132 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Fri, 9 Nov 2018 19:31:59 -0800 Subject: [PATCH 07/55] Do not shut down peer immediately after sending pex addrs in SeedMode --- CHANGELOG.md | 4 ++-- p2p/pex/pex_reactor.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 792386e5cd3..fd36e385383 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,8 +139,8 @@ increasing attention to backwards compatibility. Thanks for bearing with us! - [node] [\#2434](https://github.com/tendermint/tendermint/issues/2434) Make node respond to signal interrupts while sleeping for genesis time - [state] [\#2616](https://github.com/tendermint/tendermint/issues/2616) Pass nil to NewValidatorSet() when genesis file's Validators field is nil - [p2p] [\#2555](https://github.com/tendermint/tendermint/issues/2555) Fix p2p switch FlushThrottle value (@goolAdapter) -- [p2p] [\#2668](https://github.com/tendermint/tendermint/issues/2668) Reconnect to originally dialed address (not self-reported - address) for persistent peers +- [p2p] [\#2668](https://github.com/tendermint/tendermint/issues/2668) Reconnect to originally dialed address (not self-reported address) for persistent peers +- [p2p] [\#2797](https://github.com/tendermint/tendermint/pull/2797) AddressBook requires addresses to have IDs; Do not crap out immediately after sending pex addrs in seed mode ## v0.25.0 diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 46a12c48813..4e8524fd941 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -221,6 +221,7 @@ func (r *PEXReactor) Receive(chID byte, src Peer, msgBytes []byte) { // 2) limit the output size if r.config.SeedMode { r.SendAddrs(src, r.book.GetSelectionWithBias(biasToSelectNewPeers)) + time.Sleep(time.Second * 3) // TODO Rethink this. Without it, above may not actually send. r.Switch.StopPeerGracefully(src) } else { r.SendAddrs(src, r.book.GetSelection()) From 67042bee57df654425600ea121d6d8bf2aa24438 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Tue, 13 Nov 2018 20:18:17 +0800 Subject: [PATCH 08/55] Add reteat last block interface for block store --- blockchain/store.go | 17 +++++++++++++++++ consensus/replay_test.go | 2 ++ state/services.go | 1 + 3 files changed, 20 insertions(+) diff --git a/blockchain/store.go b/blockchain/store.go index 498cca68dba..3f3471fc455 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -186,6 +186,23 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s bs.db.SetSync(nil, nil) } +func (bs *BlockStore) RetreatLastBlock() { + height := bs.height + + bs.db.Delete(calcBlockMetaKey(height)) + bs.db.Delete(calcBlockCommitKey(height-1)) + bs.db.Delete(calcSeenCommitKey(height)) + BlockStoreStateJSON{Height: height-1 }.Save(bs.db) + + // Done! + bs.mtx.Lock() + bs.height = height + bs.mtx.Unlock() + + // Flush + bs.db.SetSync(nil, nil) +} + func (bs *BlockStore) saveBlockPart(height int64, index int, part *types.Part) { if height != bs.Height()+1 { cmn.PanicSanity(fmt.Sprintf("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height)) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 70c4ba332fb..4d47ced0984 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -624,6 +624,8 @@ func (bs *mockBlockStore) LoadBlockMeta(height int64) *types.BlockMeta { func (bs *mockBlockStore) LoadBlockPart(height int64, index int) *types.Part { return nil } func (bs *mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) { } +func (bs *mockBlockStore) RetreatLastBlock() { +} func (bs *mockBlockStore) LoadBlockCommit(height int64) *types.Commit { return bs.commits[height-1] } diff --git a/state/services.go b/state/services.go index b8f1febe1fb..e494124f9dd 100644 --- a/state/services.go +++ b/state/services.go @@ -74,6 +74,7 @@ type BlockStoreRPC interface { type BlockStore interface { BlockStoreRPC SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) + RetreatLastBlock() } //----------------------------------------------------------------------------------------------------- From 7ba4f02043eb5d7b68513edbaad84455a00b4fb5 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Wed, 14 Nov 2018 10:43:47 +0800 Subject: [PATCH 09/55] Add tag for future merge --- blockchain/store.go | 2 ++ consensus/replay_test.go | 2 ++ state/services.go | 2 ++ 3 files changed, 6 insertions(+) diff --git a/blockchain/store.go b/blockchain/store.go index 3f3471fc455..79fe3bd85ce 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -186,6 +186,7 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s bs.db.SetSync(nil, nil) } +//////////////////// iris/tendermint begin /////////////////////////// func (bs *BlockStore) RetreatLastBlock() { height := bs.height @@ -202,6 +203,7 @@ func (bs *BlockStore) RetreatLastBlock() { // Flush bs.db.SetSync(nil, nil) } +//////////////////// iris/tendermint end /////////////////////////// func (bs *BlockStore) saveBlockPart(height int64, index int, part *types.Part) { if height != bs.Height()+1 { diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 4d47ced0984..4d7558a0a78 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -624,8 +624,10 @@ func (bs *mockBlockStore) LoadBlockMeta(height int64) *types.BlockMeta { func (bs *mockBlockStore) LoadBlockPart(height int64, index int) *types.Part { return nil } func (bs *mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) { } +//////////////////// iris/tendermint begin /////////////////////////// func (bs *mockBlockStore) RetreatLastBlock() { } +//////////////////// iris/tendermint end /////////////////////////// func (bs *mockBlockStore) LoadBlockCommit(height int64) *types.Commit { return bs.commits[height-1] } diff --git a/state/services.go b/state/services.go index e494124f9dd..f8187f3dcbc 100644 --- a/state/services.go +++ b/state/services.go @@ -74,7 +74,9 @@ type BlockStoreRPC interface { type BlockStore interface { BlockStoreRPC SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) + //////////////////// iris/tendermint begin /////////////////////////// RetreatLastBlock() + //////////////////// iris/tendermint begin /////////////////////////// } //----------------------------------------------------------------------------------------------------- From d72aea0630be39610e4ea0ecd0abb942ab87e076 Mon Sep 17 00:00:00 2001 From: jiacheng Date: Thu, 22 Nov 2018 14:38:36 +0800 Subject: [PATCH 10/55] update --- blockchain/store.go | 16 ++++++++++++++++ consensus/replay_test.go | 4 ++++ state/execution.go | 9 +++++++++ state/services.go | 5 +++++ state/state.go | 3 +++ state/store.go | 17 ++++++++++++++++- 6 files changed, 53 insertions(+), 1 deletion(-) diff --git a/blockchain/store.go b/blockchain/store.go index 498cca68dba..cd5b64c5f71 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -186,6 +186,22 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s bs.db.SetSync(nil, nil) } +//////////////////// iris/tendermint begin /////////////////////////// +func (bs *BlockStore) RetreatLastBlock() { + height := bs.height + bs.db.Delete(calcBlockMetaKey(height)) + bs.db.Delete(calcBlockCommitKey(height-1)) + bs.db.Delete(calcSeenCommitKey(height)) + BlockStoreStateJSON{Height: height-1 }.Save(bs.db) + // Done! + bs.mtx.Lock() + bs.height = height + bs.mtx.Unlock() + // Flush + bs.db.SetSync(nil, nil) +} +//////////////////// iris/tendermint end /////////////////////////// + func (bs *BlockStore) saveBlockPart(height int64, index int, part *types.Part) { if height != bs.Height()+1 { cmn.PanicSanity(fmt.Sprintf("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height)) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 70c4ba332fb..4d7558a0a78 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -624,6 +624,10 @@ func (bs *mockBlockStore) LoadBlockMeta(height int64) *types.BlockMeta { func (bs *mockBlockStore) LoadBlockPart(height int64, index int) *types.Part { return nil } func (bs *mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) { } +//////////////////// iris/tendermint begin /////////////////////////// +func (bs *mockBlockStore) RetreatLastBlock() { +} +//////////////////// iris/tendermint end /////////////////////////// func (bs *mockBlockStore) LoadBlockCommit(height int64) *types.Commit { return bs.commits[height-1] } diff --git a/state/execution.go b/state/execution.go index cc8e7e75fd5..e3a42fb6606 100644 --- a/state/execution.go +++ b/state/execution.go @@ -102,6 +102,11 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b fail.Fail() // XXX + // update the state with the block and responses + //////////////////// iris/tendermint begin /////////////////////////// + preState := state.Copy() + //////////////////// iris/tendermint end /////////////////////////// + // Save the results before we commit. saveABCIResponses(blockExec.db, block.Height, abciResponses) @@ -128,6 +133,10 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b state.AppHash = appHash SaveState(blockExec.db, state) + //////////////////// iris/tendermint begin /////////////////////////// + SavePreState(blockExec.db, preState) + //////////////////// iris/tendermint end /////////////////////////// + fail.Fail() // XXX // Events are fired after everything else. diff --git a/state/services.go b/state/services.go index b8f1febe1fb..3745841fb5d 100644 --- a/state/services.go +++ b/state/services.go @@ -74,6 +74,11 @@ type BlockStoreRPC interface { type BlockStore interface { BlockStoreRPC SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) + + //////////////////// iris/tendermint begin /////////////////////////// + RetreatLastBlock() + //////////////////// iris/tendermint begin /////////////////////////// + } //----------------------------------------------------------------------------------------------------- diff --git a/state/state.go b/state/state.go index 0dbd718dab3..b459248c6ea 100644 --- a/state/state.go +++ b/state/state.go @@ -14,6 +14,9 @@ import ( // database keys var ( stateKey = []byte("stateKey") + //////////////////// iris/tendermint begin /////////////////////////// + statePreKey = []byte("statePreKey") + //////////////////// iris/tendermint end /////////////////////////// ) //----------------------------------------------------------------------------- diff --git a/state/store.go b/state/store.go index 7a0ef255a7a..c4e09df7428 100644 --- a/state/store.go +++ b/state/store.go @@ -62,6 +62,12 @@ func LoadState(db dbm.DB) State { return loadState(db, stateKey) } +//////////////////// iris/tendermint begin /////////////////////////// +func LoadPreState(db dbm.DB) State { + return loadState(db, statePreKey) +} +//////////////////// iris/tendermint end /////////////////////////// + func loadState(db dbm.DB, key []byte) (state State) { buf := db.Get(key) if len(buf) == 0 { @@ -85,6 +91,12 @@ func SaveState(db dbm.DB, state State) { saveState(db, state, stateKey) } +//////////////////// iris/tendermint begin /////////////////////////// +func SavePreState(db dbm.DB, state State) { + saveState(db, state, statePreKey) +} +//////////////////// iris/tendermint end /////////////////////////// + func saveState(db dbm.DB, state State, key []byte) { nextHeight := state.LastBlockHeight + 1 // If first block, save validators for block 1. @@ -96,7 +108,10 @@ func saveState(db dbm.DB, state State, key []byte) { saveValidatorsInfo(db, nextHeight+1, state.LastHeightValidatorsChanged, state.NextValidators) // Save next consensus params. saveConsensusParamsInfo(db, nextHeight, state.LastHeightConsensusParamsChanged, state.ConsensusParams) - db.SetSync(stateKey, state.Bytes()) + + //////////////////// iris/tendermint begin /////////////////////////// + db.SetSync(key, state.Bytes()) + //////////////////// iris/tendermint end /////////////////////////// } //------------------------------------------------------------------------ From 8c7f910722897fcccb59f60da01ddc27ad1f3920 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Thu, 22 Nov 2018 14:32:33 +0800 Subject: [PATCH 11/55] Enable terminate command from abci and stop p2p gossip when deprecated is true --- consensus/state.go | 4 +++- node/node.go | 4 ++++ state/execution.go | 16 +++++++++++++++- state/state.go | 3 +++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 8c2e292c8e6..bfc93a27a00 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -123,6 +123,8 @@ type ConsensusState struct { // for reporting metrics metrics *Metrics + + Deprecated bool } // StateOption sets an optional parameter on the ConsensusState. @@ -158,7 +160,7 @@ func NewConsensusState( cs.decideProposal = cs.defaultDecideProposal cs.doPrevote = cs.defaultDoPrevote cs.setProposal = cs.defaultSetProposal - + cs.Deprecated = state.Deprecated cs.updateToState(state) // Don't call scheduleRound0 yet. diff --git a/node/node.go b/node/node.go index 4710397faff..fc73eaa27e2 100644 --- a/node/node.go +++ b/node/node.go @@ -545,6 +545,10 @@ func (n *Node) OnStart() error { n.prometheusSrv = n.startPrometheusServer(n.config.Instrumentation.PrometheusListenAddr) } + if n.consensusState.Deprecated { + return nil + } + // Start the transport. addr, err := p2p.NewNetAddressStringWithOptionalID(n.config.P2P.ListenAddress) if err != nil { diff --git a/state/execution.go b/state/execution.go index cc39bfbd0bf..654a03e51b4 100644 --- a/state/execution.go +++ b/state/execution.go @@ -1,6 +1,7 @@ package state import ( + "bytes" "fmt" "time" @@ -13,6 +14,10 @@ import ( "github.com/tendermint/tendermint/types" ) +const ( + TerminateTagKey = "terminate_blockchain" + TerminateTagValue = "true" +) //----------------------------------------------------------------------------- // BlockExecutor handles block execution and state updates. // It exposes ApplyBlock(), which validates & executes the block, updates state w/ ABCI responses, @@ -115,6 +120,13 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b if err != nil { return state, fmt.Errorf("Commit failed for application: %v", err) } + length := len(abciResponses.EndBlock.Tags) + if length > 0 { + tag := abciResponses.EndBlock.Tags[length-1] + if bytes.Equal(tag.Key, []byte(TerminateTagKey)) && bytes.Equal(tag.Value, []byte(TerminateTagValue)) { + state.Deprecated = true + } + } // Lock mempool, commit app state, update mempoool. appHash, err := blockExec.Commit(state, block) @@ -139,7 +151,9 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b // Events are fired after everything else. // NOTE: if we crash between Commit and Save, events wont be fired during replay fireEvents(blockExec.logger, blockExec.eventBus, block, abciResponses) - + if state.Deprecated { + return state, fmt.Errorf("program received termination signal from ABCI app, restart your node and it will enter query mode, which means p2p gossip and consensus engine will be shutdown") + } return state, nil } diff --git a/state/state.go b/state/state.go index b459248c6ea..ebeb420e3e7 100644 --- a/state/state.go +++ b/state/state.go @@ -83,6 +83,8 @@ type State struct { // the latest AppHash we've received from calling abci.Commit() AppHash []byte + + Deprecated bool } // Copy makes a copy of the State for mutating. @@ -107,6 +109,7 @@ func (state State) Copy() State { AppHash: state.AppHash, LastResultsHash: state.LastResultsHash, + Deprecated: state.Deprecated, } } From bb400bc72e91d792c5fa9bf195123f3e7aa1e66a Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Thu, 22 Nov 2018 18:31:04 +0800 Subject: [PATCH 12/55] Add log in query mode --- node/node.go | 1 + state/execution.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/node/node.go b/node/node.go index fc73eaa27e2..0f5cae706c6 100644 --- a/node/node.go +++ b/node/node.go @@ -546,6 +546,7 @@ func (n *Node) OnStart() error { } if n.consensusState.Deprecated { + n.Logger.Info("This blockchain has been terminated. The consensus engine and p2p gossip have been disabled. Only query rpc interfaces are available") return nil } diff --git a/state/execution.go b/state/execution.go index 654a03e51b4..555f90385a6 100644 --- a/state/execution.go +++ b/state/execution.go @@ -152,7 +152,7 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b // NOTE: if we crash between Commit and Save, events wont be fired during replay fireEvents(blockExec.logger, blockExec.eventBus, block, abciResponses) if state.Deprecated { - return state, fmt.Errorf("program received termination signal from ABCI app, restart your node and it will enter query mode, which means p2p gossip and consensus engine will be shutdown") + return state, fmt.Errorf("program received termination signal from ABCI app, restart your node and it will enter query-only mode") } return state, nil } From cfd38fa1a29fb41d7b7cfd472183978c9cc32105 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Fri, 23 Nov 2018 10:42:31 +0800 Subject: [PATCH 13/55] Add deprecated flag --- cmd/tendermint/commands/run_node.go | 1 + config/config.go | 6 ++++++ config/toml.go | 5 +++++ node/node.go | 2 +- 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index 6dabacb1f0a..f36dfbcc2d3 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -22,6 +22,7 @@ func AddNodeFlags(cmd *cobra.Command) { // node flags cmd.Flags().Bool("fast_sync", config.FastSync, "Fast blockchain syncing") + cmd.Flags().Bool("deprecated", config.Deprecated, "Mark blockchain as deprecated") // abci flags cmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'kvstore' for local testing.") diff --git a/config/config.go b/config/config.go index fa6182114ad..5b303eb4f2a 100644 --- a/config/config.go +++ b/config/config.go @@ -136,6 +136,11 @@ type BaseConfig struct { // and verifying their commits FastSync bool `mapstructure:"fast_sync"` + // If the blockchain is deprecated, run node with Deprecated will + // work in query only mode. Consensus engine and p2p gossip will be + // shutdown + Deprecated bool `mapstructure:"deprecated"` + // Database backend: leveldb | memdb | cleveldb DBBackend string `mapstructure:"db_backend"` @@ -181,6 +186,7 @@ func DefaultBaseConfig() BaseConfig { LogLevel: DefaultPackageLogLevels(), ProfListenAddress: "", FastSync: true, + Deprecated: false, FilterPeers: false, DBBackend: "leveldb", DBPath: "data", diff --git a/config/toml.go b/config/toml.go index d73b9c81de2..4d94aafad0b 100644 --- a/config/toml.go +++ b/config/toml.go @@ -77,6 +77,11 @@ moniker = "{{ .BaseConfig.Moniker }}" # and verifying their commits fast_sync = {{ .BaseConfig.FastSync }} +# If the blockchain is deprecated, run node with Deprecated will +# work in query only mode. Consensus engine and p2p gossip will be +# shutdown +deprecated = {{ .BaseConfig.Deprecated }} + # Database backend: leveldb | memdb | cleveldb db_backend = "{{ .BaseConfig.DBBackend }}" diff --git a/node/node.go b/node/node.go index 0f5cae706c6..55f8329b2aa 100644 --- a/node/node.go +++ b/node/node.go @@ -545,7 +545,7 @@ func (n *Node) OnStart() error { n.prometheusSrv = n.startPrometheusServer(n.config.Instrumentation.PrometheusListenAddr) } - if n.consensusState.Deprecated { + if n.consensusState.Deprecated || n.config.Deprecated { n.Logger.Info("This blockchain has been terminated. The consensus engine and p2p gossip have been disabled. Only query rpc interfaces are available") return nil } From 67b95867601e27dddb3a7b51fa66760aa00c9fcc Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Fri, 23 Nov 2018 19:43:23 +0800 Subject: [PATCH 14/55] Add interval before exit --- consensus/state.go | 7 +++++++ state/execution.go | 3 --- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index bfc93a27a00..f7d8143f3fd 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -27,6 +27,7 @@ import ( const ( proposalHeartbeatIntervalSeconds = 2 + deprecatedToShutdownInterval = 30 ) //----------------------------------------------------------------------------- @@ -546,6 +547,7 @@ func (cs *ConsensusState) updateToState(state sm.State) { cs.LastValidators = state.LastValidators cs.state = state + cs.Deprecated = state.Deprecated // Finally, broadcast RoundState cs.newStep() @@ -599,6 +601,11 @@ func (cs *ConsensusState) receiveRoutine(maxSteps int) { }() for { + if cs.Deprecated { + cs.Logger.Info(fmt.Sprintf("this blockchain has been deprecated. %d seconds later, this node will be shutdown", deprecatedToShutdownInterval)) + time.Sleep(deprecatedToShutdownInterval * time.Second) + cmn.Exit("Shutdown this blockchain node") + } if maxSteps > 0 { if cs.nSteps >= maxSteps { cs.Logger.Info("reached max steps. exiting receive routine") diff --git a/state/execution.go b/state/execution.go index 555f90385a6..45826823630 100644 --- a/state/execution.go +++ b/state/execution.go @@ -151,9 +151,6 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b // Events are fired after everything else. // NOTE: if we crash between Commit and Save, events wont be fired during replay fireEvents(blockExec.logger, blockExec.eventBus, block, abciResponses) - if state.Deprecated { - return state, fmt.Errorf("program received termination signal from ABCI app, restart your node and it will enter query-only mode") - } return state, nil } From 6a0515f51cb76b4b34363f8a0ec9a464ae982737 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Mon, 26 Nov 2018 09:56:31 +0800 Subject: [PATCH 15/55] change terminator tag to the first tag --- consensus/state.go | 1 + state/execution.go | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index f7d8143f3fd..ac5969db49b 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -161,6 +161,7 @@ func NewConsensusState( cs.decideProposal = cs.defaultDecideProposal cs.doPrevote = cs.defaultDoPrevote cs.setProposal = cs.defaultSetProposal + cs.Deprecated = state.Deprecated cs.updateToState(state) diff --git a/state/execution.go b/state/execution.go index 618369959a2..343fbba2f34 100644 --- a/state/execution.go +++ b/state/execution.go @@ -123,9 +123,8 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b return state, fmt.Errorf("Commit failed for application: %v", err) } - length := len(abciResponses.EndBlock.Tags) - if length > 0 { - tag := abciResponses.EndBlock.Tags[length-1] + if len(abciResponses.EndBlock.Tags) > 0 { + tag := abciResponses.EndBlock.Tags[0] if bytes.Equal(tag.Key, []byte(TerminateTagKey)) && bytes.Equal(tag.Value, []byte(TerminateTagValue)) { state.Deprecated = true } From 6f2d6ced48af493f5a0544b3236e1ab62d145091 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Thu, 29 Nov 2018 14:49:16 +0800 Subject: [PATCH 16/55] Add replay height --- cmd/tendermint/commands/run_node.go | 3 +++ config/config.go | 3 +++ consensus/replay.go | 16 +++++++++++----- consensus/replay_file.go | 2 +- node/node.go | 2 +- 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index f36dfbcc2d3..e324025cf03 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -24,6 +24,9 @@ func AddNodeFlags(cmd *cobra.Command) { cmd.Flags().Bool("fast_sync", config.FastSync, "Fast blockchain syncing") cmd.Flags().Bool("deprecated", config.Deprecated, "Mark blockchain as deprecated") + cmd.Flags().Bool("deprecated", config.Deprecated, "Mark blockchain as deprecated") + cmd.Flags().Bool("replay_height", config.Deprecated, "Mark blockchain as deprecated") + // abci flags cmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'kvstore' for local testing.") cmd.Flags().String("abci", config.ABCI, "Specify abci transport (socket | grpc)") diff --git a/config/config.go b/config/config.go index 5b303eb4f2a..a8bbbec221a 100644 --- a/config/config.go +++ b/config/config.go @@ -141,6 +141,8 @@ type BaseConfig struct { // shutdown Deprecated bool `mapstructure:"deprecated"` + ReplayHeight int64 `mapstructure:"replay_height"` + // Database backend: leveldb | memdb | cleveldb DBBackend string `mapstructure:"db_backend"` @@ -187,6 +189,7 @@ func DefaultBaseConfig() BaseConfig { ProfListenAddress: "", FastSync: true, Deprecated: false, + ReplayHeight: -1, FilterPeers: false, DBBackend: "leveldb", DBPath: "data", diff --git a/consensus/replay.go b/consensus/replay.go index abc43eb578d..f3fc16b8a80 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -11,6 +11,7 @@ import ( "time" abci "github.com/tendermint/tendermint/abci/types" + cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/version" //auto "github.com/tendermint/tendermint/libs/autofile" cmn "github.com/tendermint/tendermint/libs/common" @@ -224,7 +225,7 @@ func (h *Handshaker) NBlocks() int { } // TODO: retry the handshake/replay if it fails ? -func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { +func (h *Handshaker) Handshake(proxyApp proxy.AppConns, config *cfg.BaseConfig) error { // Handshake is done via ABCI Info on the query conn. res, err := proxyApp.Query().InfoSync(proxy.RequestInfo) @@ -249,7 +250,7 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { h.initialState.Version.Consensus.App = version.Protocol(res.AppVersion) // Replay blocks up to the latest in the blockstore. - _, err = h.ReplayBlocks(h.initialState, appHash, blockHeight, proxyApp) + _, err = h.ReplayBlocks(h.initialState, appHash, blockHeight, proxyApp, config) if err != nil { return fmt.Errorf("Error on replay: %v", err) } @@ -269,6 +270,7 @@ func (h *Handshaker) ReplayBlocks( appHash []byte, appBlockHeight int64, proxyApp proxy.AppConns, + config *cfg.BaseConfig, ) ([]byte, error) { storeBlockHeight := h.store.Height() stateBlockHeight := state.LastBlockHeight @@ -330,7 +332,7 @@ func (h *Handshaker) ReplayBlocks( // Either the app is asking for replay, or we're all synced up. if appBlockHeight < storeBlockHeight { // the app is behind, so replay blocks, but no need to go through WAL (state is already synced to store) - return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, false) + return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, false, config) } else if appBlockHeight == storeBlockHeight { // We're good! @@ -343,7 +345,7 @@ func (h *Handshaker) ReplayBlocks( if appBlockHeight < stateBlockHeight { // the app is further behind than it should be, so replay blocks // but leave the last block to go through the WAL - return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, true) + return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, true, config) } else if appBlockHeight == stateBlockHeight { // We haven't run Commit (both the state and app are one block behind), @@ -372,7 +374,7 @@ func (h *Handshaker) ReplayBlocks( return nil, nil } -func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int64, mutateState bool) ([]byte, error) { +func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int64, mutateState bool, config *cfg.BaseConfig) ([]byte, error) { // App is further behind than it should be, so we need to replay blocks. // We replay all blocks from appBlockHeight+1. // @@ -397,6 +399,10 @@ func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBl return nil, err } + if config.ReplayHeight > 0 && i > config.ReplayHeight { + cmn.Exit(fmt.Sprintf("Successfully replay to height %s", config.ReplayHeight)) + } + h.nBlocks++ } diff --git a/consensus/replay_file.go b/consensus/replay_file.go index 685eb71f2c3..a36eddbbc1b 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -305,7 +305,7 @@ func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusCo } handshaker := NewHandshaker(stateDB, state, blockStore, gdoc) - err = handshaker.Handshake(proxyApp) + err = handshaker.Handshake(proxyApp, &config) if err != nil { cmn.Exit(fmt.Sprintf("Error on handshake: %v", err)) } diff --git a/node/node.go b/node/node.go index 55f8329b2aa..488bc36ece3 100644 --- a/node/node.go +++ b/node/node.go @@ -200,7 +200,7 @@ func NewNode(config *cfg.Config, consensusLogger := logger.With("module", "consensus") handshaker := cs.NewHandshaker(stateDB, state, blockStore, genDoc) handshaker.SetLogger(consensusLogger) - if err := handshaker.Handshake(proxyApp); err != nil { + if err := handshaker.Handshake(proxyApp, &config.BaseConfig); err != nil { return nil, fmt.Errorf("Error during handshake: %v", err) } From 1754662b6dce3b910faa3b4743d7b766ef9cb296 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Thu, 29 Nov 2018 14:54:56 +0800 Subject: [PATCH 17/55] Fix replay_height flag error --- cmd/tendermint/commands/run_node.go | 4 +--- consensus/replay.go | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index e324025cf03..d6ae83b8dcc 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -23,9 +23,7 @@ func AddNodeFlags(cmd *cobra.Command) { // node flags cmd.Flags().Bool("fast_sync", config.FastSync, "Fast blockchain syncing") cmd.Flags().Bool("deprecated", config.Deprecated, "Mark blockchain as deprecated") - - cmd.Flags().Bool("deprecated", config.Deprecated, "Mark blockchain as deprecated") - cmd.Flags().Bool("replay_height", config.Deprecated, "Mark blockchain as deprecated") + cmd.Flags().Int64("replay_height", config.ReplayHeight, "Specify replay height") // abci flags cmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'kvstore' for local testing.") diff --git a/consensus/replay.go b/consensus/replay.go index f3fc16b8a80..a61848c1a5f 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -400,7 +400,7 @@ func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBl } if config.ReplayHeight > 0 && i > config.ReplayHeight { - cmn.Exit(fmt.Sprintf("Successfully replay to height %s", config.ReplayHeight)) + cmn.Exit(fmt.Sprintf("Replay from height %d to height %d successfully", appBlockHeight, config.ReplayHeight)) } h.nBlocks++ From d19442edc59e9f1ee940fac283d2780a4e501806 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Wed, 5 Dec 2018 11:09:21 +0800 Subject: [PATCH 18/55] Rename termanate to halt --- state/execution.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/state/execution.go b/state/execution.go index b2edcfc4616..7c5fbfd759e 100644 --- a/state/execution.go +++ b/state/execution.go @@ -15,8 +15,8 @@ import ( ) const ( - TerminateTagKey = "terminate_blockchain" - TerminateTagValue = "true" + HaltTagKey = "halt_blockchain" + HaltTagValue = "true" ) //----------------------------------------------------------------------------- // BlockExecutor handles block execution and state updates. @@ -139,7 +139,7 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b if len(abciResponses.EndBlock.Tags) > 0 { tag := abciResponses.EndBlock.Tags[0] - if bytes.Equal(tag.Key, []byte(TerminateTagKey)) && bytes.Equal(tag.Value, []byte(TerminateTagValue)) { + if bytes.Equal(tag.Key, []byte(HaltTagKey)) && bytes.Equal(tag.Value, []byte(HaltTagValue)) { state.Deprecated = true } } From 789e9a187238795484bebecff1ceb500161706cc Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Tue, 11 Dec 2018 13:47:00 +0800 Subject: [PATCH 19/55] Improve log --- node/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/node.go b/node/node.go index bd866055d8c..019d8e3c1fb 100644 --- a/node/node.go +++ b/node/node.go @@ -533,7 +533,7 @@ func (n *Node) OnStart() error { } if n.consensusState.Deprecated || n.config.Deprecated { - n.Logger.Info("This blockchain has been terminated. The consensus engine and p2p gossip have been disabled. Only query rpc interfaces are available") + n.Logger.Info("This blockchain was halt. The consensus engine and p2p gossip have been disabled. Only query rpc interfaces are available") return nil } From 8c40185f8b1bffbdbbd401dd159284d402fcaaf6 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Tue, 11 Dec 2018 13:52:45 +0800 Subject: [PATCH 20/55] Remmove useless change --- blockchain/store.go | 16 ---------------- consensus/replay_test.go | 4 ---- state/execution.go | 14 +++----------- state/services.go | 5 ----- state/state.go | 3 --- state/store.go | 12 ------------ 6 files changed, 3 insertions(+), 51 deletions(-) diff --git a/blockchain/store.go b/blockchain/store.go index cd5b64c5f71..498cca68dba 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -186,22 +186,6 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s bs.db.SetSync(nil, nil) } -//////////////////// iris/tendermint begin /////////////////////////// -func (bs *BlockStore) RetreatLastBlock() { - height := bs.height - bs.db.Delete(calcBlockMetaKey(height)) - bs.db.Delete(calcBlockCommitKey(height-1)) - bs.db.Delete(calcSeenCommitKey(height)) - BlockStoreStateJSON{Height: height-1 }.Save(bs.db) - // Done! - bs.mtx.Lock() - bs.height = height - bs.mtx.Unlock() - // Flush - bs.db.SetSync(nil, nil) -} -//////////////////// iris/tendermint end /////////////////////////// - func (bs *BlockStore) saveBlockPart(height int64, index int, part *types.Part) { if height != bs.Height()+1 { cmn.PanicSanity(fmt.Sprintf("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height)) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 6b34fa11815..7cd32c7aa2f 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -617,10 +617,6 @@ func (bs *mockBlockStore) LoadBlockMeta(height int64) *types.BlockMeta { func (bs *mockBlockStore) LoadBlockPart(height int64, index int) *types.Part { return nil } func (bs *mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) { } -//////////////////// iris/tendermint begin /////////////////////////// -func (bs *mockBlockStore) RetreatLastBlock() { -} -//////////////////// iris/tendermint end /////////////////////////// func (bs *mockBlockStore) LoadBlockCommit(height int64) *types.Commit { return bs.commits[height-1] } diff --git a/state/execution.go b/state/execution.go index 7c5fbfd759e..57282e262e2 100644 --- a/state/execution.go +++ b/state/execution.go @@ -15,9 +15,10 @@ import ( ) const ( - HaltTagKey = "halt_blockchain" - HaltTagValue = "true" + HaltTagKey = "halt_blockchain" + HaltTagValue = "true" ) + //----------------------------------------------------------------------------- // BlockExecutor handles block execution and state updates. // It exposes ApplyBlock(), which validates & executes the block, updates state w/ ABCI responses, @@ -107,11 +108,6 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b fail.Fail() // XXX - // update the state with the block and responses - //////////////////// iris/tendermint begin /////////////////////////// - preState := state.Copy() - //////////////////// iris/tendermint end /////////////////////////// - // Save the results before we commit. saveABCIResponses(blockExec.db, block.Height, abciResponses) @@ -159,10 +155,6 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b state.AppHash = appHash SaveState(blockExec.db, state) - //////////////////// iris/tendermint begin /////////////////////////// - SavePreState(blockExec.db, preState) - //////////////////// iris/tendermint end /////////////////////////// - fail.Fail() // XXX // Events are fired after everything else. diff --git a/state/services.go b/state/services.go index 3745841fb5d..b8f1febe1fb 100644 --- a/state/services.go +++ b/state/services.go @@ -74,11 +74,6 @@ type BlockStoreRPC interface { type BlockStore interface { BlockStoreRPC SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) - - //////////////////// iris/tendermint begin /////////////////////////// - RetreatLastBlock() - //////////////////// iris/tendermint begin /////////////////////////// - } //----------------------------------------------------------------------------------------------------- diff --git a/state/state.go b/state/state.go index 3735cada951..2df6516a723 100644 --- a/state/state.go +++ b/state/state.go @@ -14,9 +14,6 @@ import ( // database keys var ( stateKey = []byte("stateKey") - //////////////////// iris/tendermint begin /////////////////////////// - statePreKey = []byte("statePreKey") - //////////////////// iris/tendermint end /////////////////////////// ) //----------------------------------------------------------------------------- diff --git a/state/store.go b/state/store.go index d0c504d567e..6b01a829575 100644 --- a/state/store.go +++ b/state/store.go @@ -62,12 +62,6 @@ func LoadState(db dbm.DB) State { return loadState(db, stateKey) } -//////////////////// iris/tendermint begin /////////////////////////// -func LoadPreState(db dbm.DB) State { - return loadState(db, statePreKey) -} -//////////////////// iris/tendermint end /////////////////////////// - func loadState(db dbm.DB, key []byte) (state State) { buf := db.Get(key) if len(buf) == 0 { @@ -91,12 +85,6 @@ func SaveState(db dbm.DB, state State) { saveState(db, state, stateKey) } -//////////////////// iris/tendermint begin /////////////////////////// -func SavePreState(db dbm.DB, state State) { - saveState(db, state, statePreKey) -} -//////////////////// iris/tendermint end /////////////////////////// - func saveState(db dbm.DB, state State, key []byte) { nextHeight := state.LastBlockHeight + 1 // If first block, save validators for block 1. From 79f7453510a5c292f85e0a4c867960a820c7238f Mon Sep 17 00:00:00 2001 From: jiacheng Date: Wed, 12 Dec 2018 15:55:58 +0800 Subject: [PATCH 21/55] modify for class2 upgrade --- consensus/replay.go | 6 ++++++ state/execution.go | 8 ++++++++ state/validation.go | 16 ++++++++++++---- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index 501e35f791a..2391a83bb20 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -249,6 +249,12 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns, config *cfg.BaseConfig) // Set AppVersion on the state. h.initialState.Version.Consensus.App = version.Protocol(res.AppVersion) + //////////////////// iris/tendermint begin /////////////////////////// + state := sm.LoadState(h.stateDB) + state.Version.Consensus.App = version.Protocol(res.AppVersion) + sm.SaveState(h.stateDB, state) + //////////////////// iris/tendermint end /////////////////////////// + // Replay blocks up to the latest in the blockstore. _, err = h.ReplayBlocks(h.initialState, appHash, blockHeight, proxyApp, config) if err != nil { diff --git a/state/execution.go b/state/execution.go index 57282e262e2..a2534574a52 100644 --- a/state/execution.go +++ b/state/execution.go @@ -17,6 +17,8 @@ import ( const ( HaltTagKey = "halt_blockchain" HaltTagValue = "true" + UpgradeTagKey = "upgrade_blockchain" + ) //----------------------------------------------------------------------------- @@ -281,6 +283,12 @@ func execBlockOnProxyApp( return nil, err } + if len(abciResponses.EndBlock.Tags) > 0 { + tag := abciResponses.EndBlock.Tags[len(abciResponses.EndBlock.Tags)-1] + if bytes.Equal(tag.Key, []byte(UpgradeTagKey)) { + return nil, fmt.Errorf(string(tag.Value)) + } + } logger.Info("Executed block", "height", block.Height, "validTxs", validTxs, "invalidTxs", invalidTxs) return abciResponses, nil diff --git a/state/validation.go b/state/validation.go index e28d40e8bd9..2bd86c3360d 100644 --- a/state/validation.go +++ b/state/validation.go @@ -20,12 +20,20 @@ func validateBlock(stateDB dbm.DB, state State, block *types.Block) error { } // Validate basic info. - if block.Version != state.Version.Consensus { - return fmt.Errorf("Wrong Block.Header.Version. Expected %v, got %v", - state.Version.Consensus, - block.Version, + //if block.Version != state.Version.Consensus { + // return fmt.Errorf("Wrong Block.Header.Version. Expected %v, got %v", + // state.Version.Consensus, + // block.Version, + // ) + //} + + if block.Version.Block != state.Version.Consensus.Block { + return fmt.Errorf("Wrong Block.Header.Version.Block Expected %v, got %v", + state.Version.Consensus.Block, + block.Version.Block, ) } + if block.ChainID != state.ChainID { return fmt.Errorf("Wrong Block.Header.ChainID. Expected %v, got %v", state.ChainID, From c09e95e05f492126891afa407ddc3a4a20c7e215 Mon Sep 17 00:00:00 2001 From: jiacheng Date: Fri, 14 Dec 2018 16:18:59 +0800 Subject: [PATCH 22/55] fix the comment --- abci/types/util.go | 21 +++++++++++++++++++++ state/execution.go | 11 +++++------ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/abci/types/util.go b/abci/types/util.go index 3cde882320a..c2c518e817b 100644 --- a/abci/types/util.go +++ b/abci/types/util.go @@ -3,6 +3,7 @@ package types import ( "bytes" "sort" + common "github.com/tendermint/tendermint/libs/common" ) //------------------------------------------------------------------------------ @@ -32,3 +33,23 @@ func (v ValidatorUpdates) Swap(i, j int) { v[i] = v[j] v[j] = v1 } + +func GetTagByKey(tags []common.KVPair, key string ) (common.KVPair , bool) { + for _, tag := range tags { + if bytes.Equal(tag.Key, []byte(key)){ + return tag , true + } + } + return (common.KVPair)(nil) ,false +} + +func DeleteTagByKey(oldTags []common.KVPair, key string ) []common.KVPair { + var tags []common.KVPair + for _ , tag := range oldTags { + if bytes.Equal(tag.Key, []byte(key)){ + continue + } + tags = append(tags,tag) + } + return tags +} diff --git a/state/execution.go b/state/execution.go index a2534574a52..c0d37740dd5 100644 --- a/state/execution.go +++ b/state/execution.go @@ -17,7 +17,7 @@ import ( const ( HaltTagKey = "halt_blockchain" HaltTagValue = "true" - UpgradeTagKey = "upgrade_blockchain" + UpgradeFailureTagKey = "upgrade_failure" ) @@ -283,12 +283,11 @@ func execBlockOnProxyApp( return nil, err } - if len(abciResponses.EndBlock.Tags) > 0 { - tag := abciResponses.EndBlock.Tags[len(abciResponses.EndBlock.Tags)-1] - if bytes.Equal(tag.Key, []byte(UpgradeTagKey)) { - return nil, fmt.Errorf(string(tag.Value)) - } + + if tag, ok := abci.GetTagByKey(abciResponses.EndBlock.Tags, UpgradeFailureTagKey); ok{ + return nil, fmt.Errorf(string(tag.Value)) } + logger.Info("Executed block", "height", block.Height, "validTxs", validTxs, "invalidTxs", invalidTxs) return abciResponses, nil From 6472c02676ae89c466b908a1832108645554400d Mon Sep 17 00:00:00 2001 From: jiacheng Date: Mon, 17 Dec 2018 09:32:25 +0800 Subject: [PATCH 23/55] fix the comment --- abci/types/util.go | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/abci/types/util.go b/abci/types/util.go index c2c518e817b..8702ed284f8 100644 --- a/abci/types/util.go +++ b/abci/types/util.go @@ -2,8 +2,8 @@ package types import ( "bytes" - "sort" common "github.com/tendermint/tendermint/libs/common" + "sort" ) //------------------------------------------------------------------------------ @@ -34,22 +34,11 @@ func (v ValidatorUpdates) Swap(i, j int) { v[j] = v1 } -func GetTagByKey(tags []common.KVPair, key string ) (common.KVPair , bool) { +func GetTagByKey(tags []common.KVPair, key string) (common.KVPair, bool) { for _, tag := range tags { - if bytes.Equal(tag.Key, []byte(key)){ - return tag , true - } - } - return (common.KVPair)(nil) ,false -} - -func DeleteTagByKey(oldTags []common.KVPair, key string ) []common.KVPair { - var tags []common.KVPair - for _ , tag := range oldTags { - if bytes.Equal(tag.Key, []byte(key)){ - continue + if bytes.Equal(tag.Key, []byte(key)) { + return tag, true } - tags = append(tags,tag) } - return tags + return common.KVPair{}, false } From 3df7ea045debb0a1cd1ca894d584490db09e7ca2 Mon Sep 17 00:00:00 2001 From: chengwenxi Date: Fri, 21 Dec 2018 17:11:58 +0800 Subject: [PATCH 24/55] update HaltTagKey tag handler --- state/execution.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/state/execution.go b/state/execution.go index c0d37740dd5..f2c035cb706 100644 --- a/state/execution.go +++ b/state/execution.go @@ -135,11 +135,8 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b return state, fmt.Errorf("Commit failed for application: %v", err) } - if len(abciResponses.EndBlock.Tags) > 0 { - tag := abciResponses.EndBlock.Tags[0] - if bytes.Equal(tag.Key, []byte(HaltTagKey)) && bytes.Equal(tag.Value, []byte(HaltTagValue)) { - state.Deprecated = true - } + if tag, ok := abci.GetTagByKey(abciResponses.EndBlock.Tags, HaltTagKey); ok && bytes.Equal(tag.Value, []byte(HaltTagValue)) { + state.Deprecated = true } // Lock mempool, commit app state, update mempoool. From 2892bedf73b53d95fe69ea6b7ddc53d9712c0563 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Mon, 24 Dec 2018 17:52:55 +0800 Subject: [PATCH 25/55] Improve replay_height description --- cmd/tendermint/commands/run_node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index d6ae83b8dcc..ed898580e2a 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -23,7 +23,7 @@ func AddNodeFlags(cmd *cobra.Command) { // node flags cmd.Flags().Bool("fast_sync", config.FastSync, "Fast blockchain syncing") cmd.Flags().Bool("deprecated", config.Deprecated, "Mark blockchain as deprecated") - cmd.Flags().Int64("replay_height", config.ReplayHeight, "Specify replay height") + cmd.Flags().Int64("replay_height", config.ReplayHeight, "Specify which height to replay to, this is useful for exporting at any height") // abci flags cmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'kvstore' for local testing.") From 7e7484d5def4366949395946b5703ea5f51fbdcc Mon Sep 17 00:00:00 2001 From: chengwenxi Date: Fri, 28 Dec 2018 14:18:22 +0800 Subject: [PATCH 26/55] Add reteat last block --- blockchain/store.go | 14 ++++++++++++++ state/execution.go | 2 ++ state/services.go | 1 + state/state.go | 5 +++-- state/store.go | 8 ++++++++ 5 files changed, 28 insertions(+), 2 deletions(-) diff --git a/blockchain/store.go b/blockchain/store.go index 498cca68dba..4b48eff4693 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -186,6 +186,20 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s bs.db.SetSync(nil, nil) } +func (bs *BlockStore) RetreatLastBlock() { + height := bs.height + bs.db.Delete(calcBlockMetaKey(height)) + bs.db.Delete(calcBlockCommitKey(height-1)) + bs.db.Delete(calcSeenCommitKey(height)) + BlockStoreStateJSON{Height: height-1 }.Save(bs.db) + // Done! + bs.mtx.Lock() + bs.height = height + bs.mtx.Unlock() + // Flush + bs.db.SetSync(nil, nil) +} + func (bs *BlockStore) saveBlockPart(height int64, index int, part *types.Part) { if height != bs.Height()+1 { cmn.PanicSanity(fmt.Sprintf("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height)) diff --git a/state/execution.go b/state/execution.go index f2c035cb706..04e3e22e566 100644 --- a/state/execution.go +++ b/state/execution.go @@ -109,6 +109,7 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b } fail.Fail() // XXX + preState := state.Copy() // Save the results before we commit. saveABCIResponses(blockExec.db, block.Height, abciResponses) @@ -153,6 +154,7 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b // Update the app hash and save the state. state.AppHash = appHash SaveState(blockExec.db, state) + SavePreState(blockExec.db, preState) fail.Fail() // XXX diff --git a/state/services.go b/state/services.go index b8f1febe1fb..e494124f9dd 100644 --- a/state/services.go +++ b/state/services.go @@ -74,6 +74,7 @@ type BlockStoreRPC interface { type BlockStore interface { BlockStoreRPC SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) + RetreatLastBlock() } //----------------------------------------------------------------------------------------------------- diff --git a/state/state.go b/state/state.go index 2df6516a723..29599d4d551 100644 --- a/state/state.go +++ b/state/state.go @@ -13,7 +13,8 @@ import ( // database keys var ( - stateKey = []byte("stateKey") + stateKey = []byte("stateKey") + statePreKey = []byte("statePreKey") ) //----------------------------------------------------------------------------- @@ -107,7 +108,7 @@ func (state State) Copy() State { AppHash: state.AppHash, LastResultsHash: state.LastResultsHash, - Deprecated: state.Deprecated, + Deprecated: state.Deprecated, } } diff --git a/state/store.go b/state/store.go index 6b01a829575..13b57b9de2a 100644 --- a/state/store.go +++ b/state/store.go @@ -62,6 +62,10 @@ func LoadState(db dbm.DB) State { return loadState(db, stateKey) } +func LoadPreState(db dbm.DB) State { + return loadState(db, statePreKey) +} + func loadState(db dbm.DB, key []byte) (state State) { buf := db.Get(key) if len(buf) == 0 { @@ -85,6 +89,10 @@ func SaveState(db dbm.DB, state State) { saveState(db, state, stateKey) } +func SavePreState(db dbm.DB, state State) { + saveState(db, state, statePreKey) +} + func saveState(db dbm.DB, state State, key []byte) { nextHeight := state.LastBlockHeight + 1 // If first block, save validators for block 1. From d060768c3764d4d41e45c45dee3283cede2e8272 Mon Sep 17 00:00:00 2001 From: chengwenxi Date: Fri, 4 Jan 2019 15:53:24 +0800 Subject: [PATCH 27/55] Use Goroutine exit --- consensus/replay.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/consensus/replay.go b/consensus/replay.go index 93d1132f4d6..215477af72c 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -8,6 +8,7 @@ import ( "reflect" //"strconv" //"strings" + "runtime" "time" abci "github.com/tendermint/tendermint/abci/types" @@ -420,7 +421,8 @@ func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBl } if config.ReplayHeight > 0 && i > config.ReplayHeight { - cmn.Exit(fmt.Sprintf("Replay from height %d to height %d successfully", appBlockHeight, config.ReplayHeight)) + fmt.Printf("Replay from height %d to height %d successfully", appBlockHeight, config.ReplayHeight) + runtime.Goexit() } h.nBlocks++ From 8f081083447fe4b411614a8c0f61cef702702a86 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Wed, 9 Jan 2019 11:45:44 +0800 Subject: [PATCH 28/55] Fix bug in include evidece into block --- evidence/store.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/evidence/store.go b/evidence/store.go index ccfd2d4876b..639c5a7e8f2 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -97,11 +97,18 @@ func (store *EvidenceStore) PendingEvidence(maxBytes int64) (evidence []types.Ev // If maxBytes is -1, there's no cap on the size of returned evidence. func (store *EvidenceStore) listEvidence(prefixKey string, maxBytes int64) (evidence []types.Evidence) { var bytes int64 + var count int64 iter := dbm.IteratePrefix(store.db, []byte(prefixKey)) defer iter.Close() for ; iter.Valid(); iter.Next() { val := iter.Value() - + count++ + // In block validation, the evidence total size is calculated by (evidence quantity) * (maximum evidence size) + if maxBytes > 0 && count * int64(types.MaxEvidenceBytes) > maxBytes { + return evidence + } + // types.MaxEvidenceBytes must be greater than len(val) + // The following check might not be necessary, but to reduce risk, just keep them if maxBytes > 0 && bytes+int64(len(val)) > maxBytes { return evidence } From 71343008daf9a50e63f146ae5658437245de3ef6 Mon Sep 17 00:00:00 2001 From: chengwenxi Date: Thu, 10 Jan 2019 11:08:04 +0800 Subject: [PATCH 29/55] Update block protocol version --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index ace1b41d273..0cd97c1c84a 100644 --- a/version/version.go +++ b/version/version.go @@ -39,7 +39,7 @@ var ( P2PProtocol Protocol = 5 // BlockProtocol versions all block data structures and processing. - BlockProtocol Protocol = 8 + BlockProtocol Protocol = 9 ) //------------------------------------------------------------------------ From 2313260a41bf5a3a03fa83891698502f1b6eaa9f Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Tue, 15 Jan 2019 22:23:59 +0800 Subject: [PATCH 30/55] disable this check for replay blocks --- state/execution.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/state/execution.go b/state/execution.go index f2c035cb706..f1ef5c39e85 100644 --- a/state/execution.go +++ b/state/execution.go @@ -292,6 +292,12 @@ func execBlockOnProxyApp( func getBeginBlockValidatorInfo(block *types.Block, lastValSet *types.ValidatorSet, stateDB dbm.DB) (abci.LastCommitInfo, []abci.Evidence) { + // Just return empty commitInfo and evidence when replaying blocks + state := LoadState(stateDB) + if block.Height != state.LastBlockHeight + 1 { + return abci.LastCommitInfo{}, nil + } + // Sanity check that commit length matches validator set size - // only applies after first block if block.Height > 1 { From c171a68eafb2e2d6e1afe9fcb14472dc67e8d0e1 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Wed, 16 Jan 2019 11:15:09 +0800 Subject: [PATCH 31/55] For replaying blocks, load history validator set --- state/execution.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/state/execution.go b/state/execution.go index f1ef5c39e85..826d8761544 100644 --- a/state/execution.go +++ b/state/execution.go @@ -292,10 +292,14 @@ func execBlockOnProxyApp( func getBeginBlockValidatorInfo(block *types.Block, lastValSet *types.ValidatorSet, stateDB dbm.DB) (abci.LastCommitInfo, []abci.Evidence) { - // Just return empty commitInfo and evidence when replaying blocks state := LoadState(stateDB) - if block.Height != state.LastBlockHeight + 1 { - return abci.LastCommitInfo{}, nil + // For replaying blocks, load history validator set + if block.Height > 1 && block.Height != state.LastBlockHeight + 1 { + var err error + lastValSet, err = LoadValidators(stateDB, block.Height - 1) + if err != nil { + panic(fmt.Sprintf("failed to load validatorset at heith %d", state.LastBlockHeight)) + } } // Sanity check that commit length matches validator set size - From ef07f82167c421d564a980546fcf6dfaa6b17980 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Wed, 16 Jan 2019 12:15:23 +0800 Subject: [PATCH 32/55] Add appHash checking in replay mode --- consensus/replay.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/consensus/replay.go b/consensus/replay.go index 93d1132f4d6..fa2131750d3 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -414,12 +414,19 @@ func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBl for i := appBlockHeight + 1; i <= finalBlock; i++ { h.logger.Info("Applying block", "height", i) block := h.store.LoadBlock(i) + + if len(appHash) != 0 { + if !bytes.Equal(block.Header.AppHash, appHash) { + panic(fmt.Sprintf("Mismatch replay appHash, expected %s, actual %s", block.AppHash.String(), cmn.HexBytes(appHash).String())) + } + } + appHash, err = sm.ExecCommitBlock(proxyApp.Consensus(), block, h.logger, state.LastValidators, h.stateDB) if err != nil { return nil, err } - if config.ReplayHeight > 0 && i > config.ReplayHeight { + if config.ReplayHeight > 0 && i >= config.ReplayHeight { cmn.Exit(fmt.Sprintf("Replay from height %d to height %d successfully", appBlockHeight, config.ReplayHeight)) } From db83c67377bf5c125e792bfb909ae4cfd9c9c502 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Wed, 16 Jan 2019 13:06:07 +0800 Subject: [PATCH 33/55] stop gorouting instead of stoping process --- consensus/replay.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/consensus/replay.go b/consensus/replay.go index fa2131750d3..66251ccfa46 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -6,6 +6,7 @@ import ( "hash/crc32" "io" "reflect" + "runtime" //"strconv" //"strings" "time" @@ -427,7 +428,8 @@ func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBl } if config.ReplayHeight > 0 && i >= config.ReplayHeight { - cmn.Exit(fmt.Sprintf("Replay from height %d to height %d successfully", appBlockHeight, config.ReplayHeight)) + fmt.Printf("Replay from height %d to height %d successfully", appBlockHeight, config.ReplayHeight) + runtime.Goexit() } h.nBlocks++ From 49a1e66e78c8968ac0719d27bcbd35acb050c228 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Wed, 16 Jan 2019 13:24:47 +0800 Subject: [PATCH 34/55] Improve panic message --- consensus/replay.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/replay.go b/consensus/replay.go index 66251ccfa46..7eff55e4deb 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -418,7 +418,7 @@ func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBl if len(appHash) != 0 { if !bytes.Equal(block.Header.AppHash, appHash) { - panic(fmt.Sprintf("Mismatch replay appHash, expected %s, actual %s", block.AppHash.String(), cmn.HexBytes(appHash).String())) + panic(fmt.Sprintf("AppHash mismatch: expected %s, actual %s", block.AppHash.String(), cmn.HexBytes(appHash).String())) } } From ab90c37168c14396e8636519325dd62cd3a19a05 Mon Sep 17 00:00:00 2001 From: chengwenxi Date: Fri, 25 Jan 2019 14:00:19 +0800 Subject: [PATCH 35/55] Avoid one evidence being committed multiple times --- consensus/reactor_test.go | 1 + evidence/pool.go | 8 ++++++++ evidence/store.go | 1 + state/execution.go | 2 +- state/services.go | 2 ++ state/validation.go | 6 +++++- 6 files changed, 18 insertions(+), 2 deletions(-) diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 1636785c06a..e67856a9c71 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -210,6 +210,7 @@ func (m *mockEvidencePool) Update(block *types.Block, state sm.State) { } m.height++ } +func (m *mockEvidencePool) IsCommitted(types.Evidence) bool { return false } //------------------------------------ diff --git a/evidence/pool.go b/evidence/pool.go index da00a348187..53537b67c56 100644 --- a/evidence/pool.go +++ b/evidence/pool.go @@ -117,6 +117,14 @@ func (evpool *EvidencePool) AddEvidence(evidence types.Evidence) (err error) { return nil } +func (evpool *EvidencePool) IsCommitted(evidence types.Evidence) bool { + ei_ := evpool.evidenceStore.GetEvidence(evidence.Height(), evidence.Hash()) + if ei_ != nil && ei_.Evidence != nil && ei_.Committed == true { + return true + } + return false +} + // MarkEvidenceAsCommitted marks all the evidence as committed and removes it from the queue. func (evpool *EvidencePool) MarkEvidenceAsCommitted(height int64, evidence []types.Evidence) { // make a map of committed evidence to remove from the clist diff --git a/evidence/store.go b/evidence/store.go index 639c5a7e8f2..cc58e6ca2e2 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -186,6 +186,7 @@ func (store *EvidenceStore) MarkEvidenceAsCommitted(evidence types.Evidence) { ei := store.getEvidenceInfo(evidence) ei.Committed = true + ei.Evidence = evidence lookupKey := keyLookup(evidence) store.db.SetSync(lookupKey, cdc.MustMarshalBinaryBare(ei)) diff --git a/state/execution.go b/state/execution.go index 1a2f36265ea..de4d61e6614 100644 --- a/state/execution.go +++ b/state/execution.go @@ -86,7 +86,7 @@ func (blockExec *BlockExecutor) SetEventBus(eventBus types.BlockEventPublisher) // Validation does not mutate state, but does require historical information from the stateDB, // ie. to verify evidence from a validator at an old height. func (blockExec *BlockExecutor) ValidateBlock(state State, block *types.Block) error { - return validateBlock(blockExec.db, state, block) + return validateBlock(blockExec.db, blockExec.evpool, state, block) } // ApplyBlock validates the block against the state, executes it against the app, diff --git a/state/services.go b/state/services.go index e494124f9dd..f6fa3a1dbb5 100644 --- a/state/services.go +++ b/state/services.go @@ -85,6 +85,7 @@ type EvidencePool interface { PendingEvidence(int64) []types.Evidence AddEvidence(types.Evidence) error Update(*types.Block, State) + IsCommitted(types.Evidence) bool } // MockMempool is an empty implementation of a Mempool, useful for testing. @@ -93,3 +94,4 @@ type MockEvidencePool struct{} func (m MockEvidencePool) PendingEvidence(int64) []types.Evidence { return nil } func (m MockEvidencePool) AddEvidence(types.Evidence) error { return nil } func (m MockEvidencePool) Update(*types.Block, State) {} +func (m MockEvidencePool) IsCommitted(types.Evidence) bool {return false} diff --git a/state/validation.go b/state/validation.go index 2bd86c3360d..d11b7b985dc 100644 --- a/state/validation.go +++ b/state/validation.go @@ -13,7 +13,7 @@ import ( //----------------------------------------------------- // Validate block -func validateBlock(stateDB dbm.DB, state State, block *types.Block) error { +func validateBlock(stateDB dbm.DB, evpool EvidencePool, state State, block *types.Block) error { // Validate internal consistency. if err := block.ValidateBasic(); err != nil { return err @@ -149,6 +149,10 @@ func validateBlock(stateDB dbm.DB, state State, block *types.Block) error { // Validate all evidence. for _, ev := range block.Evidence.Evidence { + if evpool.IsCommitted(ev){ + err := errors.New("This evidence has been commited") + return types.NewErrEvidenceInvalid(ev, err) + } if err := VerifyEvidence(stateDB, state, ev); err != nil { return types.NewErrEvidenceInvalid(ev, err) } From 41c9964f4f856283f49293c3b8604387db93a39e Mon Sep 17 00:00:00 2001 From: chengwenxi Date: Fri, 25 Jan 2019 16:08:57 +0800 Subject: [PATCH 36/55] Fix comment --- evidence/pool.go | 2 +- state/validation.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/evidence/pool.go b/evidence/pool.go index 53537b67c56..318cc62b921 100644 --- a/evidence/pool.go +++ b/evidence/pool.go @@ -119,7 +119,7 @@ func (evpool *EvidencePool) AddEvidence(evidence types.Evidence) (err error) { func (evpool *EvidencePool) IsCommitted(evidence types.Evidence) bool { ei_ := evpool.evidenceStore.GetEvidence(evidence.Height(), evidence.Hash()) - if ei_ != nil && ei_.Evidence != nil && ei_.Committed == true { + if ei_ != nil && ei_.Evidence != nil && ei_.Committed { return true } return false diff --git a/state/validation.go b/state/validation.go index d11b7b985dc..9de9094cd8e 100644 --- a/state/validation.go +++ b/state/validation.go @@ -150,7 +150,7 @@ func validateBlock(stateDB dbm.DB, evpool EvidencePool, state State, block *type // Validate all evidence. for _, ev := range block.Evidence.Evidence { if evpool.IsCommitted(ev){ - err := errors.New("This evidence has been commited") + err := errors.New("This evidence has been committed") return types.NewErrEvidenceInvalid(ev, err) } if err := VerifyEvidence(stateDB, state, ev); err != nil { From 7b41941b1ececcfa9a3f1a677da46dc39f20cc2b Mon Sep 17 00:00:00 2001 From: chengwenxi Date: Fri, 25 Jan 2019 17:20:23 +0800 Subject: [PATCH 37/55] Vote nil for repeated evidence in one block --- state/validation.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/state/validation.go b/state/validation.go index 9de9094cd8e..d4dbfdeb267 100644 --- a/state/validation.go +++ b/state/validation.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "fmt" + "encoding/hex" "github.com/tendermint/tendermint/crypto" dbm "github.com/tendermint/tendermint/libs/db" @@ -147,8 +148,16 @@ func validateBlock(stateDB dbm.DB, evpool EvidencePool, state State, block *type return types.NewErrEvidenceOverflow(maxEvidenceBytes, evidenceBytes) } + // key = hex(evidence.Hash()) + // value = evidence.String() + evMap := make(map[string]string) + // Validate all evidence. for _, ev := range block.Evidence.Evidence { + if _, ok := evMap[hex.EncodeToString(ev.Address())]; ok { + err := errors.New("Repeated evidence") + return types.NewErrEvidenceInvalid(ev, err) + } if evpool.IsCommitted(ev){ err := errors.New("This evidence has been committed") return types.NewErrEvidenceInvalid(ev, err) @@ -156,6 +165,7 @@ func validateBlock(stateDB dbm.DB, evpool EvidencePool, state State, block *type if err := VerifyEvidence(stateDB, state, ev); err != nil { return types.NewErrEvidenceInvalid(ev, err) } + evMap[hex.EncodeToString(ev.Address())] = ev.String() } // NOTE: We can't actually verify it's the right proposer because we dont From 7653a781731bbae100bf0f38d0bb7257e437ec75 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Fri, 25 Jan 2019 17:33:18 +0800 Subject: [PATCH 38/55] fix DynamicVerifier for large validator set changes --- lite/base_verifier.go | 28 ++++++---- lite/commit.go | 2 +- lite/dbprovider.go | 3 + lite/doc.go | 4 -- lite/dynamic_verifier.go | 102 +++++++++++++++++++--------------- lite/dynamic_verifier_test.go | 65 ++++++++++++++++++++++ lite/errors/errors.go | 22 -------- lite/multiprovider.go | 2 + lite/provider.go | 2 +- types/block.go | 1 + types/validator_set.go | 32 +++++++++-- 11 files changed, 174 insertions(+), 89 deletions(-) diff --git a/lite/base_verifier.go b/lite/base_verifier.go index fcde01c0e22..1d8feb6ece2 100644 --- a/lite/base_verifier.go +++ b/lite/base_verifier.go @@ -35,34 +35,38 @@ func NewBaseVerifier(chainID string, height int64, valset *types.ValidatorSet) * } // Implements Verifier. -func (bc *BaseVerifier) ChainID() string { - return bc.chainID +func (bv *BaseVerifier) ChainID() string { + return bv.chainID } // Implements Verifier. -func (bc *BaseVerifier) Verify(signedHeader types.SignedHeader) error { - - // We can't verify commits older than bc.height. - if signedHeader.Height < bc.height { +func (bv *BaseVerifier) Verify(signedHeader types.SignedHeader) error { + // We can't verify commits for a different chain. + if signedHeader.ChainID != bv.chainID { + return cmn.NewError("BaseVerifier chainID is %v, cannot verify chainID %v", + bv.chainID, signedHeader.ChainID) + } + // We can't verify commits older than bv.height. + if signedHeader.Height < bv.height { return cmn.NewError("BaseVerifier height is %v, cannot verify height %v", - bc.height, signedHeader.Height) + bv.height, signedHeader.Height) } // We can't verify with the wrong validator set. if !bytes.Equal(signedHeader.ValidatorsHash, - bc.valset.Hash()) { - return lerr.ErrUnexpectedValidators(signedHeader.ValidatorsHash, bc.valset.Hash()) + bv.valset.Hash()) { + return lerr.ErrUnexpectedValidators(signedHeader.ValidatorsHash, bv.valset.Hash()) } // Do basic sanity checks. - err := signedHeader.ValidateBasic(bc.chainID) + err := signedHeader.ValidateBasic(bv.chainID) if err != nil { return cmn.ErrorWrap(err, "in verify") } // Check commit signatures. - err = bc.valset.VerifyCommit( - bc.chainID, signedHeader.Commit.BlockID, + err = bv.valset.VerifyCommit( + bv.chainID, signedHeader.Commit.BlockID, signedHeader.Height, signedHeader.Commit) if err != nil { return cmn.ErrorWrap(err, "in verify") diff --git a/lite/commit.go b/lite/commit.go index 25efb8dc088..6cd3541732a 100644 --- a/lite/commit.go +++ b/lite/commit.go @@ -8,7 +8,7 @@ import ( "github.com/tendermint/tendermint/types" ) -// FullCommit is a signed header (the block header and a commit that signs it), +// FullCommit contains a SignedHeader (the block header and a commit that signs it), // the validator set which signed the commit, and the next validator set. The // next validator set (which is proven from the block header) allows us to // revert to block-by-block updating of lite Verifier's latest validator set, diff --git a/lite/dbprovider.go b/lite/dbprovider.go index 9f4b264f782..ef1b2a5985b 100644 --- a/lite/dbprovider.go +++ b/lite/dbprovider.go @@ -13,6 +13,9 @@ import ( "github.com/tendermint/tendermint/types" ) +var _ PersistentProvider = (*DBProvider)(nil) + +// DBProvider stores commits and validator sets in a DB. type DBProvider struct { logger log.Logger label string diff --git a/lite/doc.go b/lite/doc.go index f68798dc526..429b096e22a 100644 --- a/lite/doc.go +++ b/lite/doc.go @@ -53,10 +53,6 @@ SignedHeader, and that the SignedHeader was to be signed by the exact given validator set, and that the height of the commit is at least height (or greater). -SignedHeader.Commit may be signed by a different validator set, it can get -verified with a BaseVerifier as long as sufficient signatures from the -previous validator set are present in the commit. - DynamicVerifier - this Verifier implements an auto-update and persistence strategy to verify any SignedHeader of the blockchain. diff --git a/lite/dynamic_verifier.go b/lite/dynamic_verifier.go index 6a7720913ce..a29a0ae001e 100644 --- a/lite/dynamic_verifier.go +++ b/lite/dynamic_verifier.go @@ -18,6 +18,9 @@ var _ Verifier = (*DynamicVerifier)(nil) // "source" provider to obtain the needed FullCommits to securely sync with // validator set changes. It stores properly validated data on the // "trusted" local system. +// TODO: make this single threaded and create a new +// ConcurrentDynamicVerifier that wraps it with concurrency. +// see https://github.com/tendermint/tendermint/issues/3170 type DynamicVerifier struct { logger log.Logger chainID string @@ -35,8 +38,8 @@ type DynamicVerifier struct { // trusted provider to store validated data and the source provider to // obtain missing data (e.g. FullCommits). // -// The trusted provider should a CacheProvider, MemProvider or -// files.Provider. The source provider should be a client.HTTPProvider. +// The trusted provider should be a DBProvider. +// The source provider should be a client.HTTPProvider. func NewDynamicVerifier(chainID string, trusted PersistentProvider, source Provider) *DynamicVerifier { return &DynamicVerifier{ logger: log.NewNopLogger(), @@ -47,16 +50,16 @@ func NewDynamicVerifier(chainID string, trusted PersistentProvider, source Provi } } -func (ic *DynamicVerifier) SetLogger(logger log.Logger) { +func (dv *DynamicVerifier) SetLogger(logger log.Logger) { logger = logger.With("module", "lite") - ic.logger = logger - ic.trusted.SetLogger(logger) - ic.source.SetLogger(logger) + dv.logger = logger + dv.trusted.SetLogger(logger) + dv.source.SetLogger(logger) } // Implements Verifier. -func (ic *DynamicVerifier) ChainID() string { - return ic.chainID +func (dv *DynamicVerifier) ChainID() string { + return dv.chainID } // Implements Verifier. @@ -65,50 +68,52 @@ func (ic *DynamicVerifier) ChainID() string { // ic.trusted and ic.source to prove the new validators. On success, it will // try to store the SignedHeader in ic.trusted if the next // validator can be sourced. -func (ic *DynamicVerifier) Verify(shdr types.SignedHeader) error { +func (dv *DynamicVerifier) Verify(shdr types.SignedHeader) error { // Performs synchronization for multi-threads verification at the same height. - ic.mtx.Lock() - if pending := ic.pendingVerifications[shdr.Height]; pending != nil { - ic.mtx.Unlock() + dv.mtx.Lock() + if pending := dv.pendingVerifications[shdr.Height]; pending != nil { + dv.mtx.Unlock() <-pending // pending is chan struct{} } else { pending := make(chan struct{}) - ic.pendingVerifications[shdr.Height] = pending + dv.pendingVerifications[shdr.Height] = pending defer func() { close(pending) - ic.mtx.Lock() - delete(ic.pendingVerifications, shdr.Height) - ic.mtx.Unlock() + dv.mtx.Lock() + delete(dv.pendingVerifications, shdr.Height) + dv.mtx.Unlock() }() - ic.mtx.Unlock() + dv.mtx.Unlock() } //Get the exact trusted commit for h, and if it is // equal to shdr, then don't even verify it, // and just return nil. - trustedFCSameHeight, err := ic.trusted.LatestFullCommit(ic.chainID, shdr.Height, shdr.Height) + trustedFCSameHeight, err := dv.trusted.LatestFullCommit(dv.chainID, shdr.Height, shdr.Height) if err == nil { // If loading trust commit successfully, and trust commit equal to shdr, then don't verify it, // just return nil. if bytes.Equal(trustedFCSameHeight.SignedHeader.Hash(), shdr.Hash()) { - ic.logger.Info(fmt.Sprintf("Load full commit at height %d from cache, there is not need to verify.", shdr.Height)) + dv.logger.Info(fmt.Sprintf("Load full commit at height %d from cache, there is not need to verify.", shdr.Height)) return nil } } else if !lerr.IsErrCommitNotFound(err) { // Return error if it is not CommitNotFound error - ic.logger.Info(fmt.Sprintf("Encountered unknown error in loading full commit at height %d.", shdr.Height)) + dv.logger.Info(fmt.Sprintf("Encountered unknown error in loading full commit at height %d.", shdr.Height)) return err } // Get the latest known full commit <= h-1 from our trusted providers. // The full commit at h-1 contains the valset to sign for h. - h := shdr.Height - 1 - trustedFC, err := ic.trusted.LatestFullCommit(ic.chainID, 1, h) + prevHeight := shdr.Height - 1 + trustedFC, err := dv.trusted.LatestFullCommit(dv.chainID, 1, prevHeight) if err != nil { return err } - if trustedFC.Height() == h { + // sync up to the prevHeight and assert our latest NextValidatorSet + // is the ValidatorSet for the SignedHeader + if trustedFC.Height() == prevHeight { // Return error if valset doesn't match. if !bytes.Equal( trustedFC.NextValidators.Hash(), @@ -122,7 +127,7 @@ func (ic *DynamicVerifier) Verify(shdr types.SignedHeader) error { if !bytes.Equal(trustedFC.NextValidators.Hash(), shdr.Header.ValidatorsHash) { // ... update. - trustedFC, err = ic.updateToHeight(h) + trustedFC, err = dv.updateToHeight(prevHeight) if err != nil { return err } @@ -136,15 +141,22 @@ func (ic *DynamicVerifier) Verify(shdr types.SignedHeader) error { } } + // By now, the SignedHeader is fully validated and we're synced up to + // SignedHeader.Height - 1. To sync to SignedHeader.Height, we need + // the validator set at SignedHeader.Height + 1 so we can verify the + // SignedHeader.NextValidatorSet. + // TODO: is the ValidateFull below mostly redundant with the BaseVerifier.Verify above? + // See https://github.com/tendermint/tendermint/issues/3174. + // Verify the signed header using the matching valset. - cert := NewBaseVerifier(ic.chainID, trustedFC.Height()+1, trustedFC.NextValidators) + cert := NewBaseVerifier(dv.chainID, trustedFC.Height()+1, trustedFC.NextValidators) err = cert.Verify(shdr) if err != nil { return err } // Get the next validator set. - nextValset, err := ic.source.ValidatorSet(ic.chainID, shdr.Height+1) + nextValset, err := dv.source.ValidatorSet(dv.chainID, shdr.Height+1) if lerr.IsErrUnknownValidators(err) { // Ignore this error. return nil @@ -160,31 +172,31 @@ func (ic *DynamicVerifier) Verify(shdr types.SignedHeader) error { } // Validate the full commit. This checks the cryptographic // signatures of Commit against Validators. - if err := nfc.ValidateFull(ic.chainID); err != nil { + if err := nfc.ValidateFull(dv.chainID); err != nil { return err } // Trust it. - return ic.trusted.SaveFullCommit(nfc) + return dv.trusted.SaveFullCommit(nfc) } // verifyAndSave will verify if this is a valid source full commit given the // best match trusted full commit, and if good, persist to ic.trusted. // Returns ErrTooMuchChange when >2/3 of trustedFC did not sign sourceFC. // Panics if trustedFC.Height() >= sourceFC.Height(). -func (ic *DynamicVerifier) verifyAndSave(trustedFC, sourceFC FullCommit) error { +func (dv *DynamicVerifier) verifyAndSave(trustedFC, sourceFC FullCommit) error { if trustedFC.Height() >= sourceFC.Height() { panic("should not happen") } err := trustedFC.NextValidators.VerifyFutureCommit( sourceFC.Validators, - ic.chainID, sourceFC.SignedHeader.Commit.BlockID, + dv.chainID, sourceFC.SignedHeader.Commit.BlockID, sourceFC.SignedHeader.Height, sourceFC.SignedHeader.Commit, ) if err != nil { return err } - return ic.trusted.SaveFullCommit(sourceFC) + return dv.trusted.SaveFullCommit(sourceFC) } // updateToHeight will use divide-and-conquer to find a path to h. @@ -192,29 +204,29 @@ func (ic *DynamicVerifier) verifyAndSave(trustedFC, sourceFC FullCommit) error { // for height h, using repeated applications of bisection if necessary. // // Returns ErrCommitNotFound if source provider doesn't have the commit for h. -func (ic *DynamicVerifier) updateToHeight(h int64) (FullCommit, error) { +func (dv *DynamicVerifier) updateToHeight(h int64) (FullCommit, error) { // Fetch latest full commit from source. - sourceFC, err := ic.source.LatestFullCommit(ic.chainID, h, h) + sourceFC, err := dv.source.LatestFullCommit(dv.chainID, h, h) if err != nil { return FullCommit{}, err } - // Validate the full commit. This checks the cryptographic - // signatures of Commit against Validators. - if err := sourceFC.ValidateFull(ic.chainID); err != nil { - return FullCommit{}, err - } - // If sourceFC.Height() != h, we can't do it. if sourceFC.Height() != h { return FullCommit{}, lerr.ErrCommitNotFound() } + // Validate the full commit. This checks the cryptographic + // signatures of Commit against Validators. + if err := sourceFC.ValidateFull(dv.chainID); err != nil { + return FullCommit{}, err + } + FOR_LOOP: for { // Fetch latest full commit from trusted. - trustedFC, err := ic.trusted.LatestFullCommit(ic.chainID, 1, h) + trustedFC, err := dv.trusted.LatestFullCommit(dv.chainID, 1, h) if err != nil { return FullCommit{}, err } @@ -224,21 +236,21 @@ FOR_LOOP: } // Try to update to full commit with checks. - err = ic.verifyAndSave(trustedFC, sourceFC) + err = dv.verifyAndSave(trustedFC, sourceFC) if err == nil { // All good! return sourceFC, nil } // Handle special case when err is ErrTooMuchChange. - if lerr.IsErrTooMuchChange(err) { + if types.IsErrTooMuchChange(err) { // Divide and conquer. start, end := trustedFC.Height(), sourceFC.Height() if !(start < end) { panic("should not happen") } mid := (start + end) / 2 - _, err = ic.updateToHeight(mid) + _, err = dv.updateToHeight(mid) if err != nil { return FullCommit{}, err } @@ -249,8 +261,8 @@ FOR_LOOP: } } -func (ic *DynamicVerifier) LastTrustedHeight() int64 { - fc, err := ic.trusted.LatestFullCommit(ic.chainID, 1, 1<<63-1) +func (dv *DynamicVerifier) LastTrustedHeight() int64 { + fc, err := dv.trusted.LatestFullCommit(dv.chainID, 1, 1<<63-1) if err != nil { panic("should not happen") } diff --git a/lite/dynamic_verifier_test.go b/lite/dynamic_verifier_test.go index 9ff8ed81f89..fcb97ffa5f9 100644 --- a/lite/dynamic_verifier_test.go +++ b/lite/dynamic_verifier_test.go @@ -2,6 +2,7 @@ package lite import ( "fmt" + "github.com/tendermint/tendermint/types" "sync" "testing" @@ -70,6 +71,70 @@ func TestInquirerValidPath(t *testing.T) { assert.Nil(err, "%+v", err) } +func TestDynamicVerify(t *testing.T) { + trust := NewDBProvider("trust", dbm.NewMemDB()) + source := NewDBProvider("source", dbm.NewMemDB()) + + // 10 commits with one valset, 1 to change, + // 10 commits with the next one + n1, n2 := 10, 10 + nCommits := n1 + n2 + 1 + maxHeight := int64(nCommits) + fcz := make([]FullCommit, nCommits) + + // gen the 2 val sets + chainID := "dynamic-verifier" + power := int64(10) + keys1 := genPrivKeys(5) + vals1 := keys1.ToValidators(power, 0) + keys2 := genPrivKeys(5) + vals2 := keys2.ToValidators(power, 0) + + // make some commits with the first + for i := 0; i < n1; i++ { + fcz[i] = makeFullCommit(int64(i), keys1, vals1, vals1, chainID) + } + + // update the val set + fcz[n1] = makeFullCommit(int64(n1), keys1, vals1, vals2, chainID) + + // make some commits with the new one + for i := n1 + 1; i < nCommits; i++ { + fcz[i] = makeFullCommit(int64(i), keys2, vals2, vals2, chainID) + } + + // Save everything in the source + for _, fc := range fcz { + source.SaveFullCommit(fc) + } + + // Initialize a Verifier with the initial state. + err := trust.SaveFullCommit(fcz[0]) + require.Nil(t, err) + ver := NewDynamicVerifier(chainID, trust, source) + ver.SetLogger(log.TestingLogger()) + + // fetch the latest from the source + latestFC, err := source.LatestFullCommit(chainID, 1, maxHeight) + require.NoError(t, err) + + // try to update to the latest + err = ver.Verify(latestFC.SignedHeader) + require.NoError(t, err) + +} + +func makeFullCommit(height int64, keys privKeys, vals, nextVals *types.ValidatorSet, chainID string) FullCommit { + height += 1 + consHash := []byte("special-params") + appHash := []byte(fmt.Sprintf("h=%d", height)) + resHash := []byte(fmt.Sprintf("res=%d", height)) + return keys.GenFullCommit( + chainID, height, nil, + vals, nextVals, + appHash, consHash, resHash, 0, len(keys)) +} + func TestInquirerVerifyHistorical(t *testing.T) { assert, require := assert.New(t), require.New(t) trust := NewDBProvider("trust", dbm.NewMemDB()) diff --git a/lite/errors/errors.go b/lite/errors/errors.go index 59b6380d89b..75442c726a3 100644 --- a/lite/errors/errors.go +++ b/lite/errors/errors.go @@ -25,12 +25,6 @@ func (e errUnexpectedValidators) Error() string { e.got, e.want) } -type errTooMuchChange struct{} - -func (e errTooMuchChange) Error() string { - return "Insufficient signatures to validate due to valset changes" -} - type errUnknownValidators struct { chainID string height int64 @@ -85,22 +79,6 @@ func IsErrUnexpectedValidators(err error) bool { return false } -//----------------- -// ErrTooMuchChange - -// ErrTooMuchChange indicates that the underlying validator set was changed by >1/3. -func ErrTooMuchChange() error { - return cmn.ErrorWrap(errTooMuchChange{}, "") -} - -func IsErrTooMuchChange(err error) bool { - if err_, ok := err.(cmn.Error); ok { - _, ok := err_.Data().(errTooMuchChange) - return ok - } - return false -} - //----------------- // ErrUnknownValidators diff --git a/lite/multiprovider.go b/lite/multiprovider.go index 734d042c4b9..a05e19b1a97 100644 --- a/lite/multiprovider.go +++ b/lite/multiprovider.go @@ -6,6 +6,8 @@ import ( "github.com/tendermint/tendermint/types" ) +var _ PersistentProvider = (*multiProvider)(nil) + // multiProvider allows you to place one or more caches in front of a source // Provider. It runs through them in order until a match is found. type multiProvider struct { diff --git a/lite/provider.go b/lite/provider.go index 97e06a06d3b..ebab16264dc 100644 --- a/lite/provider.go +++ b/lite/provider.go @@ -1,7 +1,7 @@ package lite import ( - log "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/types" ) diff --git a/types/block.go b/types/block.go index 15b88d81d8b..1c8c8ac651c 100644 --- a/types/block.go +++ b/types/block.go @@ -636,6 +636,7 @@ func (commit *Commit) StringIndented(indent string) string { //----------------------------------------------------------------------------- // SignedHeader is a header along with the commits that prove it. +// It is the basis of the lite client. type SignedHeader struct { *Header `json:"header"` Commit *Commit `json:"commit"` diff --git a/types/validator_set.go b/types/validator_set.go index 8b2d71b8d9f..669670ea653 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -373,8 +373,7 @@ func (vals *ValidatorSet) VerifyCommit(chainID string, blockID BlockID, height i if talliedVotingPower > vals.TotalVotingPower()*2/3 { return nil } - return fmt.Errorf("Invalid commit -- insufficient voting power: got %v, needed %v", - talliedVotingPower, vals.TotalVotingPower()*2/3+1) + return errTooMuchChange{talliedVotingPower, vals.TotalVotingPower()*2/3 + 1} } // VerifyFutureCommit will check to see if the set would be valid with a different @@ -456,12 +455,37 @@ func (vals *ValidatorSet) VerifyFutureCommit(newSet *ValidatorSet, chainID strin } if oldVotingPower <= oldVals.TotalVotingPower()*2/3 { - return cmn.NewError("Invalid commit -- insufficient old voting power: got %v, needed %v", - oldVotingPower, oldVals.TotalVotingPower()*2/3+1) + return errTooMuchChange{oldVotingPower, oldVals.TotalVotingPower()*2/3 + 1} } return nil } +//----------------- +// ErrTooMuchChange + +func IsErrTooMuchChange(err error) bool { + switch err_ := err.(type) { + case cmn.Error: + _, ok := err_.Data().(errTooMuchChange) + return ok + case errTooMuchChange: + return true + default: + return false + } +} + +type errTooMuchChange struct { + got int64 + needed int64 +} + +func (e errTooMuchChange) Error() string { + return fmt.Sprintf("Invalid commit -- insufficient old voting power: got %v, needed %v", e.got, e.needed) +} + +//---------------- + func (vals *ValidatorSet) String() string { return vals.StringIndented("") } From ea969dead9c45ebf139d9403b8203075a52c3876 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Wed, 30 Jan 2019 18:46:53 +0800 Subject: [PATCH 39/55] Add consensus failure to metrics --- consensus/metrics.go | 8 ++++++++ consensus/state.go | 2 ++ 2 files changed, 10 insertions(+) diff --git a/consensus/metrics.go b/consensus/metrics.go index 7b4a3fbc996..b8e8e31b0bc 100644 --- a/consensus/metrics.go +++ b/consensus/metrics.go @@ -14,6 +14,8 @@ const MetricsSubsystem = "consensus" type Metrics struct { // Height of the chain. Height metrics.Gauge + // Consensus Failure + ConsensusFailure metrics.Gauge // Number of rounds. Rounds metrics.Gauge @@ -58,6 +60,12 @@ func PrometheusMetrics(namespace string) *Metrics { Name: "height", Help: "Height of the chain.", }, []string{}), + ConsensusFailure: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: MetricsSubsystem, + Name: "consensus_failure", + Help: "Consensus failure", + }, []string{"height"}), Rounds: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, diff --git a/consensus/state.go b/consensus/state.go index 8706c7bd8f1..ff6577957d9 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -6,6 +6,7 @@ import ( "fmt" "reflect" "runtime/debug" + "strconv" "sync" "time" @@ -580,6 +581,7 @@ func (cs *ConsensusState) receiveRoutine(maxSteps int) { defer func() { if r := recover(); r != nil { + cs.metrics.ConsensusFailure.With("height", strconv.Itoa(int(cs.Height))).Set(float64(1)) cs.Logger.Error("CONSENSUS FAILURE!!!", "err", r, "stack", string(debug.Stack())) // stop gracefully // From 19b67868311850dac90b03f12fdb73444105a083 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Thu, 31 Jan 2019 10:39:48 +0800 Subject: [PATCH 40/55] Add appHash conflict and recheck time to metrics --- state/execution.go | 5 ++++- state/metrics.go | 21 +++++++++++++++++++++ state/validation.go | 3 ++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/state/execution.go b/state/execution.go index de4d61e6614..6c04beee089 100644 --- a/state/execution.go +++ b/state/execution.go @@ -86,7 +86,7 @@ func (blockExec *BlockExecutor) SetEventBus(eventBus types.BlockEventPublisher) // Validation does not mutate state, but does require historical information from the stateDB, // ie. to verify evidence from a validator at an old height. func (blockExec *BlockExecutor) ValidateBlock(state State, block *types.Block) error { - return validateBlock(blockExec.db, blockExec.evpool, state, block) + return validateBlock(blockExec.metrics, blockExec.db, blockExec.evpool, state, block) } // ApplyBlock validates the block against the state, executes it against the app, @@ -204,6 +204,7 @@ func (blockExec *BlockExecutor) Commit( "appHash", fmt.Sprintf("%X", res.Data), ) + startTime := time.Now().UnixNano() // Update mempool. err = blockExec.mempool.Update( block.Height, @@ -211,6 +212,8 @@ func (blockExec *BlockExecutor) Commit( TxPreCheck(state), TxPostCheck(state), ) + endTime := time.Now().UnixNano() + blockExec.metrics.RecheckTime.Observe(float64(endTime-startTime) / 1000000) return res.Data, err } diff --git a/state/metrics.go b/state/metrics.go index 4e99753f0ea..b0b0fa7ec79 100644 --- a/state/metrics.go +++ b/state/metrics.go @@ -12,6 +12,12 @@ const MetricsSubsystem = "state" type Metrics struct { // Time between BeginBlock and EndBlock. BlockProcessingTime metrics.Histogram + + // Time on recheck + RecheckTime metrics.Histogram + + // App hash conflict error + AppHashConflict metrics.Gauge } func PrometheusMetrics(namespace string) *Metrics { @@ -23,11 +29,26 @@ func PrometheusMetrics(namespace string) *Metrics { Help: "Time between BeginBlock and EndBlock in ms.", Buckets: stdprometheus.LinearBuckets(1, 10, 10), }, []string{}), + RecheckTime: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: MetricsSubsystem, + Name: "recheck_time", + Help: "Time cost on recheck in ms.", + Buckets: stdprometheus.LinearBuckets(1, 10, 10), + }, []string{}), + AppHashConflict: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: MetricsSubsystem, + Name: "app_hash_conflict", + Help: "App hash conflict error", + }, []string{"proposer"}), } } func NopMetrics() *Metrics { return &Metrics{ BlockProcessingTime: discard.NewHistogram(), + RecheckTime: discard.NewHistogram(), + AppHashConflict: discard.NewGauge(), } } diff --git a/state/validation.go b/state/validation.go index d4dbfdeb267..4cb6ed84fb8 100644 --- a/state/validation.go +++ b/state/validation.go @@ -14,7 +14,7 @@ import ( //----------------------------------------------------- // Validate block -func validateBlock(stateDB dbm.DB, evpool EvidencePool, state State, block *types.Block) error { +func validateBlock(metrics *Metrics,stateDB dbm.DB, evpool EvidencePool, state State, block *types.Block) error { // Validate internal consistency. if err := block.ValidateBasic(); err != nil { return err @@ -66,6 +66,7 @@ func validateBlock(stateDB dbm.DB, evpool EvidencePool, state State, block *type // Validate app info if !bytes.Equal(block.AppHash, state.AppHash) { + metrics.AppHashConflict.With("proposer", block.ProposerAddress.String()).Set(float64(1)) return fmt.Errorf("Wrong Block.Header.AppHash. Expected %X, got %v", state.AppHash, block.AppHash, From 9bf9ace7d5a53d4797bec5ac2c95350b497cb304 Mon Sep 17 00:00:00 2001 From: chengwenxi Date: Thu, 31 Jan 2019 10:45:46 +0800 Subject: [PATCH 41/55] Optimize transaction search --- rpc/core/tx.go | 11 +++++++--- state/txindex/indexer.go | 3 +-- state/txindex/kv/kv.go | 41 +++++++++++++++++++------------------- state/txindex/null/null.go | 4 ++-- 4 files changed, 31 insertions(+), 28 deletions(-) diff --git a/rpc/core/tx.go b/rpc/core/tx.go index 3bb0f28e824..097e5a7d726 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -9,6 +9,7 @@ import ( ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/state/txindex/null" "github.com/tendermint/tendermint/types" + "github.com/pkg/errors" ) // Tx allows you to query the transaction results. `nil` could mean the @@ -193,12 +194,12 @@ func TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSear return nil, err } - results, err := txIndexer.Search(q) + hashes, err := txIndexer.Search(q) if err != nil { return nil, err } - totalCount := len(results) + totalCount := len(hashes) perPage = validatePerPage(perPage) page = validatePage(page, perPage, totalCount) skipCount := (page - 1) * perPage @@ -206,7 +207,11 @@ func TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSear apiResults := make([]*ctypes.ResultTx, cmn.MinInt(perPage, totalCount-skipCount)) var proof types.TxProof for i := 0; i < len(apiResults); i++ { - r := results[skipCount+i] + h := hashes[skipCount+i] + r, err := txIndexer.Get(h) + if err != nil { + return nil, errors.Wrapf(err, "failed to get Tx{%X}", h) + } height := r.Height index := r.Index diff --git a/state/txindex/indexer.go b/state/txindex/indexer.go index ab509f96532..19e6103fe54 100644 --- a/state/txindex/indexer.go +++ b/state/txindex/indexer.go @@ -9,7 +9,6 @@ import ( // TxIndexer interface defines methods to index and search transactions. type TxIndexer interface { - // AddBatch analyzes, indexes and stores a batch of transactions. AddBatch(b *Batch) error @@ -21,7 +20,7 @@ type TxIndexer interface { Get(hash []byte) (*types.TxResult, error) // Search allows you to query for transactions. - Search(q *query.Query) ([]*types.TxResult, error) + Search(q *query.Query) ([][]byte, error) } //---------------------------------------------------- diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 93249b7f94e..7b9d0007303 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/hex" "fmt" - "sort" "strconv" "strings" "time" @@ -141,7 +140,7 @@ func (txi *TxIndex) Index(result *types.TxResult) error { // result for it (2) for range queries it is better for the client to provide // both lower and upper bounds, so we are not performing a full scan. Results // from querying indexes are then intersected and returned to the caller. -func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { +func (txi *TxIndex) Search(q *query.Query) ([][]byte, error) { var hashes [][]byte var hashesInitialized bool @@ -155,9 +154,9 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { } else if ok { res, err := txi.Get(hash) if res == nil { - return []*types.TxResult{}, nil + return hashes, nil } - return []*types.TxResult{res}, errors.Wrap(err, "error while retrieving the result") + return hashes, errors.Wrap(err, "error while retrieving the result") } // conditions to skip because they're handled before "everything else" @@ -197,25 +196,25 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { } } - results := make([]*types.TxResult, len(hashes)) - i := 0 - for _, h := range hashes { - results[i], err = txi.Get(h) - if err != nil { - return nil, errors.Wrapf(err, "failed to get Tx{%X}", h) - } - i++ - } + //results := make([]*types.TxResult, len(hashes)) + //i := 0 + //for _, h := range hashes { + // results[i], err = txi.Get(h) + // if err != nil { + // return nil, errors.Wrapf(err, "failed to get Tx{%X}", h) + // } + // i++ + //} // sort by height & index by default - sort.Slice(results, func(i, j int) bool { - if results[i].Height == results[j].Height { - return results[i].Index < results[j].Index - } - return results[i].Height < results[j].Height - }) - - return results, nil + //sort.Slice(results, func(i, j int) bool { + // if results[i].Height == results[j].Height { + // return results[i].Index < results[j].Index + // } + // return results[i].Height < results[j].Height + //}) + + return hashes, nil } func lookForHash(conditions []query.Condition) (hash []byte, err error, ok bool) { diff --git a/state/txindex/null/null.go b/state/txindex/null/null.go index f85de2e6f93..2cdd06b8444 100644 --- a/state/txindex/null/null.go +++ b/state/txindex/null/null.go @@ -28,6 +28,6 @@ func (txi *TxIndex) Index(result *types.TxResult) error { return nil } -func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { - return []*types.TxResult{}, nil +func (txi *TxIndex) Search(q *query.Query) ([][]byte, error) { + return [][]byte{}, nil } From 4a9b0fdfa048a56a2272ebfbd15ae7ba37510ef1 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Thu, 31 Jan 2019 12:03:27 +0800 Subject: [PATCH 42/55] Change to count type --- consensus/metrics.go | 6 ++++-- consensus/state.go | 2 +- state/execution.go | 3 ++- state/metrics.go | 10 +++++----- state/validation.go | 3 ++- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/consensus/metrics.go b/consensus/metrics.go index b8e8e31b0bc..5c1442f8602 100644 --- a/consensus/metrics.go +++ b/consensus/metrics.go @@ -15,7 +15,7 @@ type Metrics struct { // Height of the chain. Height metrics.Gauge // Consensus Failure - ConsensusFailure metrics.Gauge + ConsensusFailure metrics.Counter // Number of rounds. Rounds metrics.Gauge @@ -60,7 +60,7 @@ func PrometheusMetrics(namespace string) *Metrics { Name: "height", Help: "Height of the chain.", }, []string{}), - ConsensusFailure: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + ConsensusFailure: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "consensus_failure", @@ -161,6 +161,8 @@ func NopMetrics() *Metrics { return &Metrics{ Height: discard.NewGauge(), + ConsensusFailure: discard.NewCounter(), + Rounds: discard.NewGauge(), Validators: discard.NewGauge(), diff --git a/consensus/state.go b/consensus/state.go index ff6577957d9..b454b656a63 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -581,7 +581,7 @@ func (cs *ConsensusState) receiveRoutine(maxSteps int) { defer func() { if r := recover(); r != nil { - cs.metrics.ConsensusFailure.With("height", strconv.Itoa(int(cs.Height))).Set(float64(1)) + cs.metrics.ConsensusFailure.With("height", strconv.Itoa(int(cs.Height))).Add(float64(1)) cs.Logger.Error("CONSENSUS FAILURE!!!", "err", r, "stack", string(debug.Stack())) // stop gracefully // diff --git a/state/execution.go b/state/execution.go index 6c04beee089..69027da08e6 100644 --- a/state/execution.go +++ b/state/execution.go @@ -3,6 +3,7 @@ package state import ( "bytes" "fmt" + "strconv" "strings" "time" @@ -213,7 +214,7 @@ func (blockExec *BlockExecutor) Commit( TxPostCheck(state), ) endTime := time.Now().UnixNano() - blockExec.metrics.RecheckTime.Observe(float64(endTime-startTime) / 1000000) + blockExec.metrics.RecheckTime.With("height", strconv.FormatInt(block.Height, 10)).Observe(float64(endTime-startTime) / 1000000) return res.Data, err } diff --git a/state/metrics.go b/state/metrics.go index b0b0fa7ec79..45730cc7619 100644 --- a/state/metrics.go +++ b/state/metrics.go @@ -17,7 +17,7 @@ type Metrics struct { RecheckTime metrics.Histogram // App hash conflict error - AppHashConflict metrics.Gauge + AppHashConflict metrics.Counter } func PrometheusMetrics(namespace string) *Metrics { @@ -35,13 +35,13 @@ func PrometheusMetrics(namespace string) *Metrics { Name: "recheck_time", Help: "Time cost on recheck in ms.", Buckets: stdprometheus.LinearBuckets(1, 10, 10), - }, []string{}), - AppHashConflict: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + }, []string{"height"}), + AppHashConflict: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "app_hash_conflict", Help: "App hash conflict error", - }, []string{"proposer"}), + }, []string{"proposer", "height"}), } } @@ -49,6 +49,6 @@ func NopMetrics() *Metrics { return &Metrics{ BlockProcessingTime: discard.NewHistogram(), RecheckTime: discard.NewHistogram(), - AppHashConflict: discard.NewGauge(), + AppHashConflict: discard.NewCounter(), } } diff --git a/state/validation.go b/state/validation.go index 4cb6ed84fb8..9293784bdcb 100644 --- a/state/validation.go +++ b/state/validation.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "encoding/hex" + "strconv" "github.com/tendermint/tendermint/crypto" dbm "github.com/tendermint/tendermint/libs/db" @@ -66,7 +67,7 @@ func validateBlock(metrics *Metrics,stateDB dbm.DB, evpool EvidencePool, state S // Validate app info if !bytes.Equal(block.AppHash, state.AppHash) { - metrics.AppHashConflict.With("proposer", block.ProposerAddress.String()).Set(float64(1)) + metrics.AppHashConflict.With("proposer", block.ProposerAddress.String(), "height", strconv.FormatInt(block.Height, 10)).Add(float64(1)) return fmt.Errorf("Wrong Block.Header.AppHash. Expected %X, got %v", state.AppHash, block.AppHash, From b5282f8e5d6f28ee4b0b0e5ebce778e8f4aa20ac Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Thu, 31 Jan 2019 12:33:46 +0800 Subject: [PATCH 43/55] remove height label from recheckTime metrics --- state/execution.go | 3 +-- state/metrics.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/state/execution.go b/state/execution.go index 69027da08e6..6c04beee089 100644 --- a/state/execution.go +++ b/state/execution.go @@ -3,7 +3,6 @@ package state import ( "bytes" "fmt" - "strconv" "strings" "time" @@ -214,7 +213,7 @@ func (blockExec *BlockExecutor) Commit( TxPostCheck(state), ) endTime := time.Now().UnixNano() - blockExec.metrics.RecheckTime.With("height", strconv.FormatInt(block.Height, 10)).Observe(float64(endTime-startTime) / 1000000) + blockExec.metrics.RecheckTime.Observe(float64(endTime-startTime) / 1000000) return res.Data, err } diff --git a/state/metrics.go b/state/metrics.go index 45730cc7619..2250667dbac 100644 --- a/state/metrics.go +++ b/state/metrics.go @@ -35,7 +35,7 @@ func PrometheusMetrics(namespace string) *Metrics { Name: "recheck_time", Help: "Time cost on recheck in ms.", Buckets: stdprometheus.LinearBuckets(1, 10, 10), - }, []string{"height"}), + }, []string{}), AppHashConflict: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, From 2d13d99c0bf766ec130326687ba36941f15cfb92 Mon Sep 17 00:00:00 2001 From: chengwenxi Date: Wed, 13 Feb 2019 10:59:12 +0800 Subject: [PATCH 44/55] bug fix for evidence check --- state/validation.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/state/validation.go b/state/validation.go index 9293784bdcb..4072290987d 100644 --- a/state/validation.go +++ b/state/validation.go @@ -15,7 +15,7 @@ import ( //----------------------------------------------------- // Validate block -func validateBlock(metrics *Metrics,stateDB dbm.DB, evpool EvidencePool, state State, block *types.Block) error { +func validateBlock(metrics *Metrics, stateDB dbm.DB, evpool EvidencePool, state State, block *types.Block) error { // Validate internal consistency. if err := block.ValidateBasic(); err != nil { return err @@ -152,22 +152,22 @@ func validateBlock(metrics *Metrics,stateDB dbm.DB, evpool EvidencePool, state S // key = hex(evidence.Hash()) // value = evidence.String() - evMap := make(map[string]string) + evMap := make(map[string]bool) // Validate all evidence. for _, ev := range block.Evidence.Evidence { - if _, ok := evMap[hex.EncodeToString(ev.Address())]; ok { - err := errors.New("Repeated evidence") + if _, ok := evMap[hex.EncodeToString(ev.Hash())]; ok { + err := errors.New("repeated evidence") return types.NewErrEvidenceInvalid(ev, err) } - if evpool.IsCommitted(ev){ - err := errors.New("This evidence has been committed") + if evpool != nil && evpool.IsCommitted(ev) { + err := errors.New("evidence was already committed") return types.NewErrEvidenceInvalid(ev, err) } if err := VerifyEvidence(stateDB, state, ev); err != nil { return types.NewErrEvidenceInvalid(ev, err) } - evMap[hex.EncodeToString(ev.Address())] = ev.String() + evMap[hex.EncodeToString(ev.Hash())] = true } // NOTE: We can't actually verify it's the right proposer because we dont From 1d89d5d5cb68e188bfe36ff4fdb88d208411110c Mon Sep 17 00:00:00 2001 From: Adrian Brink Date: Thu, 21 Feb 2019 12:36:54 +0100 Subject: [PATCH 45/55] Patch remote signer to work with the KMS --- privval/remote_signer.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/privval/remote_signer.go b/privval/remote_signer.go index 5d6339c3e7e..94ad84a2399 100644 --- a/privval/remote_signer.go +++ b/privval/remote_signer.go @@ -56,7 +56,7 @@ func (sc *RemoteSignerClient) getPubKey() (crypto.PubKey, error) { sc.lock.Lock() defer sc.lock.Unlock() - err := writeMsg(sc.conn, &PubKeyMsg{}) + err := writeMsg(sc.conn, &PubKeyRequest{}) if err != nil { return nil, err } @@ -66,7 +66,7 @@ func (sc *RemoteSignerClient) getPubKey() (crypto.PubKey, error) { return nil, err } - return res.(*PubKeyMsg).PubKey, nil + return res.(*PubKeyResponse).PubKey, nil } // SignVote implements PrivValidator. @@ -152,7 +152,8 @@ type RemoteSignerMsg interface{} func RegisterRemoteSignerMsg(cdc *amino.Codec) { cdc.RegisterInterface((*RemoteSignerMsg)(nil), nil) - cdc.RegisterConcrete(&PubKeyMsg{}, "tendermint/remotesigner/PubKeyMsg", nil) + cdc.RegisterConcrete(&PubKeyRequest{}, "tendermint/remotesigner/PubKeyRequest", nil) + cdc.RegisterConcrete(&PubKeyResponse{}, "tendermint/remotesigner/PubKeyResponse", nil) cdc.RegisterConcrete(&SignVoteRequest{}, "tendermint/remotesigner/SignVoteRequest", nil) cdc.RegisterConcrete(&SignedVoteResponse{}, "tendermint/remotesigner/SignedVoteResponse", nil) cdc.RegisterConcrete(&SignProposalRequest{}, "tendermint/remotesigner/SignProposalRequest", nil) @@ -161,9 +162,13 @@ func RegisterRemoteSignerMsg(cdc *amino.Codec) { cdc.RegisterConcrete(&PingResponse{}, "tendermint/remotesigner/PingResponse", nil) } -// PubKeyMsg is a PrivValidatorSocket message containing the public key. -type PubKeyMsg struct { +// PubKeyRequest requests the consensus public key from the remote signer. +type PubKeyRequest struct{} + +// PubKeyResponse is a PrivValidatorSocket message containing the public key. +type PubKeyResponse struct { PubKey crypto.PubKey + Error *RemoteSignerError } // SignVoteRequest is a PrivValidatorSocket message containing a vote. @@ -227,10 +232,10 @@ func handleRequest(req RemoteSignerMsg, chainID string, privVal types.PrivValida var err error switch r := req.(type) { - case *PubKeyMsg: + case *PubKeyRequest: var p crypto.PubKey p = privVal.GetPubKey() - res = &PubKeyMsg{p} + res = &PubKeyResponse{p, nil} case *SignVoteRequest: err = privVal.SignVote(chainID, r.Vote) if err != nil { From 410f72a66962856e901be8c0b57d9e429bf35abd Mon Sep 17 00:00:00 2001 From: chengwenxi Date: Fri, 22 Feb 2019 16:02:51 +0800 Subject: [PATCH 46/55] switch fields ordering --- types/canonical.go | 2 +- types/vote.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/types/canonical.go b/types/canonical.go index eabd7684887..7369d2c0745 100644 --- a/types/canonical.go +++ b/types/canonical.go @@ -36,8 +36,8 @@ type CanonicalVote struct { Type SignedMsgType // type alias for byte Height int64 `binary:"fixed64"` Round int64 `binary:"fixed64"` - Timestamp time.Time BlockID CanonicalBlockID + Timestamp time.Time ChainID string } diff --git a/types/vote.go b/types/vote.go index bf14d403bcb..030a6bb24b0 100644 --- a/types/vote.go +++ b/types/vote.go @@ -52,8 +52,8 @@ type Vote struct { Type SignedMsgType `json:"type"` Height int64 `json:"height"` Round int `json:"round"` - Timestamp time.Time `json:"timestamp"` BlockID BlockID `json:"block_id"` // zero if vote is nil. + Timestamp time.Time `json:"timestamp"` ValidatorAddress Address `json:"validator_address"` ValidatorIndex int `json:"validator_index"` Signature []byte `json:"signature"` From a7e51230fc651aad405797deac73fb3111103e20 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Wed, 27 Feb 2019 17:19:38 +0800 Subject: [PATCH 47/55] Cherry pick from https://github.com/tendermint/tendermint/pull/3150 --- p2p/conn_set.go | 8 ++++++ p2p/dummy/peer.go | 10 +++++++ p2p/peer.go | 18 +++++++++---- p2p/peer_set_test.go | 2 ++ p2p/peer_test.go | 53 +++++++++++++++++++++--------------- p2p/pex/pex_reactor_test.go | 2 ++ p2p/switch_test.go | 54 +++++++++++++++++++++++++++++++++++++ p2p/test_util.go | 22 +++++++++++++-- p2p/transport.go | 16 ++++++----- 9 files changed, 150 insertions(+), 35 deletions(-) diff --git a/p2p/conn_set.go b/p2p/conn_set.go index f960c0e8836..d646227831a 100644 --- a/p2p/conn_set.go +++ b/p2p/conn_set.go @@ -11,6 +11,7 @@ type ConnSet interface { HasIP(net.IP) bool Set(net.Conn, []net.IP) Remove(net.Conn) + RemoveAddr(net.Addr) } type connSetItem struct { @@ -62,6 +63,13 @@ func (cs *connSet) Remove(c net.Conn) { delete(cs.conns, c.RemoteAddr().String()) } +func (cs *connSet) RemoveAddr(addr net.Addr) { + cs.Lock() + defer cs.Unlock() + + delete(cs.conns, addr.String()) +} + func (cs *connSet) Set(c net.Conn, ips []net.IP) { cs.Lock() defer cs.Unlock() diff --git a/p2p/dummy/peer.go b/p2p/dummy/peer.go index 71def27e0c3..57edafc6769 100644 --- a/p2p/dummy/peer.go +++ b/p2p/dummy/peer.go @@ -55,6 +55,16 @@ func (p *peer) RemoteIP() net.IP { return net.ParseIP("127.0.0.1") } +// Addr always returns tcp://localhost:8800. +func (p *peer) RemoteAddr() net.Addr { + return &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 8800} +} + +// CloseConn always returns nil. +func (p *peer) CloseConn() error { + return nil +} + // Status always returns empry connection status. func (p *peer) Status() tmconn.ConnectionStatus { return tmconn.ConnectionStatus{} diff --git a/p2p/peer.go b/p2p/peer.go index da301d4978e..73332a2aa8f 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -18,15 +18,18 @@ type Peer interface { cmn.Service FlushStop() - ID() ID // peer's cryptographic ID - RemoteIP() net.IP // remote IP of the connection + ID() ID // peer's cryptographic ID + RemoteIP() net.IP // remote IP of the connection + RemoteAddr() net.Addr // remote address of the connection IsOutbound() bool // did we dial the peer IsPersistent() bool // do we redial this peer when we disconnect + CloseConn() error // close original connection + NodeInfo() NodeInfo // peer's info Status() tmconn.ConnectionStatus - OriginalAddr() *NetAddress + OriginalAddr() *NetAddress // original address for outbound peers Send(byte, []byte) bool TrySend(byte, []byte) bool @@ -296,6 +299,11 @@ func (p *peer) hasChannel(chID byte) bool { return false } +// CloseConn closes original connection. Used for cleaning up in cases where the peer had not been started at all. +func (p *peer) CloseConn() error { + return p.peerConn.conn.Close() +} + //--------------------------------------------------- // methods only used for testing // TODO: can we remove these? @@ -305,8 +313,8 @@ func (pc *peerConn) CloseConn() { pc.conn.Close() // nolint: errcheck } -// Addr returns peer's remote network address. -func (p *peer) Addr() net.Addr { +// RemoteAddr returns peer's remote network address. +func (p *peer) RemoteAddr() net.Addr { return p.peerConn.conn.RemoteAddr() } diff --git a/p2p/peer_set_test.go b/p2p/peer_set_test.go index 3eb5357d3e6..1d2372fb087 100644 --- a/p2p/peer_set_test.go +++ b/p2p/peer_set_test.go @@ -30,6 +30,8 @@ func (mp *mockPeer) Get(s string) interface{} { return s } func (mp *mockPeer) Set(string, interface{}) {} func (mp *mockPeer) RemoteIP() net.IP { return mp.ip } func (mp *mockPeer) OriginalAddr() *NetAddress { return nil } +func (mp *mockPeer) RemoteAddr() net.Addr { return &net.TCPAddr{IP: mp.ip, Port: 8800} } +func (mp *mockPeer) CloseConn() error { return nil } // Returns a mock peer func newMockPeer(ip net.IP) *mockPeer { diff --git a/p2p/peer_test.go b/p2p/peer_test.go index e53d6013b99..90be311317c 100644 --- a/p2p/peer_test.go +++ b/p2p/peer_test.go @@ -39,7 +39,7 @@ func TestPeerBasic(t *testing.T) { assert.False(p.IsPersistent()) p.persistent = true assert.True(p.IsPersistent()) - assert.Equal(rp.Addr().DialString(), p.Addr().String()) + assert.Equal(rp.Addr().DialString(), p.RemoteAddr().String()) assert.Equal(rp.ID(), p.ID()) } @@ -137,9 +137,9 @@ type remotePeer struct { PrivKey crypto.PrivKey Config *config.P2PConfig addr *NetAddress - quit chan struct{} channels cmn.HexBytes listenAddr string + listener net.Listener } func (rp *remotePeer) Addr() *NetAddress { @@ -159,25 +159,45 @@ func (rp *remotePeer) Start() { if e != nil { golog.Fatalf("net.Listen tcp :0: %+v", e) } + rp.listener = l rp.addr = NewNetAddress(PubKeyToID(rp.PrivKey.PubKey()), l.Addr()) - rp.quit = make(chan struct{}) if rp.channels == nil { rp.channels = []byte{testCh} } - go rp.accept(l) + go rp.accept() } func (rp *remotePeer) Stop() { - close(rp.quit) + rp.listener.Close() } -func (rp *remotePeer) accept(l net.Listener) { +func (rp *remotePeer) Dial(addr *NetAddress) (net.Conn, error) { + conn, err := addr.DialTimeout(1 * time.Second) + if err != nil { + return nil, err + } + pc, err := testInboundPeerConn(conn, rp.Config, rp.PrivKey) + if err != nil { + return nil, err + } + _, err = handshake(pc.conn, time.Second, rp.nodeInfo()) + if err != nil { + return nil, err + } + return conn, err +} + +func (rp *remotePeer) accept() { conns := []net.Conn{} for { - conn, err := l.Accept() + conn, err := rp.listener.Accept() if err != nil { - golog.Fatalf("Failed to accept conn: %+v", err) + golog.Printf("Failed to accept conn: %+v", err) + for _, conn := range conns { + _ = conn.Close() + } + return } pc, err := testInboundPeerConn(conn, rp.Config, rp.PrivKey) @@ -185,31 +205,20 @@ func (rp *remotePeer) accept(l net.Listener) { golog.Fatalf("Failed to create a peer: %+v", err) } - _, err = handshake(pc.conn, time.Second, rp.nodeInfo(l)) + _, err = handshake(pc.conn, time.Second, rp.nodeInfo()) if err != nil { golog.Fatalf("Failed to perform handshake: %+v", err) } conns = append(conns, conn) - - select { - case <-rp.quit: - for _, conn := range conns { - if err := conn.Close(); err != nil { - golog.Fatal(err) - } - } - return - default: - } } } -func (rp *remotePeer) nodeInfo(l net.Listener) NodeInfo { +func (rp *remotePeer) nodeInfo() NodeInfo { return DefaultNodeInfo{ ProtocolVersion: defaultProtocolVersion, ID_: rp.Addr().ID, - ListenAddr: l.Addr().String(), + ListenAddr: rp.listener.Addr().String(), Network: "testing", Version: "1.2.3-rc0-deadbeef", Channels: rp.channels, diff --git a/p2p/pex/pex_reactor_test.go b/p2p/pex/pex_reactor_test.go index 2e2f3f249d6..f5125c6036b 100644 --- a/p2p/pex/pex_reactor_test.go +++ b/p2p/pex/pex_reactor_test.go @@ -404,6 +404,8 @@ func (mockPeer) TrySend(byte, []byte) bool { return false } func (mockPeer) Set(string, interface{}) {} func (mockPeer) Get(string) interface{} { return nil } func (mockPeer) OriginalAddr() *p2p.NetAddress { return nil } +func (mockPeer) RemoteAddr() net.Addr { return &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 8800} } +func (mockPeer) CloseConn() error { return nil } func assertPeersWithTimeout( t *testing.T, diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 6c515be0296..4f8d26b6a5d 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -3,7 +3,9 @@ package p2p import ( "bytes" "fmt" + "io" "io/ioutil" + "net" "net/http" "net/http/httptest" "regexp" @@ -477,6 +479,58 @@ func TestSwitchFullConnectivity(t *testing.T) { } } +func TestSwitchAcceptRoutine(t *testing.T) { + cfg.MaxNumInboundPeers = 5 + + // make switch + sw := MakeSwitch(cfg, 1, "testing", "123.123.123", initSwitchFunc) + err := sw.Start() + require.NoError(t, err) + defer sw.Stop() + + remotePeers := make([]*remotePeer, 0) + assert.Equal(t, 0, sw.Peers().Size()) + + // 1. check we connect up to MaxNumInboundPeers + for i := 0; i < cfg.MaxNumInboundPeers; i++ { + rp := &remotePeer{PrivKey: ed25519.GenPrivKey(), Config: cfg} + remotePeers = append(remotePeers, rp) + rp.Start() + c, err := rp.Dial(sw.NodeInfo().NetAddress()) + require.NoError(t, err) + // spawn a reading routine to prevent connection from closing + go func(c net.Conn) { + for { + one := make([]byte, 1) + _, err := c.Read(one) + if err != nil { + return + } + } + }(c) + } + time.Sleep(10 * time.Millisecond) + assert.Equal(t, cfg.MaxNumInboundPeers, sw.Peers().Size()) + + // 2. check we close new connections if we already have MaxNumInboundPeers peers + rp := &remotePeer{PrivKey: ed25519.GenPrivKey(), Config: cfg} + rp.Start() + conn, err := rp.Dial(sw.NodeInfo().NetAddress()) + require.NoError(t, err) + // check conn is closed + one := make([]byte, 1) + conn.SetReadDeadline(time.Now().Add(10 * time.Millisecond)) + _, err = conn.Read(one) + assert.Equal(t, io.EOF, err) + assert.Equal(t, cfg.MaxNumInboundPeers, sw.Peers().Size()) + rp.Stop() + + // stop remote peers + for _, rp := range remotePeers { + rp.Stop() + } +} + func BenchmarkSwitchBroadcast(b *testing.B) { s1, s2 := MakeSwitchPair(b, func(i int, sw *Switch) *Switch { // Make bar reactors of bar channels each diff --git a/p2p/test_util.go b/p2p/test_util.go index ea788b79fdb..04629fcae08 100644 --- a/p2p/test_util.go +++ b/p2p/test_util.go @@ -247,17 +247,35 @@ func testNodeInfo(id ID, name string) NodeInfo { } func testNodeInfoWithNetwork(id ID, name, network string) NodeInfo { + port, err := getFreePort() + if err != nil { + panic(err) + } return DefaultNodeInfo{ ProtocolVersion: defaultProtocolVersion, ID_: id, - ListenAddr: fmt.Sprintf("127.0.0.1:%d", cmn.RandIntn(64512)+1023), + ListenAddr: fmt.Sprintf("127.0.0.1:%d", port), Network: network, Version: "1.2.3-rc0-deadbeef", Channels: []byte{testCh}, Moniker: name, Other: DefaultNodeInfoOther{ TxIndex: "on", - RPCAddress: fmt.Sprintf("127.0.0.1:%d", cmn.RandIntn(64512)+1023), + RPCAddress: fmt.Sprintf("127.0.0.1:%d", port), }, } } + +func getFreePort() (int, error) { + addr, err := net.ResolveTCPAddr("tcp", "localhost:0") + if err != nil { + return 0, err + } + + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return 0, err + } + defer l.Close() + return l.Addr().(*net.TCPAddr).Port, nil +} diff --git a/p2p/transport.go b/p2p/transport.go index 69fab312ce3..2d4420a11ad 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -52,6 +52,9 @@ type Transport interface { // Dial connects to the Peer for the address. Dial(NetAddress, peerConfig) (Peer, error) + + // Cleanup any resources associated with Peer. + Cleanup(Peer) } // transportLifecycle bundles the methods for callers to control start and stop @@ -274,6 +277,13 @@ func (mt *MultiplexTransport) acceptPeers() { } } +// Cleanup removes the given address from the connections set and +// closes the connection. +func (mt *MultiplexTransport) Cleanup(peer Peer) { + mt.conns.RemoveAddr(peer.RemoteAddr()) + _ = peer.CloseConn() +} + func (mt *MultiplexTransport) cleanup(c net.Conn) error { mt.conns.Remove(c) @@ -418,12 +428,6 @@ func (mt *MultiplexTransport) wrapPeer( PeerMetrics(cfg.metrics), ) - // Wait for Peer to Stop so we can cleanup. - go func(c net.Conn) { - <-p.Quit() - _ = mt.cleanup(c) - }(c) - return p } From fdbb67655db45726173895fd4ca6458180a8fc11 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Wed, 27 Feb 2019 17:20:12 +0800 Subject: [PATCH 48/55] Fix mempool reactor goroutine leak --- mempool/reactor.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/mempool/reactor.go b/mempool/reactor.go index 072f966753b..9483cf36fac 100644 --- a/mempool/reactor.go +++ b/mempool/reactor.go @@ -116,19 +116,21 @@ func (memR *MempoolReactor) broadcastTxRoutine(peer p2p.Peer) { var next *clist.CElement for { + select { + case <-peer.Quit(): + return + case <-memR.Quit(): + return + default: + // continue to broadcast transaction + } // This happens because the CElement we were looking at got garbage // collected (removed). That is, .NextWait() returned nil. Go ahead and // start from the beginning. if next == nil { - select { - case <-memR.Mempool.TxsWaitChan(): // Wait until a tx is available - if next = memR.Mempool.TxsFront(); next == nil { - continue - } - case <-peer.Quit(): - return - case <-memR.Quit(): - return + <-memR.Mempool.TxsWaitChan() // Wait until a tx is available + if next = memR.Mempool.TxsFront(); next == nil { + continue } } From 0481372b5e3a51228f043c1a461a88c4868f92fd Mon Sep 17 00:00:00 2001 From: chengwenxi Date: Tue, 5 Mar 2019 11:13:57 +0800 Subject: [PATCH 49/55] Fix consensus round issue --- consensus/state.go | 12 +++++------ consensus/types/round_state.go | 39 +++++++++++++++++----------------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index b454b656a63..3450574375a 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -78,9 +78,8 @@ type ConsensusState struct { evpool sm.EvidencePool // internal state - mtx sync.RWMutex + mtx sync.RWMutex cstypes.RoundState - triggeredTimeoutPrecommit bool state sm.State // State until height-1. // state changes may be triggered by: msgs from peers, @@ -728,6 +727,7 @@ func (cs *ConsensusState) handleTxsAvailable() { cs.mtx.Lock() defer cs.mtx.Unlock() // we only need to do this for round 0 + cs.enterNewRound(cs.Height, 0) cs.enterPropose(cs.Height, 0) } @@ -778,7 +778,7 @@ func (cs *ConsensusState) enterNewRound(height int64, round int) { cs.ProposalBlockParts = nil } cs.Votes.SetRound(round + 1) // also track next round (round+1) to allow round-skipping - cs.triggeredTimeoutPrecommit = false + cs.TriggeredTimeoutPrecommit = false cs.eventBus.PublishEventNewRound(cs.NewRoundEvent()) cs.metrics.Rounds.Set(float64(round)) @@ -1135,12 +1135,12 @@ func (cs *ConsensusState) enterPrecommit(height int64, round int) { func (cs *ConsensusState) enterPrecommitWait(height int64, round int) { logger := cs.Logger.With("height", height, "round", round) - if cs.Height != height || round < cs.Round || (cs.Round == round && cs.triggeredTimeoutPrecommit) { + if cs.Height != height || round < cs.Round || (cs.Round == round && cs.TriggeredTimeoutPrecommit) { logger.Debug( fmt.Sprintf( "enterPrecommitWait(%v/%v): Invalid args. "+ "Current state is Height/Round: %v/%v/, triggeredTimeoutPrecommit:%v", - height, round, cs.Height, cs.Round, cs.triggeredTimeoutPrecommit)) + height, round, cs.Height, cs.Round, cs.TriggeredTimeoutPrecommit)) return } if !cs.Votes.Precommits(round).HasTwoThirdsAny() { @@ -1150,7 +1150,7 @@ func (cs *ConsensusState) enterPrecommitWait(height int64, round int) { defer func() { // Done enterPrecommitWait: - cs.triggeredTimeoutPrecommit = true + cs.TriggeredTimeoutPrecommit = true cs.newStep() }() diff --git a/consensus/types/round_state.go b/consensus/types/round_state.go index 418f73a8ef6..eab13b6c70b 100644 --- a/consensus/types/round_state.go +++ b/consensus/types/round_state.go @@ -65,25 +65,26 @@ func (rs RoundStepType) String() string { // NOTE: Not thread safe. Should only be manipulated by functions downstream // of the cs.receiveRoutine type RoundState struct { - Height int64 `json:"height"` // Height we are working on - Round int `json:"round"` - Step RoundStepType `json:"step"` - StartTime time.Time `json:"start_time"` - CommitTime time.Time `json:"commit_time"` // Subjective time when +2/3 precommits for Block at Round were found - Validators *types.ValidatorSet `json:"validators"` - Proposal *types.Proposal `json:"proposal"` - ProposalBlock *types.Block `json:"proposal_block"` - ProposalBlockParts *types.PartSet `json:"proposal_block_parts"` - LockedRound int `json:"locked_round"` - LockedBlock *types.Block `json:"locked_block"` - LockedBlockParts *types.PartSet `json:"locked_block_parts"` - ValidRound int `json:"valid_round"` // Last known round with POL for non-nil valid block. - ValidBlock *types.Block `json:"valid_block"` // Last known block of POL mentioned above. - ValidBlockParts *types.PartSet `json:"valid_block_parts"` // Last known block parts of POL metnioned above. - Votes *HeightVoteSet `json:"votes"` - CommitRound int `json:"commit_round"` // - LastCommit *types.VoteSet `json:"last_commit"` // Last precommits at Height-1 - LastValidators *types.ValidatorSet `json:"last_validators"` + Height int64 `json:"height"` // Height we are working on + Round int `json:"round"` + Step RoundStepType `json:"step"` + StartTime time.Time `json:"start_time"` + CommitTime time.Time `json:"commit_time"` // Subjective time when +2/3 precommits for Block at Round were found + Validators *types.ValidatorSet `json:"validators"` + Proposal *types.Proposal `json:"proposal"` + ProposalBlock *types.Block `json:"proposal_block"` + ProposalBlockParts *types.PartSet `json:"proposal_block_parts"` + LockedRound int `json:"locked_round"` + LockedBlock *types.Block `json:"locked_block"` + LockedBlockParts *types.PartSet `json:"locked_block_parts"` + ValidRound int `json:"valid_round"` // Last known round with POL for non-nil valid block. + ValidBlock *types.Block `json:"valid_block"` // Last known block of POL mentioned above. + ValidBlockParts *types.PartSet `json:"valid_block_parts"` // Last known block parts of POL metnioned above. + Votes *HeightVoteSet `json:"votes"` + CommitRound int `json:"commit_round"` // + LastCommit *types.VoteSet `json:"last_commit"` // Last precommits at Height-1 + LastValidators *types.ValidatorSet `json:"last_validators"` + TriggeredTimeoutPrecommit bool `json:"triggered_timeout_precommit"` } // Compressed version of the RoundState for use in RPC From 9960b2513ae7455a90f9948e2392c0f2504b15b0 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Tue, 5 Mar 2019 15:32:46 +0800 Subject: [PATCH 50/55] Add maximum msg size in CheckTx --- mempool/mempool.go | 10 ++++++++++ mempool/reactor.go | 5 +++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mempool/mempool.go b/mempool/mempool.go index c5f966c4ef0..bf91fe124cc 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -65,6 +65,9 @@ var ( // ErrMempoolIsFull means Tendermint & an application can't handle that much load ErrMempoolIsFull = errors.New("Mempool is full") + + // ErrTxTooLarge means the tx is too big to be sent in a message to other peers + ErrTxTooLarge = fmt.Errorf("Tx too large. Max size is %d", maxTxSize) ) // ErrPreCheck is returned when tx is too big @@ -309,6 +312,13 @@ func (mem *Mempool) CheckTx(tx types.Tx, cb func(*abci.Response)) (err error) { return ErrMempoolIsFull } + // The size of the corresponding amino-encoded TxMessage + // can't be larger than the maxMsgSize, otherwise we can't + // relay it to peers. + if len(tx) > maxTxSize { + return ErrTxTooLarge + } + if mem.preCheck != nil { if err := mem.preCheck(tx); err != nil { return ErrPreCheck{err} diff --git a/mempool/reactor.go b/mempool/reactor.go index 072f966753b..f9f39602840 100644 --- a/mempool/reactor.go +++ b/mempool/reactor.go @@ -18,8 +18,9 @@ import ( const ( MempoolChannel = byte(0x30) - maxMsgSize = 1048576 // 1MB TODO make it configurable - peerCatchupSleepIntervalMS = 100 // If peer is behind, sleep this amount + maxMsgSize = 1048576 // 1MB TODO make it configurable + maxTxSize = maxMsgSize - 8 // account for amino overhead of TxMessage + peerCatchupSleepIntervalMS = 100 // If peer is behind, sleep this amount ) // MempoolReactor handles mempool tx broadcasting amongst peers. From 2a506343d45fb4ac921fe7e2d17cb77d64ed77de Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Tue, 5 Mar 2019 16:00:54 +0800 Subject: [PATCH 51/55] Remove useless func --- mempool/reactor.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/mempool/reactor.go b/mempool/reactor.go index f9f39602840..36aece9051e 100644 --- a/mempool/reactor.go +++ b/mempool/reactor.go @@ -5,8 +5,7 @@ import ( "reflect" "time" - amino "github.com/tendermint/go-amino" - abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/go-amino" "github.com/tendermint/tendermint/libs/clist" "github.com/tendermint/tendermint/libs/log" @@ -99,11 +98,6 @@ func (memR *MempoolReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { } } -// BroadcastTx is an alias for Mempool.CheckTx. Broadcasting itself happens in peer routines. -func (memR *MempoolReactor) BroadcastTx(tx types.Tx, cb func(*abci.Response)) error { - return memR.Mempool.CheckTx(tx, cb) -} - // PeerState describes the state of a peer. type PeerState interface { GetHeight() int64 From 3267dd1d695be58e9ddf662abd819507044be6e6 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Tue, 5 Mar 2019 16:20:06 +0800 Subject: [PATCH 52/55] Add missed cherry pick --- p2p/switch.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/p2p/switch.go b/p2p/switch.go index 0490eebb8cd..383543b8844 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -210,6 +210,7 @@ func (sw *Switch) OnStart() error { func (sw *Switch) OnStop() { // Stop peers for _, p := range sw.peers.List() { + sw.transport.Cleanup(p) p.Stop() if sw.peers.Remove(p) { sw.metrics.Peers.Add(float64(-1)) @@ -304,6 +305,7 @@ func (sw *Switch) stopAndRemovePeer(peer Peer, reason interface{}) { if sw.peers.Remove(peer) { sw.metrics.Peers.Add(float64(-1)) } + sw.transport.Cleanup(peer) peer.Stop() for _, reactor := range sw.reactors { reactor.RemovePeer(peer, reason) @@ -529,13 +531,16 @@ func (sw *Switch) acceptRoutine() { "max", sw.config.MaxNumInboundPeers, ) - _ = p.Stop() + sw.transport.Cleanup(p) continue } if err := sw.addPeer(p); err != nil { - _ = p.Stop() + sw.transport.Cleanup(p) + if p.IsRunning() { + _ = p.Stop() + } sw.Logger.Info( "Ignoring inbound connection: error while adding peer", "err", err, @@ -593,7 +598,10 @@ func (sw *Switch) addOutboundPeerWithConfig( } if err := sw.addPeer(p); err != nil { - _ = p.Stop() + sw.transport.Cleanup(p) + if p.IsRunning() { + _ = p.Stop() + } return err } @@ -628,7 +636,8 @@ func (sw *Switch) filterPeer(p Peer) error { return nil } -// addPeer starts up the Peer and adds it to the Switch. +// addPeer starts up the Peer and adds it to the Switch. Error is returned if +// the peer is filtered out or failed to start or can't be added. func (sw *Switch) addPeer(p Peer) error { if err := sw.filterPeer(p); err != nil { return err @@ -641,6 +650,8 @@ func (sw *Switch) addPeer(p Peer) error { if err := sw.startInitPeer(p); err != nil { return err } + } else { + sw.Logger.Error("Won't start a peer - switch is not running", "peer", p) } // Add the peer to .peers. From 30797717dfa7d627d4be06467bf6d2da89b95f80 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Wed, 6 Mar 2019 14:28:52 +0800 Subject: [PATCH 53/55] Add latest tx to the back of cache list, in case of being removed by other tx insertion --- mempool/mempool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mempool/mempool.go b/mempool/mempool.go index c5f966c4ef0..3a1921bc21d 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -676,7 +676,7 @@ func (cache *mapTxCache) Push(tx types.Tx) bool { // Use the tx hash in the cache txHash := sha256.Sum256(tx) if moved, exists := cache.map_[txHash]; exists { - cache.list.MoveToFront(moved) + cache.list.MoveToBack(moved) return false } From 0a950f92bdb2a5c3ef6feebcae99adfb21752ad9 Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Thu, 7 Mar 2019 17:16:22 +0800 Subject: [PATCH 54/55] Cherry pick from https://github.com/tendermint/tendermint/pull/3232 --- p2p/pex/addrbook.go | 27 ++++- p2p/pex/addrbook_test.go | 256 +++++++++++++++++++++++++++++++++++---- 2 files changed, 257 insertions(+), 26 deletions(-) diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index cfeefb34310..3cda9ac7473 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -369,6 +369,10 @@ func (a *addrBook) GetSelection() []*p2p.NetAddress { return allAddr[:numAddresses] } +func percentageOfNum(p, n int) int { + return int(math.Round((float64(p) / float64(100)) * float64(n))) +} + // GetSelectionWithBias implements AddrBook. // It randomly selects some addresses (old & new). Suitable for peer-exchange protocols. // Must never return a nil address. @@ -408,11 +412,28 @@ func (a *addrBook) GetSelectionWithBias(biasTowardsNewAddrs int) []*p2p.NetAddre newBucketToAddrsMap := make(map[int]map[string]struct{}) var newIndex int + // initialize counters used to count old and new added addresses. + // len(oldBucketToAddrsMap) cannot be used as multiple addresses can endup in the same bucket. + var oldAddressesAdded int + var newAddressesAdded int + + // number of new addresses that, if possible, should be in the beginning of the selection + numRequiredNewAdd := percentageOfNum(biasTowardsNewAddrs, numAddresses) + selectionIndex := 0 ADDRS_LOOP: for selectionIndex < numAddresses { - pickFromOldBucket := int((float64(selectionIndex)/float64(numAddresses))*100) >= biasTowardsNewAddrs - pickFromOldBucket = (pickFromOldBucket && a.nOld > 0) || a.nNew == 0 + // biasedTowardsOldAddrs indicates if the selection can switch to old addresses + biasedTowardsOldAddrs := selectionIndex >= numRequiredNewAdd + // An old addresses is selected if: + // - the bias is for old and old addressees are still available or, + // - there are no new addresses or all new addresses have been selected. + // numAddresses <= a.nOld + a.nNew therefore it is guaranteed that there are enough + // addresses to fill the selection + pickFromOldBucket := + (biasedTowardsOldAddrs && oldAddressesAdded < a.nOld) || + a.nNew == 0 || newAddressesAdded >= a.nNew + bucket := make(map[string]*knownAddress) // loop until we pick a random non-empty bucket @@ -450,6 +471,7 @@ ADDRS_LOOP: oldBucketToAddrsMap[oldIndex] = make(map[string]struct{}) } oldBucketToAddrsMap[oldIndex][selectedAddr.String()] = struct{}{} + oldAddressesAdded++ } else { if addrsMap, ok := newBucketToAddrsMap[newIndex]; ok { if _, ok = addrsMap[selectedAddr.String()]; ok { @@ -459,6 +481,7 @@ ADDRS_LOOP: newBucketToAddrsMap[newIndex] = make(map[string]struct{}) } newBucketToAddrsMap[newIndex][selectedAddr.String()] = struct{}{} + newAddressesAdded++ } selection[selectionIndex] = selectedAddr diff --git a/p2p/pex/addrbook_test.go b/p2p/pex/addrbook_test.go index ade02d49011..78dd7bce338 100644 --- a/p2p/pex/addrbook_test.go +++ b/p2p/pex/addrbook_test.go @@ -4,38 +4,17 @@ import ( "encoding/hex" "fmt" "io/ioutil" + "math" "os" "testing" "github.com/stretchr/testify/require" - "github.com/stretchr/testify/assert" - cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/p2p" ) -func createTempFileName(prefix string) string { - f, err := ioutil.TempFile("", prefix) - if err != nil { - panic(err) - } - fname := f.Name() - err = f.Close() - if err != nil { - panic(err) - } - return fname -} - -func deleteTempFile(fname string) { - err := os.Remove(fname) - if err != nil { - panic(err) - } -} - func TestAddrBookPickAddress(t *testing.T) { fname := createTempFileName("addrbook_test") defer deleteTempFile(fname) @@ -239,6 +218,34 @@ func TestAddrBookRemoveAddress(t *testing.T) { assert.Equal(t, 0, book.Size()) } +func TestAddrBookGetSelectionWithOneMarkedGood(t *testing.T) { + // create a book with 10 addresses, 1 good/old and 9 new + book, fname := createAddrBookWithMOldAndNNewAddrs(t, 1, 9) + defer deleteTempFile(fname) + + addrs := book.GetSelectionWithBias(biasToSelectNewPeers) + assert.NotNil(t, addrs) + assertMOldAndNNewAddrsInSelection(t, 1, 9, addrs, book) +} + +func TestAddrBookGetSelectionWithOneNotMarkedGood(t *testing.T) { + // create a book with 10 addresses, 9 good/old and 1 new + book, fname := createAddrBookWithMOldAndNNewAddrs(t, 9, 1) + defer deleteTempFile(fname) + + addrs := book.GetSelectionWithBias(biasToSelectNewPeers) + assert.NotNil(t, addrs) + assertMOldAndNNewAddrsInSelection(t, 9, 1, addrs, book) +} + +func TestAddrBookGetSelectionReturnsNilWhenAddrBookIsEmpty(t *testing.T) { + book, fname := createAddrBookWithMOldAndNNewAddrs(t, 0, 0) + defer deleteTempFile(fname) + + addrs := book.GetSelectionWithBias(biasToSelectNewPeers) + assert.Nil(t, addrs) +} + func TestAddrBookGetSelection(t *testing.T) { fname := createTempFileName("addrbook_test") defer deleteTempFile(fname) @@ -336,8 +343,13 @@ func TestAddrBookGetSelectionWithBias(t *testing.T) { } } got, expected := int((float64(good)/float64(len(selection)))*100), (100 - biasTowardsNewAddrs) - if got >= expected { - t.Fatalf("expected more good peers (%% got: %d, %% expected: %d, number of good addrs: %d, total: %d)", got, expected, good, len(selection)) + // compute some slack to protect against small differences due to rounding: + slack := int(math.Round(float64(100) / float64(len(selection)))) + if got > expected+slack { + t.Fatalf("got more good peers (%% got: %d, %% expected: %d, number of good addrs: %d, total: %d)", got, expected, good, len(selection)) + } + if got < expected-slack { + t.Fatalf("got fewer good peers (%% got: %d, %% expected: %d, number of good addrs: %d, total: %d)", got, expected, good, len(selection)) } } @@ -417,3 +429,199 @@ func TestPrivatePeers(t *testing.T) { assert.True(t, ok) } } + +func testAddrBookAddressSelection(t *testing.T, bookSize int) { + // generate all combinations of old (m) and new addresses + for nOld := 0; nOld <= bookSize; nOld++ { + nNew := bookSize - nOld + dbgStr := fmt.Sprintf("book of size %d (new %d, old %d)", bookSize, nNew, nOld) + + // create book and get selection + book, fname := createAddrBookWithMOldAndNNewAddrs(t, nOld, nNew) + defer deleteTempFile(fname) + addrs := book.GetSelectionWithBias(biasToSelectNewPeers) + assert.NotNil(t, addrs, "%s - expected a non-nil selection", dbgStr) + nAddrs := len(addrs) + assert.NotZero(t, nAddrs, "%s - expected at least one address in selection", dbgStr) + + // check there's no nil addresses + for _, addr := range addrs { + if addr == nil { + t.Fatalf("%s - got nil address in selection %v", dbgStr, addrs) + } + } + + // XXX: shadowing + nOld, nNew := countOldAndNewAddrsInSelection(addrs, book) + + // Given: + // n - num new addrs, m - num old addrs + // k - num new addrs expected in the beginning (based on bias %) + // i=min(n, k), aka expFirstNew + // j=min(m, r-i), aka expOld + // + // We expect this layout: + // indices: 0...i-1 i...i+j-1 i+j...r + // addresses: N0..Ni-1 O0..Oj-1 Ni... + // + // There is at least one partition and at most three. + var ( + k = percentageOfNum(biasToSelectNewPeers, nAddrs) + expFirstNew = cmn.MinInt(nNew, k) + expOld = cmn.MinInt(nOld, nAddrs-expFirstNew) + expNew = nAddrs - expOld + expLastNew = expNew - expFirstNew + ) + + // Verify that the number of old and new addresses are as expected + if nNew < expNew || nNew > expNew { + t.Fatalf("%s - expected new addrs %d, got %d", dbgStr, expNew, nNew) + } + if nOld < expOld || nOld > expOld { + t.Fatalf("%s - expected old addrs %d, got %d", dbgStr, expOld, nOld) + } + + // Verify that the order of addresses is as expected + // Get the sequence types and lengths of the selection + seqLens, seqTypes, err := analyseSelectionLayout(book, addrs) + assert.NoError(t, err, "%s", dbgStr) + + // Build a list with the expected lengths of partitions and another with the expected types, e.g.: + // expSeqLens = [10, 22], expSeqTypes = [1, 2] + // means we expect 10 new (type 1) addresses followed by 22 old (type 2) addresses. + var expSeqLens []int + var expSeqTypes []int + + switch { + case expOld == 0: // all new addresses + expSeqLens = []int{nAddrs} + expSeqTypes = []int{1} + case expFirstNew == 0: // all old addresses + expSeqLens = []int{nAddrs} + expSeqTypes = []int{2} + case nAddrs-expFirstNew-expOld == 0: // new addresses, old addresses + expSeqLens = []int{expFirstNew, expOld} + expSeqTypes = []int{1, 2} + default: // new addresses, old addresses, new addresses + expSeqLens = []int{expFirstNew, expOld, expLastNew} + expSeqTypes = []int{1, 2, 1} + } + + assert.Equal(t, expSeqLens, seqLens, + "%s - expected sequence lengths of old/new %v, got %v", + dbgStr, expSeqLens, seqLens) + assert.Equal(t, expSeqTypes, seqTypes, + "%s - expected sequence types (1-new, 2-old) was %v, got %v", + dbgStr, expSeqTypes, seqTypes) + } +} + +func TestMultipleAddrBookAddressSelection(t *testing.T) { + // test books with smaller size, < N + const N = 32 + for bookSize := 1; bookSize < N; bookSize++ { + testAddrBookAddressSelection(t, bookSize) + } + + // Test for two books with sizes from following ranges + ranges := [...][]int{{33, 100}, {100, 175}} + var bookSizes []int + for _, r := range ranges { + bookSizes = append(bookSizes, cmn.RandIntn(r[1]-r[0])+r[0]) + } + t.Logf("Testing address selection for the following book sizes %v\n", bookSizes) + for _, bookSize := range bookSizes { + testAddrBookAddressSelection(t, bookSize) + } +} + +func assertMOldAndNNewAddrsInSelection(t *testing.T, m, n int, addrs []*p2p.NetAddress, book *addrBook) { + nOld, nNew := countOldAndNewAddrsInSelection(addrs, book) + assert.Equal(t, m, nOld, "old addresses") + assert.Equal(t, n, nNew, "new addresses") +} + +func createTempFileName(prefix string) string { + f, err := ioutil.TempFile("", prefix) + if err != nil { + panic(err) + } + fname := f.Name() + err = f.Close() + if err != nil { + panic(err) + } + return fname +} + +func deleteTempFile(fname string) { + err := os.Remove(fname) + if err != nil { + panic(err) + } +} + +func createAddrBookWithMOldAndNNewAddrs(t *testing.T, nOld, nNew int) (book *addrBook, fname string) { + fname = createTempFileName("addrbook_test") + + book = NewAddrBook(fname, true) + book.SetLogger(log.TestingLogger()) + assert.Zero(t, book.Size()) + + randAddrs := randNetAddressPairs(t, nOld) + for _, addr := range randAddrs { + book.AddAddress(addr.addr, addr.src) + book.MarkGood(addr.addr) + } + + randAddrs = randNetAddressPairs(t, nNew) + for _, addr := range randAddrs { + book.AddAddress(addr.addr, addr.src) + } + + return +} + +func countOldAndNewAddrsInSelection(addrs []*p2p.NetAddress, book *addrBook) (nOld, nNew int) { + for _, addr := range addrs { + if book.IsGood(addr) { + nOld++ + } else { + nNew++ + } + } + return +} + +// Analyse the layout of the selection specified by 'addrs' +// Returns: +// - seqLens - the lengths of the sequences of addresses of same type +// - seqTypes - the types of sequences in selection +func analyseSelectionLayout(book *addrBook, addrs []*p2p.NetAddress) (seqLens, seqTypes []int, err error) { + // address types are: 0 - nil, 1 - new, 2 - old + var ( + prevType = 0 + currentSeqLen = 0 + ) + + for _, addr := range addrs { + addrType := 0 + if book.IsGood(addr) { + addrType = 2 + } else { + addrType = 1 + } + if addrType != prevType && prevType != 0 { + seqLens = append(seqLens, currentSeqLen) + seqTypes = append(seqTypes, prevType) + currentSeqLen = 0 + } + currentSeqLen++ + prevType = addrType + } + + seqLens = append(seqLens, currentSeqLen) + seqTypes = append(seqTypes, prevType) + + return +} From 20afc34cd560a51f9550835345ecdfaa823dedee Mon Sep 17 00:00:00 2001 From: HaoyangLiu Date: Thu, 7 Mar 2019 17:24:15 +0800 Subject: [PATCH 55/55] add stopMtx for FlushStop and OnStop --- p2p/conn/connection.go | 19 +++++++++- p2p/pex/pex_reactor_test.go | 75 +++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index fb20c47756a..9207349146f 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -8,6 +8,7 @@ import ( "math" "net" "reflect" + "sync" "sync/atomic" "time" @@ -89,6 +90,10 @@ type MConnection struct { quitSendRoutine chan struct{} doneSendRoutine chan struct{} + // used to ensure FlushStop and OnStop + // are safe to call concurrently. + stopMtx sync.Mutex + flushTimer *cmn.ThrottleTimer // flush writes as necessary but throttled. pingTimer *cmn.RepeatTimer // send pings periodically @@ -210,8 +215,17 @@ func (c *MConnection) OnStart() error { // It additionally ensures that all successful // .Send() calls will get flushed before closing // the connection. -// NOTE: it is not safe to call this method more than once. func (c *MConnection) FlushStop() { + c.stopMtx.Lock() + defer c.stopMtx.Unlock() + + select { + case <-c.quitSendRoutine: + // already quit via OnStop + return + default: + } + c.BaseService.OnStop() c.flushTimer.Stop() c.pingTimer.Stop() @@ -247,6 +261,9 @@ func (c *MConnection) FlushStop() { // OnStop implements BaseService func (c *MConnection) OnStop() { + c.stopMtx.Lock() + defer c.stopMtx.Unlock() + select { case <-c.quitSendRoutine: // already quit via FlushStop diff --git a/p2p/pex/pex_reactor_test.go b/p2p/pex/pex_reactor_test.go index 2e2f3f249d6..275ba9fbbf4 100644 --- a/p2p/pex/pex_reactor_test.go +++ b/p2p/pex/pex_reactor_test.go @@ -316,6 +316,81 @@ func TestPEXReactorCrawlStatus(t *testing.T) { // TODO: test } +// connect a peer to a seed, wait a bit, then stop it. +// this should give it time to request addrs and for the seed +// to call FlushStop, and allows us to test calling Stop concurrently +// with FlushStop. Before a fix, this non-deterministically reproduced +// https://github.com/tendermint/tendermint/issues/3231. +func TestPEXReactorSeedModeFlushStop(t *testing.T) { + N := 2 + switches := make([]*p2p.Switch, N) + + // directory to store address books + dir, err := ioutil.TempDir("", "pex_reactor") + require.Nil(t, err) + defer os.RemoveAll(dir) // nolint: errcheck + + books := make([]*addrBook, N) + logger := log.TestingLogger() + + // create switches + for i := 0; i < N; i++ { + switches[i] = p2p.MakeSwitch(cfg, i, "testing", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { + books[i] = NewAddrBook(filepath.Join(dir, fmt.Sprintf("addrbook%d.json", i)), false) + books[i].SetLogger(logger.With("pex", i)) + sw.SetAddrBook(books[i]) + + sw.SetLogger(logger.With("pex", i)) + + config := &PEXReactorConfig{} + if i == 0 { + // first one is a seed node + config = &PEXReactorConfig{SeedMode: true} + } + r := NewPEXReactor(books[i], config) + r.SetLogger(logger.With("pex", i)) + r.SetEnsurePeersPeriod(250 * time.Millisecond) + sw.AddReactor("pex", r) + + return sw + }) + } + + for _, sw := range switches { + err := sw.Start() // start switch and reactors + require.Nil(t, err) + } + + reactor := switches[0].Reactors()["pex"].(*PEXReactor) + peerID := switches[1].NodeInfo().ID() + + err = switches[1].DialPeerWithAddress(switches[0].NodeInfo().NetAddress(), false) + assert.NoError(t, err) + + // sleep up to a second while waiting for the peer to send us a message. + // this isn't perfect since it's possible the peer sends us a msg and we FlushStop + // before this loop catches it. but non-deterministically it works pretty well. + for i := 0; i < 1000; i++ { + v := reactor.lastReceivedRequests.Get(string(peerID)) + if v != nil { + break + } + time.Sleep(time.Millisecond) + } + + // by now the FlushStop should have happened. Try stopping the peer. + // it should be safe to do this. + peers := switches[0].Peers().List() + for _, peer := range peers { + peer.Stop() + } + + // stop the switches + for _, s := range switches { + s.Stop() + } +} + func TestPEXReactorDoesNotAddPrivatePeersToAddrBook(t *testing.T) { peer := p2p.CreateRandomPeer(false)