Consensus failure on empty validators set #2634
Description
Tendermint version: 0.25.0 + 3744e82 cherry picked from develop
While I've been working on cosmos/cosmos-sdk#2422 I've experienced another consensus failure - I now think that #2617 is not enough to correctly handle the no-validators genesis scenario:
$ gaiad start --home=GAIAD
I[10-12|17:57:38.601] Starting ABCI with Tendermint module=main
I[10-12|17:57:38.666] Starting multiAppConn module=proxy impl=multiAppConn
I[10-12|17:57:38.666] Starting localClient module=abci-client connection=query impl=localClient
I[10-12|17:57:38.666] Starting localClient module=abci-client connection=mempool impl=localClient
I[10-12|17:57:38.667] Starting localClient module=abci-client connection=consensus impl=localClient
I[10-12|17:57:38.667] ABCI Handshake module=consensus appHeight=0 appHash=
I[10-12|17:57:38.667] ABCI Replay Blocks module=consensus appHeight=0 storeHeight=0 stateHeight=0
Block Height: 0[{PubKey:{Type:ed25519 Data:[231 167 208 65 203 202 171 87 111 10 101 39 30 27 22 118 97 38 12 11 192 20 112 109 87 8 111 60 169 102 8 19] XXX_NoUnkeyedLiteral:{} XXX_unrecognized:[] XXX_sizecache:0} Power:10 XXX_NoUnkeyedLiteral:{} XXX_unrecognized:[] XXX_sizecache:0}]
I[10-12|17:57:38.701] Completed ABCI Handshake - Tendermint and App are synced module=consensus appHeight=0 appHash=
I[10-12|17:57:38.702] This node is a validator module=consensus addr=E34363369BD8FA1C7ABFBB9F645FE989488597C8 pubKey=PubKeyEd25519{E7A7D041CBCAAB576F0A65271E1B167661260C0BC014706D57086F3CA9660813}
I[10-12|17:57:38.763] P2P Node ID module=p2p ID=89acbe8e9a8e613cc790a84ddf61c6feb9b89345 file=GAIAD/config/node_key.json
I[10-12|17:57:38.763] Starting Node module=node impl=Node
I[10-12|17:57:38.763] Starting EventBus module=events impl=EventBus
I[10-12|17:57:38.764] Starting RPC HTTP server on tcp://0.0.0.0:26657 module=rpc-server
I[10-12|17:57:38.764] Starting P2P Switch module=p2p impl="P2P Switch"
I[10-12|17:57:38.764] Starting MempoolReactor module=mempool impl=MempoolReactor
I[10-12|17:57:38.764] Starting BlockchainReactor module=blockchain impl=BlockchainReactor
I[10-12|17:57:38.764] Starting ConsensusReactor module=consensus impl=ConsensusReactor
I[10-12|17:57:38.764] ConsensusReactor module=consensus fastSync=false
I[10-12|17:57:38.765] Starting ConsensusState module=consensus impl=ConsensusState
I[10-12|17:57:38.765] Starting baseWAL module=consensus wal=GAIAD/data/cs.wal/wal impl=baseWAL
I[10-12|17:57:38.775] Starting TimeoutTicker module=consensus impl=TimeoutTicker
I[10-12|17:57:38.775] Catchup by replaying consensus messages module=consensus height=1
I[10-12|17:57:38.775] Replay: Done module=consensus
I[10-12|17:57:38.775] Starting EvidenceReactor module=evidence impl=EvidenceReactor
I[10-12|17:57:38.776] Starting PEXReactor module=p2p impl=PEXReactor
I[10-12|17:57:38.776] Starting AddrBook module=p2p book=GAIAD/config/addrbook.json impl=AddrBook
I[10-12|17:57:38.776] Starting IndexerService module=txindex impl=IndexerService
I[10-12|17:57:38.776] Ensure peers module=p2p numOutPeers=0 numInPeers=0 numDialing=0 numToDial=10
I[10-12|17:57:38.776] No addresses to dial nor connected peers. Falling back to seeds module=p2p
E[10-12|17:57:38.776] Couldn't connect to any seeds module=p2p
I[10-12|17:57:43.730] Timed out module=consensus dur=4.954032201s height=1 round=0 step=RoundStepNewHeight
I[10-12|17:57:43.730] enterNewRound(1/0). Current: 1/0/RoundStepNewHeight module=consensus height=1 round=0
I[10-12|17:57:43.730] enterPropose(1/0). Current: 1/0/RoundStepNewRound module=consensus height=1 round=0
I[10-12|17:57:43.730] enterPropose: Our turn to propose module=consensus height=1 round=0 proposer=E34363369BD8FA1C7ABFBB9F645FE989488597C8 privValidator="PrivValidator{E34363369BD8FA1C7ABFBB9F645FE989488597C8 LH:0, LR:0, LS:0}"
I[10-12|17:57:43.744] Signed proposal module=consensus height=1 round=0 proposal="Proposal{1/0 1:6327C7C282F2 (-1,:0:000000000000) C1E264591C19 @ 2018-10-13T00:57:43.731242402Z}"
I[10-12|17:57:43.749] Received proposal module=consensus proposal="Proposal{1/0 1:6327C7C282F2 (-1,:0:000000000000) C1E264591C19 @ 2018-10-13T00:57:43.731242402Z}"
I[10-12|17:57:43.753] Received complete proposal block module=consensus height=1 hash=A4AB5053A2AD3BEA28C15BFCE12E84718110E76D
I[10-12|17:57:43.753] enterPrevote(1/0). Current: 1/0/RoundStepPropose module=consensus
I[10-12|17:57:43.753] enterPrevote: ProposalBlock is valid module=consensus height=1 round=0
I[10-12|17:57:43.757] Signed and pushed vote module=consensus height=1 round=0 vote="Vote{0:E34363369BD8 1/00/1(Prevote) A4AB5053A2AD 1682E1A2BAB5 @ 2018-10-13T00:57:44.730889298Z}" err=null
I[10-12|17:57:43.768] Added to prevote module=consensus vote="Vote{0:E34363369BD8 1/00/1(Prevote) A4AB5053A2AD 1682E1A2BAB5 @ 2018-10-13T00:57:44.730889298Z}" prevotes="VoteSet{H:1 R:0 T:1 +2/3:A4AB5053A2AD3BEA28C15BFCE12E84718110E76D:1:6327C7C282F2(1) BA{1:x} map[]}"
I[10-12|17:57:43.768] enterPrecommit(1/0). Current: 1/0/RoundStepPrevote module=consensus height=1 round=0
I[10-12|17:57:43.768] enterPrecommit: +2/3 prevoted proposal block. Locking module=consensus height=1 round=0 hash=A4AB5053A2AD3BEA28C15BFCE12E84718110E76D
I[10-12|17:57:43.784] Signed and pushed vote module=consensus height=1 round=0 vote="Vote{0:E34363369BD8 1/00/2(Precommit) A4AB5053A2AD 1B47FBB80B3E @ 2018-10-13T00:57:44.730889298Z}" err=null
I[10-12|17:57:43.791] Added to precommit module=consensus vote="Vote{0:E34363369BD8 1/00/2(Precommit) A4AB5053A2AD 1B47FBB80B3E @ 2018-10-13T00:57:44.730889298Z}" precommits="VoteSet{H:1 R:0 T:2 +2/3:A4AB5053A2AD3BEA28C15BFCE12E84718110E76D:1:6327C7C282F2(1) BA{1:x} map[]}"
I[10-12|17:57:43.791] enterCommit(1/0). Current: 1/0/RoundStepPrecommit module=consensus height=1 commitRound=0
I[10-12|17:57:43.791] Commit is for locked block. Set ProposalBlock=LockedBlock module=consensus height=1 commitRound=0 blockHash=A4AB5053A2AD3BEA28C15BFCE12E84718110E76D
I[10-12|17:57:43.793] Finalizing commit of block with 0 txs module=consensus height=1 hash=A4AB5053A2AD3BEA28C15BFCE12E84718110E76D root=
I[10-12|17:57:43.793] Block{
Header{
ChainID: test
Height: 1
Time: 2018-10-13 00:57:43.730889298 +0000 UTC
NumTxs: 0
TotalTxs: 0
LastBlockID: :0:000000000000
LastCommit:
Data:
Validators: CFA4E37BE764E9EA3B33ACDCAC713240A6F2F0D8
NextValidators:
App:
Consensus: 0E520AF30D47BE28F293E040E418D0361BFB5370
Results:
Evidence:
Proposer: E34363369BD8FA1C7ABFBB9F645FE989488597C8
}#A4AB5053A2AD3BEA28C15BFCE12E84718110E76D
Data{
}#
EvidenceData{
}#
Commit{
BlockID: :0:000000000000
Precommits:
}#
}#A4AB5053A2AD3BEA28C15BFCE12E84718110E76D module=consensus
I[10-12|17:57:43.806] Executed block module=state height=1 validTxs=0 invalidTxs=0
E[10-12|17:57:43.822] CONSENSUS FAILURE!!! module=consensus err="interface conversion: interface {} is nil, not *types.Validator" stack="goroutine 92 [running]:\nruntime/debug.Stack(0xc4209b6768, 0xdb27c0, 0xc4201fea80)\n\t/usr/local/go/src/runtime/debug/stack.go:24 +0xa7\ngithub.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).receiveRoutine.func2(0xc420134900, 0x1042458)\n\t/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:585 +0x57\npanic(0xdb27c0, 0xc4201fea80)\n\t/usr/local/go/src/runtime/panic.go:502 +0x229\ngithub.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/types.(*ValidatorSet).IncrementAccum(0xc4209d84e0, 0x1)\n\t/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/types/validator_set.go:77 +0x3a3\ngithub.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/state.updateState(0xc420207670, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)\n\t/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/state/execution.go:355 +0xae\ngithub.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/state.(*BlockExecutor).ApplyBlock(0xc4200a7260, 0xc420207670, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)\n\t/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/state/execution.go:90 +0x212\ngithub.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).finalizeCommit(0xc420134900, 0x1)\n\t/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:1334 +0xa7f\ngithub.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).tryFinalizeCommit(0xc420134900, 0x1)\n\t/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:1265 +0x462\ngithub.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).enterCommit.func1(0xc420134900, 0x0, 0x1)\n\t/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:1213 +0x98\ngithub.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).enterCommit(0xc420134900, 0x1, 0x0)\n\t/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:1242 +0x72c\ngithub.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).addVote(0xc420134900, 0xc4202352c0, 0x0, 0x0, 0xc420a87a78, 0x4c015d, 0xc4209fa000)\n\t/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:1649 +0xb7f\ngithub.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).tryAddVote(0xc420134900, 0xc4202352c0, 0x0, 0x0, 0x0, 0x42b289, 0xc420a87b30)\n\t/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:1507 +0x59\ngithub.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).handleMsg(0xc420134900, 0xd91140, 0xc42000e850, 0x0, 0x0)\n\t/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:659 +0x667\ngithub.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).receiveRoutine(0xc420134900, 0x0)\n\t/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:620 +0x587\ncreated by github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).OnStart\n\t/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:310 +0x140\n"
I[10-12|17:57:43.823] Stopping baseWAL module=consensus wal=GAIAD/data/cs.wal/wal impl=baseWAL
I[10-12|17:57:46.731] Timed out module=consensus dur=3s height=1 round=0 step=RoundStepPropose
I[10-12|17:58:08.776] Ensure peers module=p2p numOutPeers=0 numInPeers=0 numDialing=0 numToDial=10
I[10-12|17:58:08.776] No addresses to dial nor connected peers. Falling back to seeds module=p2p
A prettier log excerpt follows:
interface conversion: interface {} is nil, not *types.Validator" stack="goroutine 92 [running]:
runtime/debug.Stack(0xc4209b6768, 0xdb27c0, 0xc4201fea80)
/usr/local/go/src/runtime/debug/stack.go:24 +0xa7
github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).receiveRoutine.func2(0xc420134900, 0x1042458)
/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:585 +0x57
panic(0xdb27c0, 0xc4201fea80)
/usr/local/go/src/runtime/panic.go:502 +0x229
github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/types.(*ValidatorSet).IncrementAccum(0xc4209d84e0, 0x1)
/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/types/validator_set.go:77 +0x3a3
github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/state.updateState(0xc420207670, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/state/execution.go:355 +0xae
github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/state.(*BlockExecutor).ApplyBlock(0xc4200a7260, 0xc420207670, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/state/execution.go:90 +0x212
github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).finalizeCommit(0xc420134900, 0x1)
/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:1334 +0xa7f
github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).tryFinalizeCommit(0xc420134900, 0x1)
/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:1265 +0x462
github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).enterCommit.func1(0xc420134900, 0x0, 0x1)
/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:1213 +0x98
github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).enterCommit(0xc420134900, 0x1, 0x0)
/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:1242 +0x72c
github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).addVote(0xc420134900, 0xc4202352c0, 0x0, 0x0, 0xc420a87a78, 0x4c015d, 0xc4209fa000)
/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:1649 +0xb7f
github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).tryAddVote(0xc420134900, 0xc4202352c0, 0x0, 0x0, 0x0, 0x42b289, 0xc420a87b30)
/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:1507 +0x59
github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).handleMsg(0xc420134900, 0xd91140, 0xc42000e850, 0x0, 0x0)
/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:659 +0x667
github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).receiveRoutine(0xc420134900, 0x0)
/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:620 +0x587
created by github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus.(*ConsensusState).OnStart
/home/alessio/work/tendermint/src/github.com/cosmos/cosmos-sdk/vendor/github.com/tendermint/tendermint/consensus/state.go:310 +0x140
Attaching genesis.json:
{
"genesis_time": "2018-10-14T00:32:45.468735885Z",
"chain_id": "test",
"consensus_params": {
"block_size_params": {
"max_bytes": "22020096",
"max_gas": "-1"
},
"evidence_params": {
"max_age": "100000"
}
},
"app_hash": "",
"app_state": {
"accounts": [
{
"address": "cosmos1l09hltljujjcuccjhkqx3rt2slhd2rmelh5yk2",
"coins": [
{
"denom": "steak",
"amount": "10"
}
]
}
],
"txs": [
{
"type": "auth/StdTx",
"value": {
"msg": [
{
"type": "cosmos-sdk/MsgCreateValidator",
"value": {
"Description": {
"moniker": "alessio",
"identity": "",
"website": "",
"details": ""
},
"Commission": {
"rate": "10000000000",
"max_rate": "20000000000",
"max_change_rate": "10000000000"
},
"delegator_address": "cosmos1l09hltljujjcuccjhkqx3rt2slhd2rmelh5yk2",
"validator_address": "cosmosvaloper1l09hltljujjcuccjhkqx3rt2slhd2rme6rq36e",
"pubkey": {
"type": "tendermint/PubKeyEd25519",
"value": "56fQQcvKq1dvCmUnHhsWdmEmDAvAFHBtVwhvPKlmCBM="
},
"delegation": {
"denom": "steak",
"amount": "10"
}
}
}
],
"fee": {
"amount": [
{
"denom": "",
"amount": "0"
}
],
"gas": "200000"
},
"signatures": [
{
"pub_key": {
"type": "tendermint/PubKeySecp256k1",
"value": "Aiy9vNFVJ0zf53d4pg9Ph2/UhvX65IqvZkg7ZcAlsaCq"
},
"signature": "dIPllripZpPlUpRA698p8/6/3kc/GlEwbAgMSzlRBTl8IYTfbDm7Ftb9O5JHmOSknrjnSxjKtBjNxEs5un17Ew==",
"account_number": "0",
"sequence": "0"
}
],
"memo": "c17bc9032a673d89187290cfa88a398d858c857e@127.0.0.1:26656"
}
}
],
"stake": {
"pool": {
"loose_tokens": "100000000000",
"bonded_tokens": "0",
"inflation_last_time": "1970-01-01T00:00:00Z",
"inflation": "700000000",
"date_last_commission_reset": "0",
"prev_bonded_shares": "0"
},
"params": {
"inflation_rate_change": "1300000000",
"inflation_max": "2000000000",
"inflation_min": "700000000",
"goal_bonded": "6700000000",
"unbonding_time": "259200000000000",
"max_validators": 100,
"bond_denom": "steak"
},
"validators": null,
"bonds": null
},
"gov": {
"starting_proposalID": "1",
"deposit_period": {
"min_deposit": [
{
"denom": "steak",
"amount": "10"
}
],
"max_deposit_period": "172800000000000"
},
"voting_period": {
"voting_period": "172800000000000"
},
"tallying_procedure": {
"threshold": "5000000000",
"veto": "3340000000",
"governance_penalty": "100000000"
}
}
}
}