F170509 20:02:12.165939 136 storage/replica.go:1353 [n5,s5,r2/5:/System/{-tsd}] on-disk and in-memory state diverged:
[Lease.Start.WallTime: 1494353965502260882 != 0 Lease.Expiration.WallTime: 1494360140105540031 != 0 Lease.Replica.NodeID: 3 != 0 Lease.Replica.StoreID: 3 != 0 Lease.Replica.ReplicaID: 3 != 0 Lease.DeprecatedStartStasis.WallTime: 1494360140105540031 != 0 Lease.ProposedTS: &hlc.Timestamp{WallTime:1494360131105562632, Logical:0} != nil]
goroutine 136 [running]:
github.com/cockroachdb/cockroach/pkg/util/log.getStacks(0x1, 0x9e40b1c, 0x2c18620, 0x10)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:858 +0xa7
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).outputLogEntry(0x2c19e00, 0xc400000004, 0x24e206d, 0x12, 0x549, 0xc43631e000, 0x191)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:726 +0x56d
github.com/cockroachdb/cockroach/pkg/util/log.addStructured(0x7f19ea15bd38, 0xc420cc2a50, 0x7f1900000004, 0x2, 0x1b4c4ef, 0x28, 0xc432ebb8a0, 0x1, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/structured.go:158 +0x4b3
github.com/cockroachdb/cockroach/pkg/util/log.logDepth(0x7f19ea15bd38, 0xc420cc2a50, 0x1, 0xc400000004, 0x1b4c4ef, 0x28, 0xc432ebb8a0, 0x1, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:68 +0x8c
github.com/cockroachdb/cockroach/pkg/util/log.Fatalf(0x7f19ea15bd38, 0xc420cc2a50, 0x1b4c4ef, 0x28, 0xc432ebb8a0, 0x1, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:162 +0x7e
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).assertStateRLocked(0xc420cd6000, 0x27e7060, 0xc4203f60b0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1353 +0x40a
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).assertState(0xc420cd6000, 0x27e7060, 0xc4203f60b0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1338 +0x8c
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleEvalResult(0xc420cd6000, 0x7f19ea15bd38, 0xc420cc2a50, 0x0, 0xc42d6ca610, 0xe, 0x10, 0xc42d6ca620, 0xf, 0x10, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_proposal.go:790 +0x113
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).processRaftCommand(0xc420cd6000, 0x7f19ea15bd38, 0xc420cc2a50, 0xc42d6ca5f0, 0x8, 0x29ba8b, 0x300000003, 0x3, 0x14bd0303a2756692, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3711 +0x837
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc420cd6000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2878 +0xd70
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaftRequest(0xc420cbc000, 0x7f19ea15bd38, 0xc4330c6390, 0xc438324480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3110 +0xcc5
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue(0xc420cbc000, 0x2)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3389 +0x172
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420183ef0, 0xc4206606e0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:228 +0x24b
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f19ea15bd38, 0xc421c36300)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216984d0, 0xc4206606e0, 0xc421c29340)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xf7
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xad
We were calling
handleEvalResulton a replica that had zero Lease state, but nonzero on-disk Lease state.Stack trace: