Skip to content

multiraft/multiraft.go:648#handleWriteResponse: panic (close of closed channel) #281

@tbg

Description

@tbg

during build of #280, so possibly an error on that branch only, but it seems unlikely to me.
Full log here

....
I0123 08:24:20.247870     151 raft.go:546] raft: 1 no leader at term 5; dropping proposal
I0123 08:24:20.247934     151 raft.go:546] raft: 1 no leader at term 5; dropping proposal
I0123 08:24:20.249467     151 raft.go:546] raft: 1 no leader at term 5; dropping proposal
I0123 08:24:20.252873     151 raft.go:546] raft: 1 no leader at term 5; dropping proposal
I0123 08:24:20.252895     151 raft.go:546] raft: 1 no leader at term 5; dropping proposal
I0123 08:24:20.257519     151 raft.go:438] raft: 1 is starting a new election at term 5
I0123 08:24:20.257589     151 raft.go:377] raft: 1 became candidate at term 6
I0123 08:24:20.257617     151 raft.go:421] raft: 1 received vote from 1 at term 6
I0123 08:24:20.257647     151 raft.go:400] raft: 1 became leader at term 6
panic: close of closed channel

goroutine 118 [running]:
github.com/cockroachdb/cockroach/multiraft.(*state).handleWriteResponse(0xc20813dec0, 0xc208038c08, 0xc208394540)
    /go/src/github.com/cockroachdb/cockroach/multiraft/multiraft.go:648 +0x681
github.com/cockroachdb/cockroach/multiraft.(*state).start(0xc20813dec0)
    /go/src/github.com/cockroachdb/cockroach/multiraft/multiraft.go:413 +0x1b73
created by github.com/cockroachdb/cockroach/multiraft.(*MultiRaft).Start
    /go/src/github.com/cockroachdb/cockroach/multiraft/multiraft.go:143 +0x52

goroutine 1 [chan receive]:
testing.RunTests(0xea4568, 0x1314ca0, 0x2a, 0x2a, 0xc20802bb01)
    /usr/src/go/src/testing/testing.go:556 +0xad6
testing.(*M).Run(0xc20803ac30, 0x133c9a0)
    /usr/src/go/src/testing/testing.go:485 +0x6c
main.main()
    github.com/cockroachdb/cockroach/kv/_test/_testmain.go:138 +0x1d5

goroutine 5 [chan receive]:
github.com/golang/glog.(*loggingT).flushDaemon(0x1332ca0)
    /go/src/github.com/golang/glog/glog.go:879 +0x78
created by github.com/golang/glog.init·1
    /go/src/github.com/golang/glog/glog.go:410 +0x2a7

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
    /usr/src/go/src/runtime/asm_amd64.s:2232 +0x1

goroutine 8 [syscall]:
os/signal.loop()
    /usr/src/go/src/os/signal/signal_unix.go:21 +0x1f
created by os/signal.init·1
    /usr/src/go/src/os/signal/signal_unix.go:27 +0x35

goroutine 111 [select]:
github.com/coreos/etcd/raft.(*multiNode).run(0xc208180600)
    /go/src/github.com/cockroachdb/cockroach/_vendor/src/github.com/coreos/etcd/raft/multinode.go:140 +0x1348
created by github.com/coreos/etcd/raft.StartMultiNode
    /go/src/github.com/cockroachdb/cockroach/_vendor/src/github.com/coreos/etcd/raft/multinode.go:28 +0xdf

goroutine 113 [select]:
github.com/cockroachdb/cockroach/multiraft.(*state).start(0xc20813d1a0)
    /go/src/github.com/cockroachdb/cockroach/multiraft/multiraft.go:351 +0x20db
created by github.com/cockroachdb/cockroach/multiraft.(*MultiRaft).Start
    /go/src/github.com/cockroachdb/cockroach/multiraft/multiraft.go:143 +0x52

goroutine 119 [runnable]:
github.com/cockroachdb/cockroach/storage.(*singleNodeRaft).run(0xc20813ddd0)
    /go/src/github.com/cockroachdb/cockroach/storage/raft.go:151 +0x3d6
created by github.com/cockroachdb/cockroach/storage.newSingleNodeRaft
    /go/src/github.com/cockroachdb/cockroach/storage/raft.go:91 +0x3f7

goroutine 14 [chan receive]:
github.com/cockroachdb/cockroach/gossip.func·014()
    /go/src/github.com/cockroachdb/cockroach/gossip/server.go:140 +0x73
created by github.com/cockroachdb/cockroach/gossip.(*server).start
    /go/src/github.com/cockroachdb/cockroach/gossip/server.go:145 +0x28c

goroutine 120 [select]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft(0xc208058400, 0x7f7cb3726f90, 0xc20813ddd0, 0xc2081803c0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:901 +0x331
created by github.com/cockroachdb/cockroach/storage.(*Store).Start
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:305 +0x553

goroutine 16 [semacquire]:
sync.(*Mutex).Lock(0xc208080908)
    /usr/src/go/src/sync/mutex.go:66 +0xd3
github.com/cockroachdb/cockroach/gossip.(*Gossip).manage(0xc208011ce0)
    /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:397 +0xff
created by github.com/cockroachdb/cockroach/gossip.(*Gossip).Start
    /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:265 +0x68

goroutine 18 [sleep]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).maybeWarnAboutInit(0xc208011ce0)
    /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:470 +0x35
created by github.com/cockroachdb/cockroach/gossip.(*Gossip).Start
    /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:266 +0x82

goroutine 132 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc20826aa00, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831a780, 0x7f7cb3727d58, 0xc208114000, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc20826aa00, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831a780, 0x7f7cb3727d58, 0xc208114000, 0xc20811e201, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc208099680, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd4b480, 0x3, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2083dd178, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831a780, 0x7f7cb3727d58, 0xc208114000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc2082839e0, 0x13, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc2083dd3e0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc208394390)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendOne(0xc20809e440, 0xc208394390)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:263 +0x218
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc20809e440, 0xc208394390)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:191 +0x15b
github.com/cockroachdb/cockroach/client.(*txnSender).Send(0xc208282940, 0xc208394390)
    /go/src/github.com/cockroachdb/cockroach/client/txn_sender.go:55 +0xcb
github.com/cockroachdb/cockroach/client.(*KV).Call(0xc2082c0960, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831a780, 0x7f7cb3727d58, 0xc208114000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:122 +0x32a
github.com/cockroachdb/cockroach/kv.func·014(0xc2082c0960, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:73 +0x378
github.com/cockroachdb/cockroach/client.func·003(0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:228 +0x69
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc2082828e0, 0x13, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2083dde00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/client.(*KV).RunTransaction(0xc20815e0a0, 0xc208282900, 0xc2083ddf60, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:260 +0x469
github.com/cockroachdb/cockroach/kv.startTestWriter(0xc20815e0a0, 0x2, 0x80, 0xc20816ede0, 0xc2081a0f10, 0xc208145620, 0xc2081455c0, 0xc208160000)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:79 +0x33d
created by github.com/cockroachdb/cockroach/kv.TestRangeSplitsWithConcurrentTxns
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:112 +0x3f9

goroutine 20 [chan receive]:
github.com/cockroachdb/cockroach/gossip.func·014()
    /go/src/github.com/cockroachdb/cockroach/gossip/server.go:140 +0x73
created by github.com/cockroachdb/cockroach/gossip.(*server).start
    /go/src/github.com/cockroachdb/cockroach/gossip/server.go:145 +0x28c

goroutine 76 [sleep]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).maybeWarnAboutInit(0xc2080a9f10)
    /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:470 +0x35
created by github.com/cockroachdb/cockroach/gossip.(*Gossip).Start
    /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:266 +0x82

goroutine 22 [semacquire]:
sync.(*Mutex).Lock(0xc208080a08)
    /usr/src/go/src/sync/mutex.go:66 +0xd3
github.com/cockroachdb/cockroach/gossip.(*Gossip).manage(0xc208011d50)
    /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:397 +0xff
created by github.com/cockroachdb/cockroach/gossip.(*Gossip).Start
    /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:265 +0x68

goroutine 23 [sleep]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).maybeWarnAboutInit(0xc208011d50)
    /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:470 +0x35
created by github.com/cockroachdb/cockroach/gossip.(*Gossip).Start
    /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:266 +0x82

goroutine 24 [chan receive]:
github.com/cockroachdb/cockroach/gossip.func·014()
    /go/src/github.com/cockroachdb/cockroach/gossip/server.go:140 +0x73
created by github.com/cockroachdb/cockroach/gossip.(*server).start
    /go/src/github.com/cockroachdb/cockroach/gossip/server.go:145 +0x28c

goroutine 116 [select]:
github.com/coreos/etcd/raft.(*multiNode).run(0xc20803f8c0)
    /go/src/github.com/cockroachdb/cockroach/_vendor/src/github.com/coreos/etcd/raft/multinode.go:140 +0x1348
created by github.com/coreos/etcd/raft.StartMultiNode
    /go/src/github.com/cockroachdb/cockroach/_vendor/src/github.com/coreos/etcd/raft/multinode.go:28 +0xdf

goroutine 26 [semacquire]:
sync.(*Mutex).Lock(0xc208080b08)
    /usr/src/go/src/sync/mutex.go:66 +0xd3
github.com/cockroachdb/cockroach/gossip.(*Gossip).manage(0xc208011dc0)
    /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:397 +0xff
created by github.com/cockroachdb/cockroach/gossip.(*Gossip).Start
    /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:265 +0x68

goroutine 27 [sleep]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).maybeWarnAboutInit(0xc208011dc0)
    /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:470 +0x35
created by github.com/cockroachdb/cockroach/gossip.(*Gossip).Start
    /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:266 +0x82

goroutine 42 [sleep]:
github.com/cockroachdb/cockroach/rpc.(*Client).startHeartbeat(0xc208046090)
    /go/src/github.com/cockroachdb/cockroach/rpc/client.go:215 +0x110
created by github.com/cockroachdb/cockroach/rpc.func·001
    /go/src/github.com/cockroachdb/cockroach/rpc/client.go:146 +0x6dd

goroutine 133 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc20826b700, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b980, 0x7f7cb3727d58, 0xc208085490, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc20826b700, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b980, 0x7f7cb3727d58, 0xc208085490, 0xc2081a1201, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc2080fc7e0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd4b480, 0x3, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc208139178, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b980, 0x7f7cb3727d58, 0xc208085490, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc2081b3840, 0x13, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc2081393e0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc2081b73b0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendOne(0xc20809e440, 0xc2081b73b0)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:263 +0x218
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc20809e440, 0xc2081b73b0)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:191 +0x15b
github.com/cockroachdb/cockroach/client.(*txnSender).Send(0xc2081472a0, 0xc2081b73b0)
    /go/src/github.com/cockroachdb/cockroach/client/txn_sender.go:55 +0xcb
github.com/cockroachdb/cockroach/client.(*KV).Call(0xc2083669b0, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b980, 0x7f7cb3727d58, 0xc208085490, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:122 +0x32a
github.com/cockroachdb/cockroach/kv.func·014(0xc2083669b0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:73 +0x378
github.com/cockroachdb/cockroach/client.func·003(0xc00000000000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:228 +0x69
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc208147240, 0x13, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc208139e00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/client.(*KV).RunTransaction(0xc20815e0a0, 0xc208147280, 0xc208139f60, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:260 +0x469
github.com/cockroachdb/cockroach/kv.startTestWriter(0xc20815e0a0, 0x3, 0x80, 0xc20816ede0, 0xc2081a0f10, 0xc208145620, 0xc2081455c0, 0xc208160000)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:79 +0x33d
created by github.com/cockroachdb/cockroach/kv.TestRangeSplitsWithConcurrentTxns
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:112 +0x3f9

goroutine 112 [IO wait]:
net.(*pollDesc).Wait(0xc20813ff70, 0x72, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc20813ff70, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).accept(0xc20813ff10, 0x0, 0x7f7cb3724108, 0xc2081a0a28)
    /usr/src/go/src/net/fd_unix.go:419 +0x40b
net.(*TCPListener).AcceptTCP(0xc208038138, 0x0, 0x0, 0x0)
    /usr/src/go/src/net/tcpsock_posix.go:234 +0x4e
net.(*TCPListener).Accept(0xc208038138, 0x0, 0x0, 0x0, 0x0)
    /usr/src/go/src/net/tcpsock_posix.go:244 +0x4c
github.com/cockroachdb/cockroach/multiraft.(*localRPCTransport).accept(0xc208167970, 0xc20809e5c0, 0x7f7cb3726ee0, 0xc208038138)
    /go/src/github.com/cockroachdb/cockroach/multiraft/transport.go:127 +0x3c
created by github.com/cockroachdb/cockroach/multiraft.(*localRPCTransport).Listen
    /go/src/github.com/cockroachdb/cockroach/multiraft/transport.go:121 +0x3cd

goroutine 114 [select]:
github.com/cockroachdb/cockroach/storage.(*singleNodeRaft).run(0xc20813d0b0)
    /go/src/github.com/cockroachdb/cockroach/storage/raft.go:151 +0x3d6
created by github.com/cockroachdb/cockroach/storage.newSingleNodeRaft
    /go/src/github.com/cockroachdb/cockroach/storage/raft.go:91 +0x3f7

goroutine 34 [IO wait]:
net.(*pollDesc).Wait(0xc208010220, 0x72, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc208010220, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc2080101c0, 0xc208166cc2, 0x1, 0x1, 0x0, 0x7f7cb3724108, 0xc208166cc8)
    /usr/src/go/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc208038020, 0xc208166cc2, 0x1, 0x1, 0x0, 0x0, 0x0)
    /usr/src/go/src/net/net.go:121 +0xdc
github.com/cockroachdb/cockroach/rpc/codec.read(0x7f7cb3725b70, 0xc208038020, 0xc208166cc2, 0x1, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:93 +0xc1
github.com/cockroachdb/cockroach/rpc/codec.readByte(0x7f7cb3725b70, 0xc208038020, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:106 +0x93
github.com/cockroachdb/cockroach/rpc/codec.readUvarint(0x7f7cb3725b70, 0xc208038020, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:57 +0x6b
github.com/cockroachdb/cockroach/rpc/codec.recvFrame(0x7f7cb3725b70, 0xc208038020, 0x0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:38 +0x5f
github.com/cockroachdb/cockroach/rpc/codec.readResponseHeader(0x7f7cb3725b70, 0xc208038020, 0xc208081140, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/wire.go:170 +0x50
github.com/cockroachdb/cockroach/rpc/codec.(*clientCodec).ReadResponseHeader(0xc208056400, 0xc20809b4a0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/client.go:70 +0x88
net/rpc.(*Client).input(0xc20803ea80)
    /usr/src/go/src/net/rpc/client.go:109 +0xb6
created by net/rpc.NewClientWithCodec
    /usr/src/go/src/net/rpc/client.go:201 +0xd0

goroutine 35 [IO wait]:
net.(*pollDesc).Wait(0xc208011090, 0x72, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc208011090, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc208011030, 0xc208166c72, 0x1, 0x1, 0x0, 0x7f7cb3724108, 0xc208166c78)
    /usr/src/go/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc208038068, 0xc208166c72, 0x1, 0x1, 0x0, 0x0, 0x0)
    /usr/src/go/src/net/net.go:121 +0xdc
github.com/cockroachdb/cockroach/rpc/codec.read(0x7f7cb3725b70, 0xc208038068, 0xc208166c72, 0x1, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:93 +0xc1
github.com/cockroachdb/cockroach/rpc/codec.readByte(0x7f7cb3725b70, 0xc208038068, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:106 +0x93
github.com/cockroachdb/cockroach/rpc/codec.readUvarint(0x7f7cb3725b70, 0xc208038068, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:57 +0x6b
github.com/cockroachdb/cockroach/rpc/codec.recvFrame(0x7f7cb3725b70, 0xc208038068, 0x0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:38 +0x5f
github.com/cockroachdb/cockroach/rpc/codec.readResponseHeader(0x7f7cb3725b70, 0xc208038068, 0xc208080e00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/wire.go:170 +0x50
github.com/cockroachdb/cockroach/rpc/codec.(*clientCodec).ReadResponseHeader(0xc208056700, 0xc20809b4d0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/client.go:70 +0x88
net/rpc.(*Client).input(0xc20803eba0)
    /usr/src/go/src/net/rpc/client.go:109 +0xb6
created by net/rpc.NewClientWithCodec
    /usr/src/go/src/net/rpc/client.go:201 +0xd0

goroutine 36 [IO wait]:
net.(*pollDesc).Wait(0xc2080111e0, 0x72, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc2080111e0, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc208011180, 0xc2080a00b5, 0x1, 0x1, 0x0, 0x7f7cb3724108, 0xc2080a00b8)
    /usr/src/go/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc208038070, 0xc2080a00b5, 0x1, 0x1, 0x0, 0x0, 0x0)
    /usr/src/go/src/net/net.go:121 +0xdc
github.com/cockroachdb/cockroach/rpc/codec.read(0x7f7cb3725b70, 0xc208038070, 0xc2080a00b5, 0x1, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:93 +0xc1
github.com/cockroachdb/cockroach/rpc/codec.readByte(0x7f7cb3725b70, 0xc208038070, 0x739cba, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:106 +0x93
github.com/cockroachdb/cockroach/rpc/codec.readUvarint(0x7f7cb3725b70, 0xc208038070, 0x40, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:57 +0x6b
github.com/cockroachdb/cockroach/rpc/codec.recvFrame(0x7f7cb3725b70, 0xc208038070, 0x0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:38 +0x5f
github.com/cockroachdb/cockroach/rpc/codec.readRequestHeader(0x7f7cb3725b70, 0xc208038070, 0xc208185900, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/wire.go:67 +0x4a
github.com/cockroachdb/cockroach/rpc/codec.(*serverCodec).ReadRequestHeader(0xc2080466c0, 0xc20801f220, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/server.go:49 +0x88
net/rpc.(*Server).readRequestHeader(0xc208080780, 0x7f7cb3725d20, 0xc2080466c0, 0x0, 0x0, 0xc20801f220, 0x501000, 0x0, 0x0)
    /usr/src/go/src/net/rpc/server.go:575 +0x97
net/rpc.(*Server).readRequest(0xc208080780, 0x7f7cb3725d20, 0xc2080466c0, 0xc208056900, 0xc208108260, 0xcf4e60, 0x0, 0x0, 0x0, 0x0, ...)
    /usr/src/go/src/net/rpc/server.go:542 +0xa2
net/rpc.(*Server).ServeCodec(0xc208080780, 0x7f7cb3725d20, 0xc2080466c0)
    /usr/src/go/src/net/rpc/server.go:461 +0x6d
github.com/cockroachdb/cockroach/rpc.(*Server).serveConn(0xc208011a40, 0x7f7cb3725ad8, 0xc208038070)
    /go/src/github.com/cockroachdb/cockroach/rpc/server.go:173 +0x24b
created by github.com/cockroachdb/cockroach/rpc.func·002
    /go/src/github.com/cockroachdb/cockroach/rpc/server.go:105 +0x3ff

goroutine 38 [IO wait]:
net.(*pollDesc).Wait(0xc208011950, 0x72, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc208011950, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc2080118f0, 0xc2081a00e5, 0x1, 0x1, 0x0, 0x7f7cb3724108, 0xc2081a00e8)
    /usr/src/go/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc208038078, 0xc2081a00e5, 0x1, 0x1, 0x0, 0x0, 0x0)
    /usr/src/go/src/net/net.go:121 +0xdc
github.com/cockroachdb/cockroach/rpc/codec.read(0x7f7cb3725b70, 0xc208038078, 0xc2081a00e5, 0x1, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:93 +0xc1
github.com/cockroachdb/cockroach/rpc/codec.readByte(0x7f7cb3725b70, 0xc208038078, 0x739cba, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:106 +0x93
github.com/cockroachdb/cockroach/rpc/codec.readUvarint(0x7f7cb3725b70, 0xc208038078, 0x40, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:57 +0x6b
github.com/cockroachdb/cockroach/rpc/codec.recvFrame(0x7f7cb3725b70, 0xc208038078, 0x0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:38 +0x5f
github.com/cockroachdb/cockroach/rpc/codec.readRequestHeader(0x7f7cb3725b70, 0xc208038078, 0xc20809ff00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/wire.go:67 +0x4a
github.com/cockroachdb/cockroach/rpc/codec.(*serverCodec).ReadRequestHeader(0xc2080467e0, 0xc20801f5e0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/server.go:49 +0x88
net/rpc.(*Server).readRequestHeader(0xc208080800, 0x7f7cb3725d20, 0xc2080467e0, 0x0, 0x0, 0xc20801f5e0, 0x501000, 0x0, 0x0)
    /usr/src/go/src/net/rpc/server.go:575 +0x97
net/rpc.(*Server).readRequest(0xc208080800, 0x7f7cb3725d20, 0xc2080467e0, 0xc208056980, 0xc20801f680, 0xcf4e60, 0x0, 0x0, 0x0, 0x0, ...)
    /usr/src/go/src/net/rpc/server.go:542 +0xa2
net/rpc.(*Server).ServeCodec(0xc208080800, 0x7f7cb3725d20, 0xc2080467e0)
    /usr/src/go/src/net/rpc/server.go:461 +0x6d
github.com/cockroachdb/cockroach/rpc.(*Server).serveConn(0xc208011b20, 0x7f7cb3725ad8, 0xc208038078)
    /go/src/github.com/cockroachdb/cockroach/rpc/server.go:173 +0x24b
created by github.com/cockroachdb/cockroach/rpc.func·002
    /go/src/github.com/cockroachdb/cockroach/rpc/server.go:105 +0x3ff

goroutine 40 [sleep]:
github.com/cockroachdb/cockroach/rpc.(*Client).startHeartbeat(0xc208046000)
    /go/src/github.com/cockroachdb/cockroach/rpc/client.go:215 +0x110
created by github.com/cockroachdb/cockroach/rpc.func·001
    /go/src/github.com/cockroachdb/cockroach/rpc/client.go:146 +0x6dd

goroutine 117 [IO wait]:
net.(*pollDesc).Wait(0xc20816baa0, 0x72, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc20816baa0, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).accept(0xc20816ba40, 0x0, 0x7f7cb3724108, 0xc2081a05a0)
    /usr/src/go/src/net/fd_unix.go:419 +0x40b
net.(*TCPListener).AcceptTCP(0xc208038250, 0xc20801b6d0, 0x0, 0x0)
    /usr/src/go/src/net/tcpsock_posix.go:234 +0x4e
net.(*TCPListener).Accept(0xc208038250, 0x0, 0x0, 0x0, 0x0)
    /usr/src/go/src/net/tcpsock_posix.go:244 +0x4c
github.com/cockroachdb/cockroach/multiraft.(*localRPCTransport).accept(0xc2081a0510, 0xc20809ef00, 0x7f7cb3726ee0, 0xc208038250)
    /go/src/github.com/cockroachdb/cockroach/multiraft/transport.go:127 +0x3c
created by github.com/cockroachdb/cockroach/multiraft.(*localRPCTransport).Listen
    /go/src/github.com/cockroachdb/cockroach/multiraft/transport.go:121 +0x3cd

goroutine 59 [IO wait]:
net.(*pollDesc).Wait(0xc20818df00, 0x72, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc20818df00, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc20818dea0, 0xc20802bfd5, 0x1, 0x1, 0x0, 0x7f7cb3724108, 0xc20802bfd8)
    /usr/src/go/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc208038ca8, 0xc20802bfd5, 0x1, 0x1, 0x0, 0x0, 0x0)
    /usr/src/go/src/net/net.go:121 +0xdc
github.com/cockroachdb/cockroach/rpc/codec.read(0x7f7cb3725b70, 0xc208038ca8, 0xc20802bfd5, 0x1, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:93 +0xc1
github.com/cockroachdb/cockroach/rpc/codec.readByte(0x7f7cb3725b70, 0xc208038ca8, 0x739cba, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:106 +0x93
github.com/cockroachdb/cockroach/rpc/codec.readUvarint(0x7f7cb3725b70, 0xc208038ca8, 0x40, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:57 +0x6b
github.com/cockroachdb/cockroach/rpc/codec.recvFrame(0x7f7cb3725b70, 0xc208038ca8, 0x0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:38 +0x5f
github.com/cockroachdb/cockroach/rpc/codec.readRequestHeader(0x7f7cb3725b70, 0xc208038ca8, 0xc20809fc80, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/wire.go:67 +0x4a
github.com/cockroachdb/cockroach/rpc/codec.(*serverCodec).ReadRequestHeader(0xc208046a20, 0xc208108600, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/server.go:49 +0x88
net/rpc.(*Server).readRequestHeader(0xc208080880, 0x7f7cb3725d20, 0xc208046a20, 0x0, 0x0, 0xc208108600, 0x0, 0x0, 0x0)
    /usr/src/go/src/net/rpc/server.go:575 +0x97
net/rpc.(*Server).readRequest(0xc208080880, 0x7f7cb3725d20, 0xc208046a20, 0xc208056a00, 0xc208108560, 0xcf4e60, 0x0, 0x0, 0x0, 0x0, ...)
    /usr/src/go/src/net/rpc/server.go:542 +0xa2
net/rpc.(*Server).ServeCodec(0xc208080880, 0x7f7cb3725d20, 0xc208046a20)
    /usr/src/go/src/net/rpc/server.go:461 +0x6d
github.com/cockroachdb/cockroach/rpc.(*Server).serveConn(0xc208011c00, 0x7f7cb3725ad8, 0xc208038ca8)
    /go/src/github.com/cockroachdb/cockroach/rpc/server.go:173 +0x24b
created by github.com/cockroachdb/cockroach/rpc.func·002
    /go/src/github.com/cockroachdb/cockroach/rpc/server.go:105 +0x3ff

goroutine 75 [semacquire]:
sync.(*Mutex).Lock(0xc2080812c8)
    /usr/src/go/src/sync/mutex.go:66 +0xd3
github.com/cockroachdb/cockroach/gossip.(*Gossip).manage(0xc2080a9f10)
    /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:413 +0x699
created by github.com/cockroachdb/cockroach/gossip.(*Gossip).Start
    /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:265 +0x68

goroutine 58 [IO wait]:
net.(*pollDesc).Wait(0xc20818de20, 0x72, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc20818de20, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc20818ddc0, 0xc208166c8a, 0x1, 0x1, 0x0, 0x7f7cb3724108, 0xc208166ca0)
    /usr/src/go/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc208038ca0, 0xc208166c8a, 0x1, 0x1, 0x0, 0x0, 0x0)
    /usr/src/go/src/net/net.go:121 +0xdc
github.com/cockroachdb/cockroach/rpc/codec.read(0x7f7cb3725b70, 0xc208038ca0, 0xc208166c8a, 0x1, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:93 +0xc1
github.com/cockroachdb/cockroach/rpc/codec.readByte(0x7f7cb3725b70, 0xc208038ca0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:106 +0x93
github.com/cockroachdb/cockroach/rpc/codec.readUvarint(0x7f7cb3725b70, 0xc208038ca0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:57 +0x6b
github.com/cockroachdb/cockroach/rpc/codec.recvFrame(0x7f7cb3725b70, 0xc208038ca0, 0x0, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/conn.go:38 +0x5f
github.com/cockroachdb/cockroach/rpc/codec.readResponseHeader(0x7f7cb3725b70, 0xc208038ca0, 0xc208080fc0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/wire.go:170 +0x50
github.com/cockroachdb/cockroach/rpc/codec.(*clientCodec).ReadResponseHeader(0xc208056800, 0xc2080a4d50, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/rpc/codec/client.go:70 +0x88
net/rpc.(*Client).input(0xc208180960)
    /usr/src/go/src/net/rpc/client.go:109 +0xb6
created by net/rpc.NewClientWithCodec
    /usr/src/go/src/net/rpc/client.go:201 +0xd0

goroutine 62 [sleep]:
github.com/cockroachdb/cockroach/rpc.(*Client).startHeartbeat(0xc208046900)
    /go/src/github.com/cockroachdb/cockroach/rpc/client.go:215 +0x110
created by github.com/cockroachdb/cockroach/rpc.func·001
    /go/src/github.com/cockroachdb/cockroach/rpc/client.go:146 +0x6dd

goroutine 127 [select]:
github.com/cockroachdb/cockroach/multiraft.(*writeTask).start(0xc20813d170)
    /go/src/github.com/cockroachdb/cockroach/multiraft/storage.go:135 +0x784
created by github.com/cockroachdb/cockroach/multiraft.(*state).start
    /go/src/github.com/cockroachdb/cockroach/multiraft/multiraft.go:319 +0x166

goroutine 73 [chan receive]:
github.com/cockroachdb/cockroach/gossip.func·014()
    /go/src/github.com/cockroachdb/cockroach/gossip/server.go:140 +0x73
created by github.com/cockroachdb/cockroach/gossip.(*server).start
    /go/src/github.com/cockroachdb/cockroach/gossip/server.go:145 +0x28c

goroutine 130 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc208057380, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b500, 0x7f7cb3727d58, 0xc208084770, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc208057380, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b500, 0x7f7cb3727d58, 0xc208084770, 0xc2081e3f01, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc208186480, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd4b480, 0x3, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2081d1178, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b500, 0x7f7cb3727d58, 0xc208084770, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc2081b2960, 0x13, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc2081d13e0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc20822ac60)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendOne(0xc20809e440, 0xc20822ac60)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:263 +0x218
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc20809e440, 0xc20822ac60)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:191 +0x15b
github.com/cockroachdb/cockroach/client.(*txnSender).Send(0xc208306980, 0xc20822ac60)
    /go/src/github.com/cockroachdb/cockroach/client/txn_sender.go:55 +0xcb
github.com/cockroachdb/cockroach/client.(*KV).Call(0xc2083379f0, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b500, 0x7f7cb3727d58, 0xc208084770, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:122 +0x32a
github.com/cockroachdb/cockroach/kv.func·014(0xc2083379f0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:73 +0x378
github.com/cockroachdb/cockroach/client.func·003(0xc00000000000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:228 +0x69
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc208306940, 0x13, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2081d1e00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/client.(*KV).RunTransaction(0xc20815e0a0, 0xc208306960, 0xc2081d1f60, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:260 +0x469
github.com/cockroachdb/cockroach/kv.startTestWriter(0xc20815e0a0, 0x0, 0x80, 0xc20816ede0, 0xc2081a0f10, 0xc208145620, 0xc2081455c0, 0xc208160000)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:79 +0x33d
created by github.com/cockroachdb/cockroach/kv.TestRangeSplitsWithConcurrentTxns
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:112 +0x3f9

goroutine 131 [sleep]:
github.com/cockroachdb/cockroach/kv.startTestWriter(0xc20815e0a0, 0x1, 0x80, 0xc20816ede0, 0xc2081a0f10, 0xc208145620, 0xc2081455c0, 0xc208160000)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:83 +0x461
created by github.com/cockroachdb/cockroach/kv.TestRangeSplitsWithConcurrentTxns
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:112 +0x3f9

goroutine 128 [select]:
github.com/cockroachdb/cockroach/multiraft.(*writeTask).start(0xc20813de90)
    /go/src/github.com/cockroachdb/cockroach/multiraft/storage.go:135 +0x784
created by github.com/cockroachdb/cockroach/multiraft.(*state).start
    /go/src/github.com/cockroachdb/cockroach/multiraft/multiraft.go:319 +0x166

goroutine 129 [IO wait]:
net.(*pollDesc).Wait(0xc20818c7d0, 0x72, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc20818c7d0, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc20818c770, 0xc2081c4000, 0x1000, 0x1000, 0x0, 0x7f7cb3724108, 0xc2081a0d90)
    /usr/src/go/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc208038378, 0xc2081c4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/src/go/src/net/net.go:121 +0xdc
bufio.(*Reader).fill(0xc208145380)
    /usr/src/go/src/bufio/bufio.go:97 +0x1ce
bufio.(*Reader).Read(0xc208145380, 0xc2081a0d70, 0x1, 0x9, 0x0, 0x0, 0x0)
    /usr/src/go/src/bufio/bufio.go:174 +0x26c
io.ReadAtLeast(0x7f7cb37273a8, 0xc208145380, 0xc2081a0d70, 0x1, 0x9, 0x1, 0x0, 0x0, 0x0)
    /usr/src/go/src/io/io.go:298 +0xf1
io.ReadFull(0x7f7cb37273a8, 0xc208145380, 0xc2081a0d70, 0x1, 0x9, 0x0, 0x0, 0x0)
    /usr/src/go/src/io/io.go:316 +0x6d
encoding/gob.decodeUintReader(0x7f7cb37273a8, 0xc208145380, 0xc2081a0d70, 0x9, 0x9, 0x0, 0x1, 0x0, 0x0)
    /usr/src/go/src/encoding/gob/decode.go:121 +0x99
encoding/gob.(*Decoder).recvMessage(0xc208057600, 0x50c7e2)
    /usr/src/go/src/encoding/gob/decoder.go:76 +0x55
encoding/gob.(*Decoder).decodeTypeSequence(0xc208057600, 0xc20821a900, 0x16)
    /usr/src/go/src/encoding/gob/decoder.go:140 +0x47
encoding/gob.(*Decoder).DecodeValue(0xc208057600, 0xb5b6e0, 0xc20821a930, 0x16, 0x0, 0x0)
    /usr/src/go/src/encoding/gob/decoder.go:208 +0x192
encoding/gob.(*Decoder).Decode(0xc208057600, 0xb5b6e0, 0xc20821a930, 0x0, 0x0)
    /usr/src/go/src/encoding/gob/decoder.go:185 +0x297
net/rpc.(*gobClientCodec).ReadResponseHeader(0xc20821a8a0, 0xc20821a930, 0x0, 0x0)
    /usr/src/go/src/net/rpc/client.go:223 +0x5e
net/rpc.(*Client).input(0xc208145440)
    /usr/src/go/src/net/rpc/client.go:109 +0xb6
created by net/rpc.NewClientWithCodec
    /usr/src/go/src/net/rpc/client.go:201 +0xd0

goroutine 126 [select]:
github.com/cockroachdb/cockroach/storage.(*Range).startGossip(0xc208057380)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:545 +0x12c
created by github.com/cockroachdb/cockroach/storage.(*Range).start
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:216 +0x98

goroutine 134 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc208057380, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b680, 0x7f7cb3727d58, 0xc208084fc0, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc208057380, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b680, 0x7f7cb3727d58, 0xc208084fc0, 0xc2081a0801, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc2081870e0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd4b480, 0x3, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2083a9178, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b680, 0x7f7cb3727d58, 0xc208084fc0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc2081b2fc0, 0x13, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc2083a93e0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc2081b60c0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendOne(0xc20809e440, 0xc2081b60c0)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:263 +0x218
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc20809e440, 0xc2081b60c0)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:191 +0x15b
github.com/cockroachdb/cockroach/client.(*txnSender).Send(0xc20829f960, 0xc2081b60c0)
    /go/src/github.com/cockroachdb/cockroach/client/txn_sender.go:55 +0xcb
github.com/cockroachdb/cockroach/client.(*KV).Call(0xc2083a04b0, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b680, 0x7f7cb3727d58, 0xc208084fc0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:122 +0x32a
github.com/cockroachdb/cockroach/kv.func·014(0xc2083a04b0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:73 +0x378
github.com/cockroachdb/cockroach/client.func·003(0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:228 +0x69
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc20829f920, 0x13, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2083a9e00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/client.(*KV).RunTransaction(0xc20815e0a0, 0xc20829f940, 0xc2083a9f60, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:260 +0x469
github.com/cockroachdb/cockroach/kv.startTestWriter(0xc20815e0a0, 0x4, 0x80, 0xc20816ede0, 0xc2081a0f10, 0xc208145620, 0xc2081455c0, 0xc208160000)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:79 +0x33d
created by github.com/cockroachdb/cockroach/kv.TestRangeSplitsWithConcurrentTxns
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:112 +0x3f9

goroutine 115 [select]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft(0xc208058400, 0x7f7cb3726f90, 0xc20813d0b0, 0xc2081803c0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:901 +0x331
created by github.com/cockroachdb/cockroach/storage.(*Store).Start
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:305 +0x553

goroutine 136 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc208323b00, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b800, 0x7f7cb3727d58, 0xc208085260, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc208323b00, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b800, 0x7f7cb3727d58, 0xc208085260, 0xc2081a0d01, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc208187320, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd4b480, 0x3, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2083e5178, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b800, 0x7f7cb3727d58, 0xc208085260, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc2081b3560, 0x13, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc2083e53e0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc2081b6b40)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendOne(0xc20809e440, 0xc2081b6b40)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:263 +0x218
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc20809e440, 0xc2081b6b40)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:191 +0x15b
github.com/cockroachdb/cockroach/client.(*txnSender).Send(0xc20816e440, 0xc2081b6b40)
    /go/src/github.com/cockroachdb/cockroach/client/txn_sender.go:55 +0xcb
github.com/cockroachdb/cockroach/client.(*KV).Call(0xc2083a08c0, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b800, 0x7f7cb3727d58, 0xc208085260, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:122 +0x32a
github.com/cockroachdb/cockroach/kv.func·014(0xc2083a08c0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:73 +0x378
github.com/cockroachdb/cockroach/client.func·003(0x5c154b, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:228 +0x69
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc20816e3c0, 0x13, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2083e5e00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/client.(*KV).RunTransaction(0xc20815e0a0, 0xc20816e3e0, 0xc2083e5f60, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:260 +0x469
github.com/cockroachdb/cockroach/kv.startTestWriter(0xc20815e0a0, 0x6, 0x80, 0xc20816ede0, 0xc2081a0f10, 0xc208145620, 0xc2081455c0, 0xc208160000)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:79 +0x33d
created by github.com/cockroachdb/cockroach/kv.TestRangeSplitsWithConcurrentTxns
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:112 +0x3f9

goroutine 135 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc208323b00, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc208156300, 0x7f7cb3727d58, 0xc2083d2540, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc208323b00, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc208156300, 0x7f7cb3727d58, 0xc2083d2540, 0xc20835c501, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc208187440, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd4b480, 0x3, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc20812f178, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc208156300, 0x7f7cb3727d58, 0xc2083d2540, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc20829f720, 0x13, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc20812f3e0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc2083a4f90)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendOne(0xc20809e440, 0xc2083a4f90)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:263 +0x218
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc20809e440, 0xc2083a4f90)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:191 +0x15b
github.com/cockroachdb/cockroach/client.(*txnSender).Send(0xc20829f660, 0xc2083a4f90)
    /go/src/github.com/cockroachdb/cockroach/client/txn_sender.go:55 +0xcb
github.com/cockroachdb/cockroach/client.(*KV).Call(0xc2083a0370, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc208156300, 0x7f7cb3727d58, 0xc2083d2540, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:122 +0x32a
github.com/cockroachdb/cockroach/kv.func·014(0xc2083a0370, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:73 +0x378
github.com/cockroachdb/cockroach/client.func·003(0xd60520, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:228 +0x69
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc20829f620, 0x13, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc20812fe00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/client.(*KV).RunTransaction(0xc20815e0a0, 0xc20829f640, 0xc20812ff60, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:260 +0x469
github.com/cockroachdb/cockroach/kv.startTestWriter(0xc20815e0a0, 0x5, 0x80, 0xc20816ede0, 0xc2081a0f10, 0xc208145620, 0xc2081455c0, 0xc208160000)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:79 +0x33d
created by github.com/cockroachdb/cockroach/kv.TestRangeSplitsWithConcurrentTxns
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:112 +0x3f9

goroutine 139 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc20826aa00, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b380, 0x7f7cb3727d58, 0xc208084540, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc20826aa00, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b380, 0x7f7cb3727d58, 0xc208084540, 0xc2081e3a01, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc208187560, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd4b480, 0x3, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2083e1178, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b380, 0x7f7cb3727d58, 0xc208084540, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc2081b2480, 0x13, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc2083e13e0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc20822a1e0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendOne(0xc20809e440, 0xc20822a1e0)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:263 +0x218
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc20809e440, 0xc20822a1e0)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:191 +0x15b
github.com/cockroachdb/cockroach/client.(*txnSender).Send(0xc208306640, 0xc20822a1e0)
    /go/src/github.com/cockroachdb/cockroach/client/txn_sender.go:55 +0xcb
github.com/cockroachdb/cockroach/client.(*KV).Call(0xc208337860, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b380, 0x7f7cb3727d58, 0xc208084540, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:122 +0x32a
github.com/cockroachdb/cockroach/kv.func·014(0xc208337860, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:73 +0x378
github.com/cockroachdb/cockroach/client.func·003(0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:228 +0x69
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc2083065e0, 0x13, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2083e1e00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/client.(*KV).RunTransaction(0xc20815e0a0, 0xc208306620, 0xc2083e1f60, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:260 +0x469
github.com/cockroachdb/cockroach/kv.startTestWriter(0xc20815e0a0, 0x9, 0x80, 0xc20816ede0, 0xc2081a0f10, 0xc208145620, 0xc2081455c0, 0xc208160000)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:79 +0x33d
created by github.com/cockroachdb/cockroach/kv.TestRangeSplitsWithConcurrentTxns
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:112 +0x3f9

goroutine 137 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc208057380, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b200, 0x7f7cb3727d58, 0xc2080a9f80, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc208057380, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b200, 0x7f7cb3727d58, 0xc2080a9f80, 0xc2081e3501, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc208187680, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd4b480, 0x3, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc208135178, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b200, 0x7f7cb3727d58, 0xc2080a9f80, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc20822dec0, 0x13, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc2081353e0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc208240d80)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendOne(0xc20809e440, 0xc208240d80)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:263 +0x218
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc20809e440, 0xc208240d80)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:191 +0x15b
github.com/cockroachdb/cockroach/client.(*txnSender).Send(0xc20819cce0, 0xc208240d80)
    /go/src/github.com/cockroachdb/cockroach/client/txn_sender.go:55 +0xcb
github.com/cockroachdb/cockroach/client.(*KV).Call(0xc20803b450, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc20831b200, 0x7f7cb3727d58, 0xc2080a9f80, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:122 +0x32a
github.com/cockroachdb/cockroach/kv.func·014(0xc20803b450, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:73 +0x378
github.com/cockroachdb/cockroach/client.func·003(0x0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:228 +0x69
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc20819cc80, 0x13, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc208135e00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/client.(*KV).RunTransaction(0xc20815e0a0, 0xc20819ccc0, 0xc208135f60, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:260 +0x469
github.com/cockroachdb/cockroach/kv.startTestWriter(0xc20815e0a0, 0x7, 0x80, 0xc20816ede0, 0xc2081a0f10, 0xc208145620, 0xc2081455c0, 0xc208160000)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:79 +0x33d
created by github.com/cockroachdb/cockroach/kv.TestRangeSplitsWithConcurrentTxns
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:112 +0x3f9

goroutine 110 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc208323b00, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc208157080, 0x7f7cb3727d58, 0xc2082a0070, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc208323b00, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc208157080, 0x7f7cb3727d58, 0xc2082a0070, 0xc2082a0101, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc2081877a0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd4b480, 0x3, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc20836fef8, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd4b480, 0x3, 0x7f7cb3727d18, 0xc208157080, 0x7f7cb3727d58, 0xc2082a0070, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc20819de00, 0x13, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc208370160, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc20819fe60)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendOne(0xc20809e440, 0xc20819fe60)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:263 +0x218
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendBatch(0xc20809e440, 0xc208292f00, 0xc208250e00)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:386 +0x5d6
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc20809e440, 0xc20819fce0)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:189 +0x13b
github.com/cockroachdb/cockroach/client.(*txnSender).Send(0xc20819dc80, 0xc20819fce0)
    /go/src/github.com/cockroachdb/cockroach/client/txn_sender.go:55 +0xcb
github.com/cockroachdb/cockroach/client.(*KV).Call(0xc20803b860, 0xd44e80, 0x5, 0x7f7cb37280c0, 0xc208292f00, 0x7f7cb3728100, 0xc208250e00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:122 +0x32a
github.com/cockroachdb/cockroach/client.(*KV).Flush(0xc20803b860, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:178 +0x569
github.com/cockroachdb/cockroach/client.(*KV).Call(0xc20803b860, 0xd6d6f0, 0xe, 0x7f7cb3728038, 0xc208292dc0, 0x7f7cb3728078, 0xc20816bf10, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:108 +0xb4
github.com/cockroachdb/cockroach/storage.func·009(0xc20803b860, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:1408 +0x4bf
github.com/cockroachdb/cockroach/client.func·003(0xc2081665c0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:228 +0x69
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc20819dc40, 0x14, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc208370ff0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/client.(*KV).RunTransaction(0xc20815e0a0, 0xc20819dc60, 0xc208371238, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:260 +0x469
github.com/cockroachdb/cockroach/storage.(*Range).AdminSplit(0xc208323b00, 0xc208292b40, 0xc20816b8f0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:1409 +0x1117
github.com/cockroachdb/cockroach/storage.(*Range).addAdminCmd(0xc208323b00, 0xd6b7f0, 0xa, 0x7f7cb3727dc8, 0xc208292b40, 0x7f7cb3727e08, 0xc20816b8f0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:331 +0x113
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc208323b00, 0xd6b7f0, 0xa, 0x7f7cb3727dc8, 0xc208292b40, 0x7f7cb3727e08, 0xc20816b8f0, 0xc2081a0f01, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:302 +0x1ff
github.com/cockroachdb/cockroach/storage.func·016(0x763cbb, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd6b7f0, 0xa, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc208371740, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd6b7f0, 0xa, 0x7f7cb3727dc8, 0xc208292b40, 0x7f7cb3727e08, 0xc20816b8f0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc20819dac0, 0x1a, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc2083719a8, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc20819f7a0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendOne(0xc20809e440, 0xc20819f7a0)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:263 +0x218
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc20809e440, 0xc20819f7a0)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:191 +0x15b
github.com/cockroachdb/cockroach/client.(*KV).Call(0xc20815e0a0, 0xd6b7f0, 0xa, 0x7f7cb3727dc8, 0xc208292b40, 0x7f7cb3727e08, 0xc20816b8f0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/client/kv.go:122 +0x32a
github.com/cockroachdb/cockroach/kv.TestRangeSplitsWithConcurrentTxns(0xc208160000)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:124 +0x79d
testing.tRunner(0xc208160000, 0x1314d48)
    /usr/src/go/src/testing/testing.go:447 +0xbf
created by testing.RunTests
    /usr/src/go/src/testing/testing.go:555 +0xa8b

goroutine 138 [sleep]:
github.com/cockroachdb/cockroach/kv.startTestWriter(0xc20815e0a0, 0x8, 0x80, 0xc20816ede0, 0xc2081a0f10, 0xc208145620, 0xc2081455c0, 0xc208160000)
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:83 +0x461
created by github.com/cockroachdb/cockroach/kv.TestRangeSplitsWithConcurrentTxns
    /go/src/github.com/cockroachdb/cockroach/kv/split_test.go:112 +0x3f9

goroutine 372 [select]:
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).heartbeat(0xc20809e440, 0xc2083aca20, 0xc2081adc20)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:501 +0x895
created by github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendOne
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:294 +0x7c3

goroutine 141 [IO wait]:
net.(*pollDesc).Wait(0xc20818cbc0, 0x72, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc20818cbc0, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc20818cb60, 0xc20800f000, 0x1000, 0x1000, 0x0, 0x7f7cb3724108, 0xc2081a07e8)
    /usr/src/go/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc2080380f8, 0xc20800f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/src/go/src/net/net.go:121 +0xdc
bufio.(*Reader).fill(0xc2081442a0)
    /usr/src/go/src/bufio/bufio.go:97 +0x1ce
bufio.(*Reader).Read(0xc2081442a0, 0xc2081a07c0, 0x1, 0x9, 0x0, 0x0, 0x0)
    /usr/src/go/src/bufio/bufio.go:174 +0x26c
io.ReadAtLeast(0x7f7cb37273a8, 0xc2081442a0, 0xc2081a07c0, 0x1, 0x9, 0x1, 0x0, 0x0, 0x0)
    /usr/src/go/src/io/io.go:298 +0xf1
io.ReadFull(0x7f7cb37273a8, 0xc2081442a0, 0xc2081a07c0, 0x1, 0x9, 0x502cf9, 0x0, 0x0)
    /usr/src/go/src/io/io.go:316 +0x6d
encoding/gob.decodeUintReader(0x7f7cb37273a8, 0xc2081442a0, 0xc2081a07c0, 0x9, 0x9, 0x0, 0x1, 0x0, 0x0)
    /usr/src/go/src/encoding/gob/decode.go:121 +0x99
encoding/gob.(*Decoder).recvMessage(0xc208056880, 0x0)
    /usr/src/go/src/encoding/gob/decoder.go:76 +0x55
encoding/gob.(*Decoder).decodeTypeSequence(0xc208056880, 0xc20816e700, 0x16)
    /usr/src/go/src/encoding/gob/decoder.go:140 +0x47
encoding/gob.(*Decoder).DecodeValue(0xc208056880, 0xb5b680, 0xc20816e780, 0x16, 0x0, 0x0)
    /usr/src/go/src/encoding/gob/decoder.go:208 +0x192
encoding/gob.(*Decoder).Decode(0xc208056880, 0xb5b680, 0xc20816e780, 0x0, 0x0)
    /usr/src/go/src/encoding/gob/decoder.go:185 +0x297
net/rpc.(*gobServerCodec).ReadRequestHeader(0xc20821ac90, 0xc20816e780, 0x0, 0x0)
    /usr/src/go/src/net/rpc/server.go:402 +0x5e
net/rpc.(*Server).readRequestHeader(0xc20809ef00, 0x7f7cb37276b0, 0xc20821ac90, 0x0, 0x0, 0xc20816e780, 0xc2081a0700, 0x0, 0x0)
    /usr/src/go/src/net/rpc/server.go:575 +0x97
net/rpc.(*Server).readRequest(0xc20809ef00, 0x7f7cb37276b0, 0xc20821ac90, 0xc207ff7e1b, 0xc208081e40, 0xc208081e40, 0x0, 0x0, 0x0, 0x0, ...)
    /usr/src/go/src/net/rpc/server.go:542 +0xa2
net/rpc.(*Server).ServeCodec(0xc20809ef00, 0x7f7cb37276b0, 0xc20821ac90)
    /usr/src/go/src/net/rpc/server.go:461 +0x6d
net/rpc.(*Server).ServeConn(0xc20809ef00, 0x7f7cb37272f8, 0xc2080380f8)
    /usr/src/go/src/net/rpc/server.go:453 +0x208
created by github.com/cockroachdb/cockroach/multiraft.(*localRPCTransport).accept
    /go/src/github.com/cockroachdb/cockroach/multiraft/transport.go:136 +0x2a4

goroutine 370 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc208322180, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c500, 0x7f7cb372c288, 0xc208084cb0, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc208322180, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c500, 0x7f7cb372c288, 0xc208084cb0, 0xc2081e3401, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc20813aa20, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd9f9d0, 0x15, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2083c3c30, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c500, 0x7f7cb372c288, 0xc208084cb0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc208118e70, 0x25, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc2083c3e98, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc20822b410)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.func·008()
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:138 +0x1f4
created by github.com/cockroachdb/cockroach/kv.(*txnMetadata).close
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:142 +0x867

goroutine 361 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc208322180, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2081dbb80, 0x7f7cb372c288, 0xc208084380, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc208322180, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2081dbb80, 0x7f7cb372c288, 0xc208084380, 0xc208289f01, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc20813b0e0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd9f9d0, 0x15, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2082d1c30, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2081dbb80, 0x7f7cb372c288, 0xc208084380, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc2081180f0, 0x25, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc2082d1e98, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc208241410)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.func·008()
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:138 +0x1f4
created by github.com/cockroachdb/cockroach/kv.(*txnMetadata).close
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:142 +0x867

goroutine 369 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc208322180, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c3c0, 0x7f7cb372c288, 0xc208084c40, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc208322180, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c3c0, 0x7f7cb372c288, 0xc208084c40, 0xc20802b501, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc20813b680, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd9f9d0, 0x15, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2082e1c30, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c3c0, 0x7f7cb372c288, 0xc208084c40, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc208118c00, 0x25, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc2082e1e98, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc20822b2c0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.func·008()
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:138 +0x1f4
created by github.com/cockroachdb/cockroach/kv.(*txnMetadata).close
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:142 +0x867

goroutine 359 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc20826b700, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2081db180, 0x7f7cb372c288, 0xc2080842a0, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc20826b700, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2081db180, 0x7f7cb372c288, 0xc2080842a0, 0xc2080a0f01, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc2081e87e0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd9f9d0, 0x15, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc20833fc30, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2081db180, 0x7f7cb372c288, 0xc2080842a0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc2081b7680, 0x25, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc20833fe98, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc208241290)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.func·008()
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:138 +0x1f4
created by github.com/cockroachdb/cockroach/kv.(*txnMetadata).close
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:142 +0x867

goroutine 371 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc208323b00, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c640, 0x7f7cb372c288, 0xc208084d90, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc208323b00, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c640, 0x7f7cb372c288, 0xc208084d90, 0xc2082c7001, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc2081e8b40, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd9f9d0, 0x15, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2083c7c30, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c640, 0x7f7cb372c288, 0xc208084d90, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc208118ff0, 0x25, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc2083c7e98, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc20822b440)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.func·008()
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:138 +0x1f4
created by github.com/cockroachdb/cockroach/kv.(*txnMetadata).close
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:142 +0x867

goroutine 364 [select]:
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).heartbeat(0xc20809e440, 0xc2081c17a0, 0xc2081ad920)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:501 +0x895
created by github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendOne
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:294 +0x7c3

goroutine 336 [select]:
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).heartbeat(0xc20809e440, 0xc2082dd0e0, 0xc208332ba0)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:501 +0x895
created by github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendOne
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:294 +0x7c3

goroutine 363 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc208323b00, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2081dbe00, 0x7f7cb372c288, 0xc208084460, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc208323b00, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2081dbe00, 0x7f7cb372c288, 0xc208084460, 0xc20811ec01, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc2081ab680, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd9f9d0, 0x15, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc20829dc30, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2081dbe00, 0x7f7cb372c288, 0xc208084460, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc208118450, 0x25, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc20829de98, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc208241620)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.func·008()
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:138 +0x1f4
created by github.com/cockroachdb/cockroach/kv.(*txnMetadata).close
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:142 +0x867

goroutine 366 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc20826aa00, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c000, 0x7f7cb372c288, 0xc208084a80, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc20826aa00, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c000, 0x7f7cb372c288, 0xc208084a80, 0xc2081be901, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc2081abb00, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd9f9d0, 0x15, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc208327c30, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c000, 0x7f7cb372c288, 0xc208084a80, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc2081186c0, 0x25, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc208327e98, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc20822b1d0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.func·008()
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:138 +0x1f4
created by github.com/cockroachdb/cockroach/kv.(*txnMetadata).close
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:142 +0x867

goroutine 351 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc208323b00, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2083af900, 0x7f7cb372c288, 0xc2082696c0, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc208323b00, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2083af900, 0x7f7cb372c288, 0xc2082696c0, 0xc2081e3801, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc2081abc20, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd9f9d0, 0x15, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2083d9c30, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2083af900, 0x7f7cb372c288, 0xc2082696c0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc2082a3980, 0x25, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc2083d9e98, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc2082a2150)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.func·008()
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:138 +0x1f4
created by github.com/cockroachdb/cockroach/kv.(*txnMetadata).close
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:142 +0x867

goroutine 367 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc20826aa00, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c140, 0x7f7cb372c288, 0xc208084af0, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc20826aa00, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c140, 0x7f7cb372c288, 0xc208084af0, 0xc208166901, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc2081abd40, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd9f9d0, 0x15, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc208363c30, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c140, 0x7f7cb372c288, 0xc208084af0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc2081188a0, 0x25, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc208363e98, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc20822b200)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.func·008()
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:138 +0x1f4
created by github.com/cockroachdb/cockroach/kv.(*txnMetadata).close
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:142 +0x867

goroutine 358 [select]:
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).heartbeat(0xc20809e440, 0xc2081e8480, 0xc2082854a0)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:501 +0x895
created by github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendOne
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:294 +0x7c3

goroutine 373 [select]:
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).heartbeat(0xc20809e440, 0xc2083ad320, 0xc2081adda0)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:501 +0x895
created by github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendOne
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:294 +0x7c3

goroutine 365 [select]:
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).heartbeat(0xc20809e440, 0xc2081c1c20, 0xc2081adaa0)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:501 +0x895
created by github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendOne
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:294 +0x7c3

goroutine 368 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc20826b700, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c280, 0x7f7cb372c288, 0xc208084b60, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc20826b700, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c280, 0x7f7cb372c288, 0xc208084b60, 0xc20833a601, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc208202ea0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd9f9d0, 0x15, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2081d9c30, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc20821c280, 0x7f7cb372c288, 0xc208084b60, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc208118a80, 0x25, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc2081d9e98, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc20822b290)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.func·008()
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:138 +0x1f4
created by github.com/cockroachdb/cockroach/kv.(*txnMetadata).close
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:142 +0x867

goroutine 360 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc208322180, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2081db400, 0x7f7cb372c288, 0xc208084310, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc208322180, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2081db400, 0x7f7cb372c288, 0xc208084310, 0xc208344101, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc208202fc0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd9f9d0, 0x15, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2082f3c30, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2081db400, 0x7f7cb372c288, 0xc208084310, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc2081b7bf0, 0x25, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc2082f3e98, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc2082413b0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.func·008()
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:138 +0x1f4
created by github.com/cockroachdb/cockroach/kv.(*txnMetadata).close
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:142 +0x867

goroutine 342 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc208323b00, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc208292a00, 0x7f7cb372c288, 0xc2083d2230, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc208323b00, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc208292a00, 0x7f7cb372c288, 0xc2083d2230, 0xc208166301, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc2081c0120, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd9f9d0, 0x15, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc2082efc30, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc208292a00, 0x7f7cb372c288, 0xc2083d2230, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc2083a4de0, 0x25, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc2082efe98, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc2083a4540)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.func·008()
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:138 +0x1f4
created by github.com/cockroachdb/cockroach/kv.(*txnMetadata).close
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:142 +0x867

goroutine 344 [select]:
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).heartbeat(0xc20809e440, 0xc2083adc20, 0xc208332720)
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:501 +0x895
created by github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).sendOne
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:294 +0x7c3

goroutine 362 [chan receive]:
github.com/cockroachdb/cockroach/storage.func·006(0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:481 +0xd3
github.com/cockroachdb/cockroach/storage.(*Range).addReadWriteCmd(0xc208322180, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2081dbcc0, 0x7f7cb372c288, 0xc2080843f0, 0x1, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:503 +0xc86
github.com/cockroachdb/cockroach/storage.(*Range).AddCmd(0xc208322180, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2081dbcc0, 0x7f7cb372c288, 0xc2080843f0, 0xc208167301, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/range.go:306 +0x2e7
github.com/cockroachdb/cockroach/storage.func·016(0xc208202000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:751 +0xe3
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xd9f9d0, 0x15, 0x2faf080, 0x12a05f200, 0x4000000000000000, 0x0, 0x0, 0xc20838fc30, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/storage.(*Store).ExecuteCmd(0xc208058400, 0xd9f9d0, 0x15, 0x7f7cb372c248, 0xc2081dbcc0, 0x7f7cb372c288, 0xc2080843f0, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x2e7
github.com/cockroachdb/cockroach/kv.func·006(0xc200000000, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:149 +0x3d7
github.com/cockroachdb/cockroach/util.RetryWithBackoff(0xc2081182d0, 0x25, 0x0, 0x0, 0x0, 0x2, 0x0, 0xc20838fe98, 0x0, 0x0)
    /go/src/github.com/cockroachdb/cockroach/util/retry.go:80 +0x5d
github.com/cockroachdb/cockroach/kv.(*LocalSender).Send(0xc20819ce80, 0xc2082415f0)
    /go/src/github.com/cockroachdb/cockroach/kv/local_sender.go:163 +0x175
github.com/cockroachdb/cockroach/kv.func·008()
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:138 +0x1f4
created by github.com/cockroachdb/cockroach/kv.(*txnMetadata).close
    /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:142 +0x867

Metadata

Metadata

Assignees

Labels

No labels
No labels

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions