storage: TestStoreRangeMergeTxnFailure failed under stress #27820
Description
SHA: https://github.com/cockroachdb/cockroach/commits/6a69ebdab7c4f10e0a0ad28de59207b066557973
Parameters:
TAGS=
GOFLAGS=-race
Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=789978&tab=buildLog
I180721 06:49:21.445819 49778 gossip/gossip.go:363 [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<> ServerVersion:<major_val:0 minor_val:0 patch:0 unstable:0 >
I180721 06:49:21.538683 49778 storage/replica_command.go:279 [s1,r1/1:/M{in-ax}] initiating a split of this range at key "b" [r2]
I180721 06:49:21.597053 49778 storage/replica_command.go:406 [s1,r1/1:{/Min-b}] initiating a merge of r2:{b-/Max} [(n1,s1):1, next=2] into this range
I180721 06:49:21.695870 49778 storage/replica_command.go:406 [s1,r1/1:{/Min-b}] initiating a merge of r2:{b-/Max} [(n1,s1):1, next=2] into this range
E180721 06:49:22.545450 50137 storage/consistency_queue.go:125 [replica consistency checker,s1,r1/1:{/Min-b}] computing own checksum: could not dial node ID 1: no node dialer configured
E180721 06:49:22.545859 50137 storage/queue.go:778 [replica consistency checker,s1,r1/1:{/Min-b}] computing own checksum: could not dial node ID 1: no node dialer configured
E180721 06:50:23.535169 50155 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:50:23.535536 50155 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:51:23.537610 50286 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:51:23.538121 50286 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:52:23.540229 50362 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:52:23.540599 50362 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:53:23.543988 50519 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:53:23.544323 50519 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:54:23.548648 50659 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:54:23.548983 50659 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:55:23.551732 50757 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:55:23.552092 50757 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:56:23.554483 50815 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:56:23.554816 50815 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:57:23.557075 50942 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:57:23.557463 50942 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:58:23.559546 51123 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:58:23.559904 51123 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:59:23.562746 51212 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 06:59:23.563088 51212 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:00:23.565118 51327 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:00:23.565460 51327 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:01:23.567585 51384 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:01:23.568023 51384 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:02:23.574411 51603 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:02:23.574731 51603 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:03:23.576810 51718 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:03:23.577157 51718 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:04:23.579137 51807 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:04:23.579468 51807 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:05:23.581636 51958 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:05:23.581987 51958 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:06:23.584139 52030 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:06:23.584485 52030 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:07:23.586606 52175 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:07:23.586937 52175 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:08:23.588994 52306 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:08:23.589321 52306 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:09:23.591951 52451 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:09:23.592324 52451 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:10:23.595611 52555 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:10:23.595950 52555 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:11:23.598915 52655 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:11:23.599250 52655 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:12:23.602458 52810 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:12:23.602811 52810 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:13:23.604845 52893 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:13:23.605222 52893 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:14:23.607359 53063 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:14:23.607698 53063 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:15:23.609848 53161 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:15:23.610190 53161 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:16:23.612679 48942 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:16:23.613006 48942 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:17:23.615050 53403 storage/consistency_queue.go:125 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
E180721 07:17:23.615442 53403 storage/queue.go:778 [replica consistency checker,s1,r2/1:{b-/Max}] context deadline exceeded
panic: test timed out after 30m0s
goroutine 53607 [running]:
testing.(*M).startAlarm.func1()
/usr/local/go/src/testing/testing.go:1240 +0x146
created by time.goFunc
/usr/local/go/src/time/sleep.go:172 +0x52
goroutine 1 [chan receive, 28 minutes]:
testing.(*T).Run(0xc4202281e0, 0x31190a6, 0x1d, 0x3226720, 0xc420279901)
/usr/local/go/src/testing/testing.go:825 +0x597
testing.runTests.func1(0xc4202281e0)
/usr/local/go/src/testing/testing.go:1063 +0xa5
testing.tRunner(0xc4202281e0, 0xc420279b20)
/usr/local/go/src/testing/testing.go:777 +0x16e
testing.runTests(0xc4203d47c0, 0x489cd00, 0x1fe, 0x1fe, 0x4a)
/usr/local/go/src/testing/testing.go:1061 +0x4e2
testing.(*M).Run(0xc42046a280, 0x0)
/usr/local/go/src/testing/testing.go:978 +0x2ce
github.com/cockroachdb/cockroach/pkg/storage_test.TestMain(0xc42046a280)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/main_test.go:57 +0x2a9
main.main()
_testmain.go:1076 +0x22b
goroutine 6 [syscall, 30 minutes]:
os/signal.signal_recv(0x895791)
/usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:22 +0x30
created by os/signal.init.0
/usr/local/go/src/os/signal/signal_unix.go:28 +0x4f
goroutine 7 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1171 +0x132
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:590 +0x189
goroutine 8 [chan receive, 30 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:597 +0xf2
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:591 +0x1a1
goroutine 34 [select, 30 minutes, locked to thread]:
runtime.gopark(0x3228e88, 0x0, 0x30ed307, 0x6, 0x18, 0x1)
/usr/local/go/src/runtime/proc.go:291 +0xf9
runtime.selectgo(0xc420091f50, 0xc4200b8300)
/usr/local/go/src/runtime/select.go:392 +0x11d4
runtime.ensureSigM.func1()
/usr/local/go/src/runtime/signal_unix.go:549 +0x19f
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1
goroutine 50056 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024c0)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a4090)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a4090)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204571c0, 0xc420ca6240, 0xc4204571b0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50071 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024cd)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc420ad4090)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc420ad4090)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457410, 0xc420ca6240, 0xc420457400)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50055 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024c2)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a4030)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a4030)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204571a0, 0xc420ca6240, 0xc420457190)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50066 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024cb)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a4270)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a4270)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457330, 0xc420ca6240, 0xc420457310)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50081 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024dd)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc420ad4210)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc420ad4210)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204575d0, 0xc420ca6240, 0xc4204575c0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50026 [select, 28 minutes]:
github.com/cockroachdb/cockroach/pkg/storage/compactor.(*Compactor).Start.func1.1(0x38f1800, 0xc420ad4480)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/compactor/compactor.go:127 +0x22d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420372140, 0xc420ca6240, 0xc42042c080)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50100 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024ef)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a45a0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a45a0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204578a0, 0xc420ca6240, 0xc420457890)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50064 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024c8)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a4210)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a4210)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204572d0, 0xc420ca6240, 0xc4204572c0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50084 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024da)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4211760c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4211760c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457630, 0xc420ca6240, 0xc420457620)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50105 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024f2)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a4600)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a4600)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457940, 0xc420ca6240, 0xc420457930)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50101 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024eb)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4208d1860)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4208d1860)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204578c0, 0xc420ca6240, 0xc4204578b0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50113 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024bc)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a46c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a46c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457a40, 0xc420ca6240, 0xc420457a30)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50114 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024f7)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc421176390)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc421176390)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457a60, 0xc420ca6240, 0xc420457a50)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50079 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024d9)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc420ad41e0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc420ad41e0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457590, 0xc420ca6240, 0xc420457580)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50104 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024f0)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc421176240)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc421176240)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457920, 0xc420ca6240, 0xc420457910)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50077 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024d5)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc420ad41b0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc420ad41b0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457550, 0xc420ca6240, 0xc420457540)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50093 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024e6)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a44e0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a44e0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457760, 0xc420ca6240, 0xc420457750)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50097 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024e7)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4211761b0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4211761b0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457810, 0xc420ca6240, 0xc420457800)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50088 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024de)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc421176150)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc421176150)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204576b0, 0xc420ca6240, 0xc4204576a0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50108 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024f4)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc421176300)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc421176300)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204579a0, 0xc420ca6240, 0xc420457990)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50098 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024ed)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a4570)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a4570)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457860, 0xc420ca6240, 0xc420457850)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50031 [select, 28 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x38f1800, 0xc420ad4690)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203722b0, 0xc420ca6240, 0xc42042c260)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50075 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024d6)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc420ad4120)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc420ad4120)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457510, 0xc420ca6240, 0xc420457500)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50067 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024ce)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc420ad4000)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc420ad4000)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457360, 0xc420ca6240, 0xc420457340)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50070 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024d1)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc420ad4060)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc420ad4060)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204573e0, 0xc420ca6240, 0xc4204573d0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50091 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024e5)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a4480)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a4480)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457710, 0xc420ca6240, 0xc420457700)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50051 [semacquire]:
sync.runtime_notifyListWait(0xc4207cdcc8, 0x1357)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4207cdcb8)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc4207cdba0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:648 +0xb3
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:635 +0x607
goroutine 50032 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x38f1800, 0xc420ad4600)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203722d0, 0xc420ca6240, 0xc42042c280)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50029 [select, 28 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x38f1800, 0xc420b54030)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420372290, 0xc420ca6240, 0xc42042c220)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50087 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024e2)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a43f0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a43f0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457690, 0xc420ca6240, 0xc420457680)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50023 [select, 1 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x38f1800, 0xc420ad44b0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1588 +0x499
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420339d20, 0xc420ca6240, 0xc420592820)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50078 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024d4)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a4330)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a4330)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457570, 0xc420ca6240, 0xc420457560)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50076 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024d7)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc420ad4180)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc420ad4180)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457530, 0xc420ca6240, 0xc420457520)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50058 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024c6)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a40f0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a40f0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457200, 0xc420ca6240, 0xc4204571f0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50118 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024bf)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc421176420)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc421176420)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457ae0, 0xc420ca6240, 0xc420457ad0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50119 [semacquire, 28 minutes]:
sync.runtime_Semacquire(0xc420ca67d4)
/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc420ca67c8)
/usr/local/go/src/sync/waitgroup.go:129 +0xb3
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc420ca6750, 0x38f1800, 0xc4213a46f0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:172 +0x3f
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait-fm(0x38f1800, 0xc4213a46f0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3777 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457b00, 0xc420ca6240, 0xc420457af0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50028 [select, 28 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x38f1800, 0xc420b54000)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420372270, 0xc420ca6240, 0xc42042c200)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50065 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024cc)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a4240)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a4240)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457300, 0xc420ca6240, 0xc4204572e0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50034 [select, 1 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x38f1800, 0xc420ad46f0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420372370, 0xc420ca6240, 0xc42042c2c0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50083 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024e1)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a4390)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a4390)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457610, 0xc420ca6240, 0xc420457600)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50059 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024c5)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a4120)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a4120)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457220, 0xc420ca6240, 0xc420457210)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50153 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc42129c600, 0x38f1800, 0xc420671f20)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:793 +0x365
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).startHeartbeatLoopLocked.func1(0x38f1800, 0xc4201699e0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:941 +0x54
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420ca6240, 0x38f1800, 0xc4201699e0, 0xc420846c90, 0x29, 0x391ccc0, 0xc420c58a80, 0xc4214aad40)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xf4
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x14d
goroutine 50069 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024cf)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc420ad4030)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc420ad4030)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204573b0, 0xc420ca6240, 0xc4204573a0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50123 [select, 1 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x38f1800, 0xc4213a4810)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1588 +0x499
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457ba0, 0xc420ca6240, 0xc421175b80)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50074 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024d0)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a4300)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a4300)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204574f0, 0xc420ca6240, 0xc4204574e0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50121 [runnable]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).coalescedHeartbeatsLoop(0xc420500a00, 0x38f1800, 0xc4213a4780)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3846 +0x1a6
github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.coalescedHeartbeatsLoop)-fm(0x38f1800, 0xc4213a4780)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3780 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457b60, 0xc420ca6240, 0xc420457b40)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50117 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024be)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a4750)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a4750)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457ac0, 0xc420ca6240, 0xc420457ab0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 49798 [select]:
github.com/cockroachdb/cockroach/pkg/storage/txnwait.(*Queue).MaybeWaitForPush(0xc4208d1140, 0x38f1800, 0xc420c6c240, 0x38c9d40, 0xc420b17880, 0xc420ac1a40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/txnwait/txnqueue.go:484 +0x169b
github.com/cockroachdb/cockroach/pkg/storage.(*Store).maybeWaitForPushee(0xc420500a00, 0x38f1800, 0xc421286540, 0xc420e7ab88, 0xc420b17880, 0x100000001, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3078 +0x190
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420500a00, 0x38f1800, 0xc421286540, 0x15434f8c3cc82091, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2956 +0x6b0
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc421b00000, 0x38f1800, 0xc421286540, 0x0, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:185 +0x134
github.com/cockroachdb/cockroach/pkg/kv.(*senderTransport).SendNext(0xc421442fa0, 0x38f1800, 0xc421286540, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:383 +0x2b1
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420027770, 0x38f1800, 0xc421286480, 0xc4200277c0, 0x1, 0xc4203a42e0, 0x1, 0x1, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1257 +0x5a8
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420027770, 0x38f1800, 0xc421286480, 0x1, 0xc4203a42e0, 0x1, 0x1, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:367 +0x2cc
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420027770, 0x38f1800, 0xc421286480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:434 +0x273
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420027770, 0x38f1800, 0xc421286480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1037 +0x3db
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420027770, 0x38f1800, 0xc421286480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:715 +0x17d5
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420027770, 0x38f1800, 0xc421286480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:629 +0x5a0
github.com/cockroachdb/cockroach/pkg/internal/client.(*CrossRangeTxnWrapperSender).Send(0xc421b90360, 0x38f1800, 0xc4212861b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:213 +0xe4
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc421b90300, 0x38f1800, 0xc4212861b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:617 +0x124
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc421b90300, 0x38f1800, 0xc4212861b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:599 +0xb5
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x38f1800, 0xc4212861b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:565 +0xa4
github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x38f1800, 0xc4212861b0, 0xc421a0e648, 0xc4208e2a00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:542 +0x152
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc421b90300, 0x38f1800, 0xc4212861b0, 0xc4208e2a00, 0x1, 0xc4212a5640)
/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:565 +0xee
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc420466780, 0x38f1800, 0xc4212861b0, 0xc421bcd0e0, 0x1, 0x1, 0x15434f8c3cb9b898, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:563 +0x1077
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processWriteIntentError(0xc420466780, 0x38f1800, 0xc4212861b0, 0xc420aa1860, 0x3917640, 0xc4219d2d00, 0x15434f8c3cb9b898, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:423 +0x15a
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420500a00, 0x38f1800, 0xc4212861b0, 0x15434f8c3cb9a980, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3030 +0xfd4
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc421b00000, 0x38f1800, 0xc4212860f0, 0x0, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:185 +0x134
github.com/cockroachdb/cockroach/pkg/kv.(*senderTransport).SendNext(0xc421442dc0, 0x38f1800, 0xc4212860f0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:383 +0x2b1
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420027770, 0x38f1800, 0xc421286000, 0xc4200277c0, 0x1, 0xc4203c7f00, 0x1, 0x1, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1257 +0x5a8
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420027770, 0x38f1800, 0xc421286000, 0x1, 0xc4203c7f00, 0x1, 0x1, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:367 +0x2cc
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420027770, 0x38f1800, 0xc421286000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:434 +0x273
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420027770, 0x38f1800, 0xc421286000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1037 +0x3db
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420027770, 0x38f1800, 0xc421286000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:715 +0x17d5
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420027770, 0x38f1800, 0xc421286000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:629 +0x5a0
github.com/cockroachdb/cockroach/pkg/internal/client.(*CrossRangeTxnWrapperSender).Send(0xc421b90360, 0x38f1780, 0xc420052148, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:213 +0xe4
github.com/cockroachdb/cockroach/pkg/internal/client.lookupRangeFwdScan(0x38f1780, 0xc420052148, 0x38c7500, 0xc421b90360, 0xc4207907d8, 0x1, 0x8, 0x0, 0x0, 0x1dcd6500, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/range_lookup.go:319 +0x4e5
github.com/cockroachdb/cockroach/pkg/internal/client.RangeLookup(0x38f1780, 0xc420052148, 0x38c7500, 0xc421b90360, 0xc4207907d8, 0x1, 0x8, 0x0, 0x0, 0x2b0d0572f200, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/range_lookup.go:202 +0xb0f
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).maybeWatchForMerge.func1(0x38f1780, 0xc420052148)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2749 +0x14c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420ca6240, 0x38f1780, 0xc420052148, 0xc421bc6b80, 0x16, 0x0, 0x0, 0xc4203c7da0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xf4
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x14d
goroutine 50131 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc420c41d40, 0x38f1800, 0xc420ad45d0, 0xc420ca6240, 0x115f0a53, 0xed2e4d52c, 0x0, 0xc420b17880, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:217 +0x24e
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc420b17880, 0x58)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:279 +0x13c
github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc42134b7d0, 0xc421332cc0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:324 +0x2e2
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1(0x38f1800, 0xc420ad45d0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:277 +0x392
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420372380, 0xc420ca6240, 0xc42042c2e0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50052 [chan receive, 28 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1(0x38f1800, 0xc4201973e0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:329 +0x6c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420456140, 0xc420ca6240, 0xc420412200)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50030 [select, 28 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x38f1800, 0xc42063a000)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203722a0, 0xc420ca6240, 0xc42042c240)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50112 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024fb)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc421176360)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc421176360)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420457a20, 0xc420ca6240, 0xc420457a10)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50089 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024e0)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a4420)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a4420)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204576d0, 0xc420ca6240, 0xc4204576c0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 50057 [semacquire]:
sync.runtime_notifyListWait(0xc420466b10, 0x4968c3d5000024c3)
/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420466b00)
/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ca6750, 0x38f1800, 0xc4213a40c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x38f1800, 0xc4213a40c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204571e0, 0xc420ca6240, 0xc4204571d0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xbb
goroutine 53515 [select, 1 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420500a00, 0x38f1800, 0xc4213a4c30, 0x15435113c717238d, 0x0, 0x100000001, 0x1, 0x2, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3049 +0x98b
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc421b00000, 0x38f1800, 0xc4213a4b40, 0x0, 0x0, 0x100000001, 0x1, 0x2, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:185 +0x134
github.com/cockroachdb/cockroach/pkg/kv.(*senderTransport).SendNext(0xc421238000, 0x38f1800, 0xc4213a4b40, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:383 +0x2b1
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420027770, 0x38f1800, 0xc4213a4960, 0xc4200277c0, 0x2, 0xc4203de7c0, 0x1, 0x1, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1257 +0x5a8
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420027770, 0x38f1800, 0xc4213a4960, 0x2, 0xc4203de7c0, 0x1, 0x1, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:367 +0x2cc
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420027770, 0x38f1800, 0xc4213a4960, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:434 +0x273
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420027770, 0x38f1800, 0xc4213a4960, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1037 +0x3db
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420027770, 0x38f1800, 0xc4213a4960, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:715 +0x17d5