Skip to content

storage: TestBehaviorDuringLeaseTransfer failed under stress #25573

Closed
#25625
Closed
storage: TestBehaviorDuringLeaseTransfer failed under stress#25573
#25625
Assignees
a-robinson
Labels
A-kv-distributionRelating to rebalancing and leasing.C-test-failureBroken test (automatically or manually discovered).O-robotOriginated from a bot.
Milestone
2.1
@cockroach-teamcity

Description

@cockroach-teamcity
cockroach-teamcity
opened on May 16, 2018

SHA: https://github.com/cockroachdb/cockroach/commits/6f4e9b215f9a3284a39d507b3b0ddd310a0f87da

Parameters:

TAGS=deadlock
GOFLAGS=

Stress build found a failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=660793&tab=buildLog


I180516 09:24:48.402609 801230 storage/replica_command.go:82  [s1,r1/1:/M{in-ax}] test injecting error: storage/replica_test.go:575: injected transfer error
panic: close of closed channel [recovered]
	panic: close of closed channel

goroutine 800829 [running]:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Recover(0xc423472b40, 0x26db4a0, 0xc421e47b30)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:178 +0x11f
panic(0x20bf720, 0x26b3540)
	/usr/local/go/src/runtime/panic.go:505 +0x229
github.com/cockroachdb/cockroach/pkg/storage.TestBehaviorDuringLeaseTransfer.func5(0x26f, 0xc40000000c)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_test.go:665 +0x4d
github.com/cockroachdb/cockroach/pkg/storage.TestBehaviorDuringLeaseTransfer.func1(0x26f, 0xc40000000c)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_test.go:562 +0x83
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).requestLeaseLocked(0xc4217a3500, 0x26db4a0, 0xc421e47c20, 0x0, 0x0, 0xc42044f350, 0x100000001, 0x1, 0xc42044f360, 0xc42044f370, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:516 +0x7fd
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1(0xc4217a3500, 0x26f, 0xc40000000c, 0xc420643500, 0x26db4a0, 0xc421e47c20, 0xc4233f5be0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1423 +0xce3
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc4217a3500, 0x26db4a0, 0xc421e47c20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1432 +0x1d1
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startLeaseRenewer.func1(0x26db4a0, 0xc421e47b30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1582 +0x1c0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9e60, 0xc423472b40, 0xc42040fbe0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 1 [chan receive]:
testing.(*T).Run(0xc42055ab40, 0x237bcef, 0x1f, 0x247b600, 0x8e8301)
	/usr/local/go/src/testing/testing.go:825 +0x301
testing.runTests.func1(0xc420220a50)
	/usr/local/go/src/testing/testing.go:1063 +0x64
testing.tRunner(0xc420220a50, 0xc4206f1b90)
	/usr/local/go/src/testing/testing.go:777 +0xd0
testing.runTests(0xc4203560e0, 0x349d9c0, 0x1f1, 0x1f1, 0x200)
	/usr/local/go/src/testing/testing.go:1061 +0x2c4
testing.(*M).Run(0xc420485280, 0x0)
	/usr/local/go/src/testing/testing.go:978 +0x171
github.com/cockroachdb/cockroach/pkg/storage_test.TestMain(0xc420485280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/main_test.go:57 +0x1e3
main.main()
	_testmain.go:1050 +0x151

goroutine 11 [syscall]:
os/signal.signal_recv(0xc42006e7c0)
	/usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.0
	/usr/local/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 23 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1158 +0xf1
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:591 +0x110

goroutine 24 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:598 +0xab
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:592 +0x128

goroutine 28 [select, locked to thread]:
runtime.gopark(0x247eaa8, 0x0, 0x234cf86, 0x6, 0x18, 0x1)
	/usr/local/go/src/runtime/proc.go:291 +0x11a
runtime.selectgo(0xc42006bf50, 0xc4204bc5a0)
	/usr/local/go/src/runtime/select.go:392 +0xe50
runtime.ensureSigM.func1()
	/usr/local/go/src/runtime/signal_unix.go:549 +0x1f4
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1

goroutine 800748 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000006)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083230)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083230)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b91a0, 0xc423472b40, 0xc4203b9180)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800790 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000030)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083ce0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083ce0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9910, 0xc423472b40, 0xc4203b9900)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800771 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000001d)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc4230836e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc4230836e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b95a0, 0xc423472b40, 0xc4203b9590)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800769 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000001b)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083680)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083680)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9560, 0xc423472b40, 0xc4203b9550)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800814 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x26db4a0, 0xc42263b0b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1546 +0x3ae
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9d00, 0xc423472b40, 0xc4217a6820)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800801 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000003b)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc421e476e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc421e476e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9aa0, 0xc423472b40, 0xc4203b9a90)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800800 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000003a)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc421e47650)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc421e47650)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9a80, 0xc423472b40, 0xc4203b9a70)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800741 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1(0x26db4a0, 0xc4230830e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:156 +0x49
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9040, 0xc423472b40, 0xc42040f400)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800787 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000002d)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083bc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083bc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b98b0, 0xc423472b40, 0xc4203b98a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800786 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000002c)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083b30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083b30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9820, 0xc423472b40, 0xc4203b9810)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800805 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000003f)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc421e47950)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc421e47950)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9b40, 0xc423472b40, 0xc4203b9b30)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800781 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000027)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083920)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083920)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9740, 0xc423472b40, 0xc4203b9720)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800789 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000002f)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083cb0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083cb0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b98f0, 0xc423472b40, 0xc4203b98e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800775 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000021)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc4230837a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc4230837a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9630, 0xc423472b40, 0xc4203b9620)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800758 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000010)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083440)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083440)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b93f0, 0xc423472b40, 0xc4203b93d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800750 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000008)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083290)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083290)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b91f0, 0xc423472b40, 0xc4203b91e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800778 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000024)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083830)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083830)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b96a0, 0xc423472b40, 0xc4203b9680)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800764 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000016)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083560)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083560)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b94c0, 0xc423472b40, 0xc4203b94b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800804 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000003e)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc421e47920)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc421e47920)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9b20, 0xc423472b40, 0xc4203b9af0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800811 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func2(0x26db4a0, 0xc42263a840)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1410 +0x10e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9c80, 0xc423472b40, 0xc42040f4c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800803 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000003d)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc421e47860)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc421e47860)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9ae0, 0xc423472b40, 0xc4203b9ad0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800797 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000037)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc421e47050)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc421e47050)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9a20, 0xc423472b40, 0xc4203b99f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800765 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000017)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083590)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083590)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b94e0, 0xc423472b40, 0xc4203b94d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800749 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000007)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083260)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083260)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b91c0, 0xc423472b40, 0xc4203b91b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800798 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000038)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc421e47200)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc421e47200)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9a40, 0xc423472b40, 0xc4203b9a30)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800796 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000036)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc421e46e70)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc421e46e70)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b99e0, 0xc423472b40, 0xc4203b99d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800773 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000001f)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083740)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083740)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b95f0, 0xc423472b40, 0xc4203b95e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800813 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x26db4a0, 0xc42263a000)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1546 +0x3ae
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9cf0, 0xc423472b40, 0xc4217a6780)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 801096 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x26db4a0, 0xc42263b6e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:580 +0x194
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420447eb0, 0xc423472b40, 0xc42036baa0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800795 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000035)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc421e46120)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc421e46120)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b99c0, 0xc423472b40, 0xc4203b99b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800808 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).coalescedHeartbeatsLoop(0xc4221a8400, 0x26db4a0, 0xc42263a810)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3712 +0x159
github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.coalescedHeartbeatsLoop)-fm(0x26db4a0, 0xc42263a810)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3658 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9c20, 0xc423472b40, 0xc4203b9c10)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800831 [select]:
github.com/cockroachdb/cockroach/pkg/storage/compactor.(*Compactor).Start.func1(0x26db4a0, 0xc4230830b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/compactor/compactor.go:108 +0x1b0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9e80, 0xc423472b40, 0xc423083080)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800751 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000009)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc4230832c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc4230832c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9240, 0xc423472b40, 0xc4203b9220)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800744 [runnable]:
github.com/cockroachdb/cockroach/pkg/storage.TrackRaftProtos.func1(0x26e9f40, 0xc422a5f310)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/track_raft_protos.go:90 +0x2e6
github.com/cockroachdb/cockroach/pkg/util/protoutil.MarshalToWithoutFuzzing(0x26e9f40, 0xc422a5f310, 0xc422dee505, 0x127, 0x127, 0x12c, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/protoutil/marshal.go:77 +0x40
github.com/cockroachdb/cockroach/pkg/roachpb.(*Value).SetProto(0xc4233974f0, 0x26e9f40, 0xc422a5f310, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/data.go:400 +0xd7
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).append(0xc4217a3500, 0x26db4a0, 0xc423082c90, 0x26ff760, 0xc422d1fc28, 0xd, 0x6, 0x256, 0xc422a5f310, 0x1, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_raftstorage.go:566 +0x251
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc4217a3500, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3645 +0x1acd
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4217a3500, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3457 +0xdb
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc4221a8400, 0x26db4a0, 0xc423083170, 0x1)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3601 +0xda
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083170)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:215 +0x259
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083170)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b90e0, 0xc423472b40, 0xc4203b90c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800745 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000003)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc4230831a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc4230831a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9110, 0xc423472b40, 0xc4203b90f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800770 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000001c)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc4230836b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc4230836b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9580, 0xc423472b40, 0xc4203b9570)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800742 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0x40)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083110)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083110)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9070, 0xc423472b40, 0xc4203b9060)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800784 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000002a)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083a40)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083a40)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b97c0, 0xc423472b40, 0xc4203b97b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800782 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000028)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083980)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083980)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9770, 0xc423472b40, 0xc4203b9750)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 801097 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x26db4a0, 0xc42263b710)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:580 +0x194
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420447ed0, 0xc423472b40, 0xc42036bb00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800807 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop(0xc4221a8400, 0x26db4a0, 0xc421e47b00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3671 +0x1e7
github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.raftTickLoop)-fm(0x26db4a0, 0xc421e47b00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3657 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9bf0, 0xc423472b40, 0xc4203b9be0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800774 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000020)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083770)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083770)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9610, 0xc423472b40, 0xc4203b9600)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800794 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000034)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc421e46000)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc421e46000)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b99a0, 0xc423472b40, 0xc4203b9990)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 801100 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc42141ce40, 0x26db4a0, 0xc42263b050, 0xc423472b40, 0x17d888bf, 0xed28defe0, 0x0, 0xc4217a3500, 0xc4217a35e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:212 +0x212
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc4217a3500, 0xc421d2ddd0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:274 +0x81
github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc421d2ddd0, 0xc4201af8c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:322 +0x1a6
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1(0x26db4a0, 0xc42263b050)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:272 +0x1c0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420447f10, 0xc423472b40, 0xc42036bb80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800756 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000000e)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc4230833e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc4230833e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b93a0, 0xc423472b40, 0xc4203b9390)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800763 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000015)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083530)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083530)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b94a0, 0xc423472b40, 0xc4203b9490)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800761 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000013)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc4230834d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc4230834d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9460, 0xc423472b40, 0xc4203b9450)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 801099 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x26db4a0, 0xc42263b770)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:580 +0x194
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420447ef0, 0xc423472b40, 0xc42036bb60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800743 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0x41)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083140)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083140)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b90a0, 0xc423472b40, 0xc4203b9080)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800785 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000002b)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083b00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083b00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b97f0, 0xc423472b40, 0xc4203b97e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800783 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000029)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc4230839b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc4230839b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9790, 0xc423472b40, 0xc4203b9780)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800806 [semacquire]:
sync.runtime_Semacquire(0xc420bb9bfc)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc420bb9bf0)
	/usr/local/go/src/sync/waitgroup.go:129 +0x72
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc420bb9b80, 0x26db4a0, 0xc421e479b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:172 +0x31
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait-fm(0x26db4a0, 0xc421e479b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3655 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9b80, 0xc423472b40, 0xc4203b9b70)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800747 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000005)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083200)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083200)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9170, 0xc423472b40, 0xc4203b9150)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800752 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000000a)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc4230832f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc4230832f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9280, 0xc423472b40, 0xc4203b9250)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800440 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func2(0xc42036d400, 0xc42287a8d0, 0xc4217a3500, 0x26db4a0, 0xc4220de2d0, 0xc4206508c0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1447 +0x222
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc4217a3500, 0x26db4a0, 0xc4220de2d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1500 +0x23f
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeReadOnlyBatch(0xc4217a3500, 0x26db4a0, 0xc4220de2d0, 0x26f, 0x6, 0x0, 0x0, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2534 +0xaa9
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4217a3500, 0x26db4a0, 0xc4220de2d0, 0x26f, 0x6, 0x0, 0x0, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1870 +0x2a5
github.com/cockroachdb/cockroach/pkg/internal/client.Wrap.func1(0x26db420, 0xc42003e120, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/sender.go:176 +0xe5
github.com/cockroachdb/cockroach/pkg/internal/client.SenderFunc.Send(0xc4203f6700, 0x26db420, 0xc42003e120, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/sender.go:103 +0x75
github.com/cockroachdb/cockroach/pkg/internal/client.SendWrappedWith(0x26db420, 0xc42003e120, 0x26bbb20, 0xc4203f6700, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/sender.go:152 +0x136
github.com/cockroachdb/cockroach/pkg/storage.(*testContext).SendWrappedWith(0xc4203e2000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_test.go:271 +0xca
github.com/cockroachdb/cockroach/pkg/storage.(*testContext).SendWrapped(0xc4203e2000, 0x26fb100, 0xc42263ba70, 0x0, 0x0, 0xc42212ff90)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_test.go:276 +0x85
github.com/cockroachdb/cockroach/pkg/storage.TestBehaviorDuringLeaseTransfer(0xc42055ab40)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_test.go:673 +0xba8
testing.tRunner(0xc42055ab40, 0x247b600)
	/usr/local/go/src/testing/testing.go:777 +0xd0
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:824 +0x2e0

goroutine 800791 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000031)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083e00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083e00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9930, 0xc423472b40, 0xc4203b9920)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800768 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000001a)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083650)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083650)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9540, 0xc423472b40, 0xc4203b9530)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800759 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000011)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083470)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083470)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9410, 0xc423472b40, 0xc4203b9400)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800772 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000001e)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083710)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083710)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b95d0, 0xc423472b40, 0xc4203b95b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800812 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x26db4a0, 0xc42263a870)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1546 +0x3ae
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9c90, 0xc423472b40, 0xc4217a6730)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800767 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000019)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083620)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083620)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9520, 0xc423472b40, 0xc4203b9510)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 801268 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4217a3500, 0x26db4a0, 0xc4220deed0, 0x26f, 0xa, 0x0, 0x0, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2805 +0x712
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4217a3500, 0x26db4a0, 0xc4220deed0, 0x26f, 0xa, 0x0, 0x0, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2628 +0xae
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4217a3500, 0x26db4a0, 0xc4220deed0, 0x26f, 0xa, 0x0, 0x0, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1867 +0x69d
github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).requestLeaseAsync.func2(0x26db4a0, 0xc4220deed0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:310 +0x5af
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc423472b40, 0x26db4a0, 0xc4220deed0, 0xc4203fb480, 0x35, 0x26ff000, 0xc42232d680, 0xc423193320)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xe6
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:259 +0x133

goroutine 800498 [chan receive]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1(0x26db4a0, 0xc421541fb0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:321 +0x51
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203d5870, 0xc423472b40, 0xc4202f8260)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800792 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000032)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083e30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083e30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9950, 0xc423472b40, 0xc4203b9940)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800802 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc40000003c)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc421e477d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc421e477d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9ac0, 0xc423472b40, 0xc4203b9ab0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 801094 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x26db4a0, 0xc42263b680)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:580 +0x194
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420447e70, 0xc423472b40, 0xc42036ba40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800779 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000025)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc423083860)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc423083860)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b96c0, 0xc423472b40, 0xc4203b96b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 800780 [semacquire]:
sync.runtime_notifyListWait(0xc421fc3710, 0xc400000026)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421fc3700)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420bb9b80, 0x26db4a0, 0xc4230838c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x26db4a0, 0xc4230838c0)

Metadata

Metadata

Assignees

Labels

A-kv-distributionRelating to rebalancing and leasing.C-test-failureBroken test (automatically or manually discovered).O-robotOriginated from a bot.

Type

No type

Projects

No projects

Milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions