util/log: TestCrashReportingPacket failed under stress #25570
Description
SHA: https://github.com/cockroachdb/cockroach/commits/6f4e9b215f9a3284a39d507b3b0ddd310a0f87da
Parameters:
TAGS=
GOFLAGS=
Stress build found a failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=660448&tab=buildLog
[signal SIGSEGV: segmentation violation code=0x1 addr=0x8 pc=0x10662c5]
goroutine 185 [running]:
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.(*DString).Size(0x0, 0xc4210fa3e8)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/sem/tree/datum.go:933 +0x5
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowContainer).rowSize(0xc42085f710, 0xc4210fa4c8, 0x1, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/row_container.go:247 +0x66
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowContainer).AddRow(0xc42085f710, 0x24063c0, 0xc4215b0c30, 0xc4210fa4c8, 0x1, 0x1, 0x7, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/row_container.go:272 +0x7c
github.com/cockroachdb/cockroach/pkg/sql.(*planner).ShowClusterSetting.func1(0x24063c0, 0xc4215b0c30, 0xc42165c420, 0x0, 0x0, 0x2202400, 0xc420516228)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/show_cluster_setting.go:161 +0x186
github.com/cockroachdb/cockroach/pkg/sql.doExpandPlan(0x24063c0, 0xc4215b0c30, 0xc42165c420, 0x7fffffffffffffff, 0x0, 0x0, 0x0, 0x0, 0x23fc800, 0xc42158a380, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/expand_plan.go:320 +0x2c51
github.com/cockroachdb/cockroach/pkg/sql.(*planner).expandPlan(0xc42165c420, 0x24063c0, 0xc4215b0c30, 0x23fc800, 0xc42158a380, 0x23fc800, 0xc42158a380, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/expand_plan.go:35 +0xce
github.com/cockroachdb/cockroach/pkg/sql.(*planner).optimizePlan(0xc42165c420, 0x24063c0, 0xc4215b0c30, 0x23fc800, 0xc42158a380, 0xc4202ca3b2, 0x1, 0x1, 0x23fc800, 0xc42158a380, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/optimize.go:47 +0xeb
github.com/cockroachdb/cockroach/pkg/sql.(*planner).makePlan(0xc42165c420, 0x24063c0, 0xc4215b0c30, 0x2408980, 0xc421638180, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:324 +0x2ea
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc42165c000, 0x24063c0, 0xc4215b0c30, 0x2408980, 0xc421638180, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:578 +0xb28
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc42165c000, 0x24063c0, 0xc4215b0c30, 0x2408980, 0xc421638180, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:397 +0xb33
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc42165c000, 0x24063c0, 0xc4215b0c30, 0x2408980, 0xc421638180, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:91 +0x358
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc42165c000, 0x24063c0, 0xc4215b0930, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1051 +0x2039
github.com/cockroachdb/cockroach/pkg/sql.(*internalExecutorImpl).initConnEx.func1(0xc42165c000, 0x24063c0, 0xc4215b0930, 0xc4203627e0, 0xc4202ca3c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/internal.go:211 +0x86
created by github.com/cockroachdb/cockroach/pkg/sql.(*internalExecutorImpl).initConnEx
/go/src/github.com/cockroachdb/cockroach/pkg/sql/internal.go:200 +0x268
goroutine 1 [chan receive]:
testing.(*T).Run(0xc4204d73b0, 0x2103fdb, 0x18, 0x2202380, 0x855401)
/usr/local/go/src/testing/testing.go:825 +0x301
testing.runTests.func1(0xc4201d00f0)
/usr/local/go/src/testing/testing.go:1063 +0x64
testing.tRunner(0xc4201d00f0, 0xc420685dc8)
/usr/local/go/src/testing/testing.go:777 +0xd0
testing.runTests(0xc4202da1e0, 0x30fe840, 0x27, 0x27, 0xcf37a0)
/usr/local/go/src/testing/testing.go:1061 +0x2c4
testing.(*M).Run(0xc42078c100, 0x0)
/usr/local/go/src/testing/testing.go:978 +0x171
github.com/cockroachdb/cockroach/pkg/util/log_test.TestMain(0xc42078c100)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/main_test.go:41 +0xdb
main.main()
_testmain.go:120 +0x151
goroutine 20 [syscall]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.0
/usr/local/go/src/os/signal/signal_unix.go:28 +0x41
goroutine 41 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1158 +0xf1
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:591 +0x110
goroutine 42 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:598 +0xab
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:592 +0x128
goroutine 22 [select, locked to thread]:
runtime.gopark(0x22041d0, 0x0, 0x20e7c6c, 0x6, 0x18, 0x1)
/usr/local/go/src/runtime/proc.go:291 +0x11a
runtime.selectgo(0xc42006bf50, 0xc4201ca060)
/usr/local/go/src/runtime/select.go:392 +0xe50
runtime.ensureSigM.func1()
/usr/local/go/src/runtime/signal_unix.go:549 +0x1f4
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1
goroutine 122 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/getsentry/raven-go.(*Client).worker(0xc42049e460)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/getsentry/raven-go/client.go:482 +0x170
created by github.com/cockroachdb/cockroach/vendor/github.com/getsentry/raven-go.(*Client).Capture.func1
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/getsentry/raven-go/client.go:541 +0x3e
goroutine 121 [semacquire]:
sync.runtime_notifyListWait(0xc4205b5290, 0xc400000000)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4205b5280)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Quiesce(0xc4205574d0, 0x2406340, 0xc4200ca050)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:473 +0x18d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Stop(0xc4205574d0, 0x2406340, 0xc4200ca050)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:416 +0x1ff
github.com/cockroachdb/cockroach/pkg/util/log_test.TestCrashReportingPacket.func5(0xc4209e9f68, 0x2406340, 0xc4200ca050, 0xc42058c000, 0xc4204d73b0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/crash_reporting_packet_test.go:91 +0x13e
github.com/cockroachdb/cockroach/pkg/util/log_test.TestCrashReportingPacket(0xc4204d73b0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/crash_reporting_packet_test.go:93 +0x27b
testing.tRunner(0xc4204d73b0, 0x2202380)
/usr/local/go/src/testing/testing.go:777 +0xd0
created by testing.(*T).Run
/usr/local/go/src/testing/testing.go:824 +0x2e0
goroutine 320 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xdc)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62e40)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62e40)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285850, 0xc4205574d0, 0xc420285840)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 354 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xfc)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b634a0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b634a0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285d90, 0xc4205574d0, 0xc420285d80)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 123 [select]:
github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1(0xc4205574d0, 0xc420166b60)
/go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:122 +0x108
created by github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler
/go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:119 +0x49
goroutine 125 [semacquire]:
sync.runtime_notifyListWait(0xc4205462c8, 0xc400000000)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4205462b8)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc4205461a0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:647 +0x9c
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:634 +0x418
goroutine 126 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24063c0, 0xc420b4e030)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420253030, 0xc4205574d0, 0xc420253020)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 127 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24063c0, 0xc420b4e060)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420253050, 0xc4205574d0, 0xc420253040)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 128 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24063c0, 0xc42048aff0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420253070, 0xc4205574d0, 0xc420253060)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 129 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24063c0, 0xc42048b020)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420253110, 0xc4205574d0, 0xc420253100)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 130 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24063c0, 0xc42048a120)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420253150, 0xc4205574d0, 0xc420253120)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 131 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24063c0, 0xc420b60030)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba010, 0xc4205574d0, 0xc4202ba000)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 132 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24063c0, 0xc420b8c000)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba030, 0xc4205574d0, 0xc4202ba020)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 133 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24063c0, 0xc420b4e000)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba050, 0xc4205574d0, 0xc4202ba040)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 134 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24063c0, 0xc42048a0f0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba070, 0xc4205574d0, 0xc4202ba060)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 135 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24063c0, 0xc420b60000)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba090, 0xc4205574d0, 0xc4202ba080)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 136 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24063c0, 0xc420b5a000)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba0c0, 0xc4205574d0, 0xc4202ba0b0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 137 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24063c0, 0xc42048a0c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba0f0, 0xc4205574d0, 0xc4202ba0d0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 138 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24063c0, 0xc420b7a000)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba120, 0xc4205574d0, 0xc4202ba100)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 139 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24063c0, 0xc420b62000)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba150, 0xc4205574d0, 0xc4202ba130)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 140 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24063c0, 0xc420b60090)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba170, 0xc4205574d0, 0xc4202ba160)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 141 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24063c0, 0xc420b60060)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba1a0, 0xc4205574d0, 0xc4202ba190)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 142 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc420b138c0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab
goroutine 143 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc420b139e0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab
goroutine 144 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*LeaseManager).RefreshLeases.func1(0x24063c0, 0xc420b8c030)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/lease.go:1479 +0x1d1
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba270, 0xc4205574d0, 0xc420295e90)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 145 [select]:
github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps.func1(0x2204548, 0xc4202aeaf0, 0xc4203f0070, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:185 +0x13e
created by github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps
/go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:178 +0xaf
goroutine 146 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x24063c0, 0xc420b62030)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x64
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba320, 0xc4205574d0, 0xc42033dd00)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 147 [runnable]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func2(0x24063c0, 0xc420b8c090)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1074 +0x53
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba3a0, 0xc4205574d0, 0xc42033de00)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 148 [IO wait]:
internal/poll.runtime_pollWait(0x7f38cdcb6e30, 0x72, 0x0)
/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc420082598, 0x72, 0xc420b9c100, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc420082598, 0xffffffffffffff00, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Accept(0xc420082580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:372 +0x1a8
net.(*netFD).accept(0xc420082580, 0x10, 0xc420845e70, 0x1)
/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc4204b4380, 0x81f899, 0xc420b9c170, 0xc420845f00)
/usr/local/go/src/net/tcpsock_posix.go:136 +0x2e
net.(*TCPListener).Accept(0xc4204b4380, 0x22024c8, 0xc420b2cac0, 0xc4202ca010, 0x0)
/usr/local/go/src/net/tcpsock.go:259 +0x49
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*cMux).Serve(0xc420b2cac0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0x88
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func3(0x24063c0, 0xc420b8c0c0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1086 +0x2f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba3b0, 0xc4205574d0, 0xc42033de60)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 149 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc42033de20, 0xc4200ca050, 0x1e5b1a0, 0x30e16f0, 0x20ab7e0)
<autogenerated>:1 +0x5b
net/http.(*Server).Serve(0xc42079a0d0, 0x2405180, 0xc42033de20, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:2770 +0x1a5
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func4(0x24063c0, 0xc420b8c0f0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1098 +0x119
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba3c0, 0xc4205574d0, 0xc4208380c0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 150 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc42033de40, 0xc420b8c300, 0xc420b46e88, 0x91bd5a, 0x1fd9700)
<autogenerated>:1 +0x5b
crypto/tls.(*listener).Accept(0xc42033de80, 0xc4200ca050, 0x1e5b1a0, 0x30e16f0, 0x20ab7e0)
/usr/local/go/src/crypto/tls/tls.go:52 +0x37
net/http.(*Server).Serve(0xc420442000, 0x23f5e00, 0xc42033de80, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:2770 +0x1a5
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func5(0x24063c0, 0xc420b8c2a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1105 +0x42
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba3d0, 0xc4205574d0, 0xc42033dea0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 151 [runnable]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6(0x24063c0, 0xc420b8c060)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1109 +0x7c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba3e0, 0xc4205574d0, 0xc420b2cb00)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 152 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc42033dda0, 0x2203560, 0xc42002c2c0, 0x2405180, 0xc42033dda0)
<autogenerated>:1 +0x5b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve(0xc42002c2c0, 0x2405180, 0xc42033dda0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:504 +0x1d5
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func7(0x24063c0, 0xc420b8c360)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1121 +0x43
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba3f0, 0xc4205574d0, 0xc42033dec0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 153 [runnable]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func9(0x24063c0, 0xc420b8c330)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1212 +0x7c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba4c0, 0xc4205574d0, 0xc4208383f0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 155 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc420838480)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:110 +0x182
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:96 +0x3f
goroutine 156 [runnable]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func12(0x24063c0, 0xc42048b050)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1242 +0x53
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ba600, 0xc4205574d0, 0xc42035a180)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 178 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x24063c0, 0xc420b8c270)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x64
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c7c0, 0xc4205574d0, 0xc420362020)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 403 [select]:
github.com/cockroachdb/cockroach/pkg/storage/idalloc.(*Allocator).start.func1(0x24063c0, 0xc420b8db00)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/idalloc/id_alloc.go:138 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42031c060, 0xc4205574d0, 0xc42031c050)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 163 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc420b760c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x14a
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x14c
goroutine 164 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420314000)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1234 +0x235
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc420314000)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:837 +0x1b5
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:828 +0xe1
goroutine 157 [select]:
net.(*pipe).read(0xc420082680, 0xc420922000, 0x400, 0x400, 0x8, 0x8, 0x3f3)
/usr/local/go/src/net/pipe.go:164 +0x1d7
net.(*pipe).Read(0xc420082680, 0xc420922000, 0x400, 0x400, 0x1ff3f00, 0x0, 0x7f38cdbd2020)
/usr/local/go/src/net/pipe.go:147 +0x4d
crypto/tls.(*block).readFromUntil(0xc42048b0b0, 0x7f38cdbd2020, 0xc420082680, 0x5, 0xc420082680, 0x0)
/usr/local/go/src/crypto/tls/conn.go:493 +0x96
crypto/tls.(*Conn).readRecord(0xc420860000, 0x2204317, 0xc420860120, 0x0)
/usr/local/go/src/crypto/tls/conn.go:595 +0xe0
crypto/tls.(*Conn).Read(0xc420860000, 0xc420eb2000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1156 +0x100
bufio.(*Reader).Read(0xc420bcc300, 0xc4202c2658, 0x9, 0x9, 0x30, 0x28, 0x1fb0300)
/usr/local/go/src/bufio/bufio.go:216 +0x238
io.ReadAtLeast(0x23e48c0, 0xc420bcc300, 0xc4202c2658, 0x9, 0x9, 0x9, 0xc420ba2df0, 0x7f5cce, 0xc420ba2e9f)
/usr/local/go/src/io/io.go:309 +0x86
io.ReadFull(0x23e48c0, 0xc420bcc300, 0xc4202c2658, 0x9, 0x9, 0xbe16b4, 0xc420b8cba0, 0xc420b80004)
/usr/local/go/src/io/io.go:327 +0x58
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4202c2658, 0x9, 0x9, 0x23e48c0, 0xc420bcc300, 0x0, 0x0, 0xc420b8cba0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4202c2620, 0xc420b8cba0, 0xc420b8cb00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc420852300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1173 +0xe1
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:273 +0xbf1
goroutine 194 [semacquire]:
sync.runtime_notifyListWait(0xc420547b28, 0xc4000000ca)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420547b18)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc420547a00)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:647 +0x9c
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:634 +0x418
goroutine 195 [runnable]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3(0x24063c0, 0xc420b600c0)
/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:371 +0x5a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202d11f0, 0xc4205574d0, 0xc4203800a0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 196 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1(0x24063c0, 0xc420b60120)
/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1152 +0x2fc
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202d1210, 0xc4205574d0, 0xc4202d1200)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 197 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1(0x24063c0, 0xc4208384e0)
/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1190 +0x241
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202d1230, 0xc4205574d0, 0xc4202d1220)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 158 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x2406300, 0xc420b761c0, 0xc420b8c8d0, 0xc420b52fb8)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x368
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc420852300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:305 +0x5e
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:304 +0xd42
goroutine 159 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x24063c0, 0xc420b8cab0, 0xc420b8ca50, 0xc420b53fb8)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x368
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc420852480)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:261 +0x5e
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:260 +0xe08
goroutine 160 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc420852480)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:956 +0x23b
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:264 +0xe2d
goroutine 161 [select]:
net.(*pipe).read(0xc420082600, 0xc420387c00, 0x400, 0x400, 0x8, 0x8, 0x3f3)
/usr/local/go/src/net/pipe.go:164 +0x1d7
net.(*pipe).Read(0xc420082600, 0xc420387c00, 0x400, 0x400, 0x1ff3f00, 0x0, 0x7f38cdbd2020)
/usr/local/go/src/net/pipe.go:147 +0x4d
crypto/tls.(*block).readFromUntil(0xc420b8c3c0, 0x7f38cdbd2020, 0xc420082600, 0x5, 0xc420082600, 0x0)
/usr/local/go/src/crypto/tls/conn.go:493 +0x96
crypto/tls.(*Conn).readRecord(0xc420782380, 0x2204317, 0xc4207824a0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:595 +0xe0
crypto/tls.(*Conn).Read(0xc420782380, 0xc420ec8000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1156 +0x100
bufio.(*Reader).Read(0xc420bcc480, 0xc4202c2818, 0x9, 0x9, 0x30, 0x28, 0x1fb0300)
/usr/local/go/src/bufio/bufio.go:216 +0x238
io.ReadAtLeast(0x23e48c0, 0xc420bcc480, 0xc4202c2818, 0x9, 0x9, 0x9, 0xc420ba3d88, 0x7f5cce, 0xc420ba3e37)
/usr/local/go/src/io/io.go:309 +0x86
io.ReadFull(0x23e48c0, 0xc420bcc480, 0xc4202c2818, 0x9, 0x9, 0xbe16b4, 0xc420b8cb70, 0xc420b80004)
/usr/local/go/src/io/io.go:327 +0x58
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4202c2818, 0x9, 0x9, 0x23e48c0, 0xc420bcc480, 0x0, 0x0, 0xc420b8cb70, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4202c27e0, 0xc420b8cb70, 0xc420b8cb70, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420852480, 0xc420b8cb40, 0x22035b0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:395 +0x37
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc42002c2c0, 0x2416b00, 0xc420852480)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:634 +0xeb
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2()
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:591 +0x3c
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc42034a9e0, 0xc42002c2c0, 0x7f38cdc288e0, 0xc420852480)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:599 +0x27
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:598 +0x5f9
goroutine 319 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xdb)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62e10)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62e10)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285830, 0xc4205574d0, 0xc420285820)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 314 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xd3)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62d20)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62d20)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285770, 0xc4205574d0, 0xc4202856f0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 313 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xd5)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62cf0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62cf0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202856e0, 0xc4205574d0, 0xc4202856c0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 312 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xd7)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62cc0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62cc0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202856b0, 0xc4205574d0, 0xc420285690)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 311 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xd4)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62c90)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62c90)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285680, 0xc4205574d0, 0xc420285660)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 310 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xd2)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62c60)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62c60)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285650, 0xc4205574d0, 0xc420285640)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 309 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xd1)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62c30)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62c30)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285620, 0xc4205574d0, 0xc420285610)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 322 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xdf)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62ea0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62ea0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285890, 0xc4205574d0, 0xc420285880)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 307 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xce)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62bd0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62bd0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202855c0, 0xc4205574d0, 0xc4202855b0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 306 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xcf)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62ba0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62ba0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285590, 0xc4205574d0, 0xc420285580)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 305 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xcd)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62b70)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62b70)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285570, 0xc4205574d0, 0xc420285550)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 304 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xcc)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62b40)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62b40)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285540, 0xc4205574d0, 0xc420285530)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 303 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xcb)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62b10)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62b10)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285520, 0xc4205574d0, 0xc420285510)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 302 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0x108)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62ae0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62ae0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285500, 0xc4205574d0, 0xc4202854f0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 301 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xca)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62ab0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62ab0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202854e0, 0xc4205574d0, 0xc4202854d0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
c7291 runs completed, 1 failures, over 9m22s
reated by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 300 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0x106)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62a80)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62a80)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202854c0, 0xc4205574d0, 0xc4202854b0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 299 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0x107)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62a50)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62a50)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202854a0, 0xc4205574d0, 0xc420285490)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 298 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0x105)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62a20)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62a20)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285480, 0xc4205574d0, 0xc420285470)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 297 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0x104)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b629f0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b629f0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285460, 0xc4205574d0, 0xc420285450)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 296 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0x103)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b629c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b629c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285440, 0xc4205574d0, 0xc420285430)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 295 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xff)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62990)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62990)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285420, 0xc4205574d0, 0xc420285410)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 294 [runnable]:
github.com/cockroachdb/cockroach/vendor/github.com/coreos/etcd/raft.(*raft).becomePreCandidate(0xc4201d3100)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/coreos/etcd/raft/raft.go:671 +0x22a
github.com/cockroachdb/cockroach/vendor/github.com/coreos/etcd/raft.(*raft).campaign(0xc4201d3100, 0x20fb539, 0x13)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/coreos/etcd/raft/raft.go:718 +0x6d9
github.com/cockroachdb/cockroach/vendor/github.com/coreos/etcd/raft.(*raft).Step(0xc4201d3100, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/coreos/etcd/raft/raft.go:851 +0x2ff
github.com/cockroachdb/cockroach/vendor/github.com/coreos/etcd/raft.(*RawNode).Campaign(0xc42153ea40, 0xc42157d7d0, 0x3)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/coreos/etcd/raft/rawnode.go:140 +0x61
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).maybeCampaignOnWakeLocked(0xc421720a80, 0x24063c0, 0xc42157d7d0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:603 +0x2ae
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).withRaftGroupLocked(0xc421720a80, 0xc4215398d0, 0xc421720a80, 0x24063c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:557 +0x2f4
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc421720a80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3502 +0x1b9
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc421720a80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3457 +0xdb
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420071c00, 0x24063c0, 0xc420b62960, 0x14)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3601 +0xda
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62960)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:215 +0x259
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62960)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285400, 0xc4205574d0, 0xc4202853f0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 308 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xd0)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62c00)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62c00)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202855f0, 0xc4205574d0, 0xc4202855e0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 292 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0x100)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b62900)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b62900)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202853c0, 0xc4205574d0, 0xc4202853b0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 291 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1(0x24063c0, 0xc420b628d0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:156 +0x49
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202853a0, 0xc4205574d0, 0xc4203630a0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 353 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xfb)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b63470)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b63470)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285d60, 0xc4205574d0, 0xc420285d50)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 352 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xfe)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b63440)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24063c0, 0xc420b63440)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420285d40, 0xc4205574d0, 0xc420285d30)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 351 [semacquire]:
sync.runtime_notifyListWait(0xc420b77550, 0xfd)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420b77540)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420557320, 0x24063c0, 0xc420b63410)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c