sql/opt/exec/execbuilder: TestExecBuild/explain failed under stress #25566
Description
SHA: https://github.com/cockroachdb/cockroach/commits/6f4e9b215f9a3284a39d507b3b0ddd310a0f87da
Parameters:
TAGS=
GOFLAGS=
Stress build found a failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=660366&tab=buildLog
I180516 05:57:25.806394 3977 sql/event_log.go:124 [n1,client=127.0.0.1:38656,user=root] Event: "create_database", target: 50, info: {DatabaseName:test Statement:CREATE DATABASE test User:root}
I180516 05:57:25.811763 3977 sql/event_log.go:124 [n1,client=127.0.0.1:38656,user=root] Event: "create_table", target: 51, info: {TableName:test.public.xy Statement:CREATE TABLE xy (x INT PRIMARY KEY, y INT) User:root}
testdata/explain:1:
CREATE TABLE xy (x INT PRIMARY KEY, y INT)
----
testdata/explain:5:
EXPLAIN SELECT * FROM xy
----
Tree:string Field:string Description:string
scan · ·
· table xy@primary
· spans ALL
testdata/explain:13:
EXPLAIN (TYPES) SELECT * FROM xy
----
Tree:string Field:string Description:string Columns:string Ordering:string
scan · · (x int, y int) ·
· table xy@primary · ·
· spans ALL · ·
testdata/explain:21:
EXPLAIN (VERBOSE) SELECT * FROM xy
----
Tree:string Field:string Description:string Columns:string Ordering:string
scan · · (x, y) ·
· table xy@primary · ·
· spans ALL · ·
testdata/explain:29:
EXPLAIN (VERBOSE) SELECT x, x, y FROM xy
----
Tree:string Field:string Description:string Columns:string Ordering:string
render · · (x, x, y) ·
│ render 0 x · ·
│ render 1 x · ·
│ render 2 y · ·
└── scan · · (x, y) ·
· table xy@primary · ·
· spans ALL · ·
testdata/explain:41:
EXPLAIN (VERBOSE) SELECT * FROM xy ORDER BY y
----
Tree:string Field:string Description:string Columns:string Ordering:string
sort · · (x, y) +y
│ order +y · ·
└── scan · · (x, y) ·
· table xy@primary · ·
· spans ALL · ·
testdata/explain:51:
EXPLAIN (VERBOSE) SELECT x, x, y FROM xy ORDER BY y
----
Tree:string Field:string Description:string Columns:string Ordering:string
render · · (x, x, y) ·
│ render 0 x · ·
│ render 1 x · ·
│ render 2 y · ·
└── sort · · (x, y) +y
│ order +y · ·
└── scan · · (x, y) ·
· table xy@primary · ·
· spans ALL · ·
testdata/explain:65:
EXPLAIN (VERBOSE) SELECT * FROM xy INNER JOIN (VALUES (1, 2), (3, 4)) AS t(u,v) ON x=u
----
Tree:string Field:string Description:string Columns:string Ordering:string
join · · (x, y, u, v) ·
│ type inner · ·
│ equality (x) = (column1) · ·
├── scan · · (x, y) ·
│ table xy@primary · ·
│ spans ALL · ·
└── values · · (column1, column2) ·
· size 2 columns, 2 rows · ·
· row 0, expr 0 1 · ·
· row 0, expr 1 2 · ·
· row 1, expr 0 3 · ·
· row 1, expr 1 4 · ·
testdata/explain:82:
SELECT "Tree", "Columns" FROM [ EXPLAIN (VERBOSE) SELECT * FROM xy ORDER BY y ]
----
Tree:string Columns:string
sort (x, y)
│ ·
└── scan (x, y)
· ·
· ·
testdata/explain:92:
SELECT "Level", "Type" FROM [ EXPLAIN (VERBOSE) SELECT * FROM xy ORDER BY y ]
----
Level:int Type:string
0 sort
0 ·
1 scan
1 ·
1 ·
I180516 05:57:25.822759 4035 util/stop/stopper.go:471 quiescing; tasks left:
1 [async] auto-upgrade
W180516 05:57:25.823904 3588 gossip/infostore.go:303 [n2] node unavailable; try another peer
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x8 pc=0x101b005]
goroutine 3990 [running]:
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.(*DString).Size(0x0, 0xc4216283e8)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/sem/tree/datum.go:933 +0x5
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowContainer).rowSize(0xc4219963f0, 0xc4216284c8, 0x1, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/row_container.go:247 +0x66
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowContainer).AddRow(0xc4219963f0, 0x2408700, 0xc420ebc9f0, 0xc4216284c8, 0x1, 0x1, 0x7, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/row_container.go:272 +0x7c
github.com/cockroachdb/cockroach/pkg/sql.(*planner).ShowClusterSetting.func1(0x2408700, 0xc420ebc9f0, 0xc4220b1d20, 0x0, 0x0, 0x2204d00, 0xc421a76368)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/show_cluster_setting.go:161 +0x186
github.com/cockroachdb/cockroach/pkg/sql.doExpandPlan(0x2408700, 0xc420ebc9f0, 0xc4220b1d20, 0x7fffffffffffffff, 0x0, 0x0, 0x0, 0x0, 0x23feb80, 0xc4205a8980, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/expand_plan.go:320 +0x2c51
github.com/cockroachdb/cockroach/pkg/sql.(*planner).expandPlan(0xc4220b1d20, 0x2408700, 0xc420ebc9f0, 0x23feb80, 0xc4205a8980, 0x23feb80, 0xc4205a8980, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/expand_plan.go:35 +0xce
github.com/cockroachdb/cockroach/pkg/sql.(*planner).optimizePlan(0xc4220b1d20, 0x2408700, 0xc420ebc9f0, 0x23feb80, 0xc4205a8980, 0xc42038e292, 0x1, 0x1, 0x23feb80, 0xc4205a8980, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/optimize.go:47 +0xeb
github.com/cockroachdb/cockroach/pkg/sql.(*planner).makePlan(0xc4220b1d20, 0x2408700, 0xc420ebc9f0, 0x240ad40, 0xc421d7c1a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:324 +0x2ea
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc4220b1900, 0x2408700, 0xc420ebc9f0, 0x240ad40, 0xc421d7c1a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:578 +0xb28
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc4220b1900, 0x2408700, 0xc420ebc9f0, 0x240ad40, 0xc421d7c1a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:397 +0xb33
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc4220b1900, 0x2408700, 0xc420ebc9f0, 0x240ad40, 0xc421d7c1a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:91 +0x358
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc4220b1900, 0x2408700, 0xc420ebc5d0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1051 +0x2039
github.com/cockroachdb/cockroach/pkg/sql.(*internalExecutorImpl).initConnEx.func1(0xc4220b1900, 0x2408700, 0xc420ebc5d0, 0xc4224ce380, 0xc42038e2c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/internal.go:211 +0x86
created by github.com/cockroachdb/cockroach/pkg/sql.(*internalExecutorImpl).initConnEx
/go/src/github.com/cockroachdb/cockroach/pkg/sql/internal.go:200 +0x268
goroutine 1 [chan receive]:
testing.(*T).Run(0xc4207760f0, 0x20f613d, 0xd, 0x2202f00, 0x857e7c)
/usr/local/go/src/testing/testing.go:825 +0x301
testing.runTests.func1(0xc420776000)
/usr/local/go/src/testing/testing.go:1063 +0x64
testing.tRunner(0xc420776000, 0xc4206abdd8)
/usr/local/go/src/testing/testing.go:777 +0xd0
testing.runTests(0xc420550140, 0x30ea030, 0x1, 0x1, 0xbeb70ff0f6e355bb)
/usr/local/go/src/testing/testing.go:1061 +0x2c4
testing.(*M).Run(0xc420504100, 0x0)
/usr/local/go/src/testing/testing.go:978 +0x171
github.com/cockroachdb/cockroach/pkg/sql/opt/exec/execbuilder_test.TestMain(0xc420504100)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/opt/exec/execbuilder/main_test.go:37 +0xb8
main.main()
_testmain.go:42 +0x151
goroutine 20 [syscall]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.0
/usr/local/go/src/os/signal/signal_unix.go:28 +0x41
goroutine 11 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1158 +0xf1
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:591 +0x110
goroutine 12 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:598 +0xab
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:592 +0x128
goroutine 22 [select, locked to thread]:
runtime.gopark(0x2206b28, 0x0, 0x20eb5ed, 0x6, 0x18, 0x1)
/usr/local/go/src/runtime/proc.go:291 +0x11a
runtime.selectgo(0xc42048ff50, 0xc4204de4e0)
/usr/local/go/src/runtime/select.go:392 +0xe50
runtime.ensureSigM.func1()
/usr/local/go/src/runtime/signal_unix.go:549 +0x1f4
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1
goroutine 2939 [semacquire]:
sync.runtime_Semacquire(0xc42093de4c)
/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc42093de40)
/usr/local/go/src/sync/waitgroup.go:129 +0x72
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).stopServers(0xc420fa6070)
/go/src/github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:98 +0x10b
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).(github.com/cockroachdb/cockroach/pkg/testutils/testcluster.stopServers)-fm()
/go/src/github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:166 +0x2a
github.com/cockroachdb/cockroach/pkg/util/stop.CloserFn.Close(0xc420558110)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:91 +0x25
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Stop(0xc420cae750, 0x2408680, 0xc4200cc050)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:422 +0x29d
github.com/cockroachdb/cockroach/pkg/sql/opt/exec/execbuilder_test.TestExecBuild.func1(0xc42032ca50, 0xc4203a8540, 0x10)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/opt/exec/execbuilder/builder_test.go:189 +0x355
github.com/cockroachdb/cockroach/pkg/testutils/datadriven.Walk(0xc42032ca50, 0xc4203a8540, 0x10, 0x2202ef8)
/go/src/github.com/cockroachdb/cockroach/pkg/testutils/datadriven/datadriven.go:142 +0x329
github.com/cockroachdb/cockroach/pkg/testutils/datadriven.Walk.func1(0xc42032ca50)
/go/src/github.com/cockroachdb/cockroach/pkg/testutils/datadriven/datadriven.go:151 +0xd2
testing.tRunner(0xc42032ca50, 0xc420c083c0)
/usr/local/go/src/testing/testing.go:777 +0xd0
created by testing.(*T).Run
/usr/local/go/src/testing/testing.go:824 +0x2e0
goroutine 14 [chan receive]:
testing.(*T).Run(0xc42032ca50, 0xc42054e0f9, 0x7, 0xc420c083c0, 0x1)
/usr/local/go/src/testing/testing.go:825 +0x301
github.com/cockroachdb/cockroach/pkg/testutils/datadriven.Walk(0xc4207760f0, 0x20ef220, 0x8, 0x2202ef8)
/go/src/github.com/cockroachdb/cockroach/pkg/testutils/datadriven/datadriven.go:150 +0x1c5
github.com/cockroachdb/cockroach/pkg/sql/opt/exec/execbuilder_test.TestExecBuild(0xc4207760f0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/opt/exec/execbuilder/builder_test.go:78 +0x84
testing.tRunner(0xc4207760f0, 0x2202f00)
/usr/local/go/src/testing/testing.go:777 +0xd0
created by testing.(*T).Run
/usr/local/go/src/testing/testing.go:824 +0x2e0
goroutine 3281 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1(0x2408700, 0xc42279e7e0)
/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1190 +0x241
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421d7c8d0, 0xc420cae7e0, 0xc421d7c8c0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3892 [semacquire]:
sync.runtime_notifyListWait(0xc421b0ddd0, 0xc400000004)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421b0ddc0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420c282d0, 0x2408700, 0xc421fa5aa0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc421fa5aa0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4205a0c90, 0xc420cae090, 0xc4205a0c80)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3676 [semacquire]:
sync.runtime_notifyListWait(0xc420ee8210, 0xc40000003d)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420ee8200)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066c1b0, 0x2408700, 0xc420e83710)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420e83710)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420ea5720, 0xc4222245a0, 0xc420ea5710)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2940 [select]:
github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1(0xc420cae7e0, 0xc420fa6310)
/go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:122 +0x108
created by github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler
/go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:119 +0x49
goroutine 3378 [semacquire]:
sync.runtime_notifyListWait(0xc4204ead50, 0x48)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4204ead40)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066cb40, 0x2408700, 0xc421a5aba0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc421a5aba0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420542810, 0xc420cae7e0, 0xc4205427b0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3459 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc420f9c750, 0x2408700, 0xc421fa50e0, 0xc420cae7e0, 0x30c7f47f, 0xed28dbf45, 0x0, 0xc420fb2000, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:212 +0x212
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc420fb2000, 0xc421e78a20)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:274 +0x81
github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc421e78a20, 0xc4207e4c80)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:322 +0x1a6
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1(0x2408700, 0xc421fa50e0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:272 +0x1c0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043d280, 0xc420cae7e0, 0xc4203df0a0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3823 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1(0x2408700, 0xc420e82060)
/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1190 +0x241
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42057c5d0, 0xc420cae090, 0xc42057c5c0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3498 [select]:
github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1(0xc420cae090, 0xc4203309a0)
/go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:122 +0x108
created by github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler
/go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:119 +0x49
goroutine 60 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc42069cfc0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab
goroutine 61 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc42069d0e0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab
goroutine 3721 [semacquire]:
sync.runtime_notifyListWait(0xc420ee8210, 0xc400000034)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420ee8200)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066c1b0, 0x2408700, 0xc420e83440)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420e83440)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420ea5d00, 0xc4222245a0, 0xc420ea5cf0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3955 [semacquire]:
sync.runtime_Semacquire(0xc420c28354)
/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc420c28348)
/usr/local/go/src/sync/waitgroup.go:129 +0x72
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc420c282d0, 0x2408700, 0xc420ed4690)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:172 +0x31
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait-fm(0x2408700, 0xc420ed4690)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3655 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4205a1570, 0xc420cae090, 0xc4205a1560)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3703 [semacquire]:
sync.runtime_notifyListWait(0xc420ee8210, 0xc400000013)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420ee8200)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066c1b0, 0x2408700, 0xc420e82d50)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420e82d50)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420ea5aa0, 0xc4222245a0, 0xc420ea5a90)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3666 [semacquire]:
sync.runtime_notifyListWait(0xc420ee8210, 0xc40000000c)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420ee8200)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066c1b0, 0x2408700, 0xc420ed5f20)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420ed5f20)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420ea55e0, 0xc4222245a0, 0xc420ea55d0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3977 [semacquire]:
sync.runtime_notifyListWait(0xc42109eb50, 0xc400000002)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42109eb40)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).curCmd(0xc420f99ea0, 0x0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:424 +0x8a
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc4212d6000, 0x2408640, 0xc42109ebc0, 0xc4200c1e60, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1019 +0x3d2
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc421506900, 0x2408640, 0xc42109ebc0, 0x0, 0x0, 0xc42060a022, 0x4, 0x0, 0x0, 0x23f2940, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:401 +0x1bb
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc421506900, 0x2408640, 0xc42109ebc0, 0xc420413180, 0x5400, 0x15000, 0xc4225d2840, 0xc4200c1e60, 0xc4200c1e40, 0xc421b0e730, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:266 +0x122
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:265 +0xf27
goroutine 3655 [select]:
github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).Start.func2(0x2408700, 0xc420015bc0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:269 +0x127
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420558f50, 0xc4222245a0, 0xc4209d4090)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3653 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1(0x2408700, 0xc420014d50)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:473 +0x264
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420558f20, 0xc4222245a0, 0xc420c252c0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3799 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc4205a53b0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420407e90, 0xc420cae090, 0xc420407e60)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3556 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc4210f96b0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4205a1910, 0xc4222245a0, 0xc4205a1900)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3833 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x2408700, 0xc420ea0210)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:580 +0x194
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204d6140, 0xc420cae090, 0xc421bb4100)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3255 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc421703a40)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421d7c150, 0xc420cae7e0, 0xc421d7c140)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3725 [semacquire]:
sync.runtime_notifyListWait(0xc420ee8210, 0xc400000038)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420ee8200)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066c1b0, 0x2408700, 0xc420e83500)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420e83500)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420ea5d80, 0xc4222245a0, 0xc420ea5d70)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3794 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc4205a5170)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420407cf0, 0xc420cae090, 0xc420407ce0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3254 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc42279e5a0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421d7c130, 0xc420cae7e0, 0xc421d7c120)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3680 [semacquire]:
sync.runtime_notifyListWait(0xc420ee8210, 0xc400000003)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420ee8200)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066c1b0, 0x2408700, 0xc420ed54a0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420ed54a0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420ea57a0, 0xc4222245a0, 0xc420ea5790)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3681 [semacquire]:
sync.runtime_notifyListWait(0xc420ee8210, 0xc400000004)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420ee8200)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066c1b0, 0x2408700, 0xc420ed54d0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420ed54d0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420ea57c0, 0xc4222245a0, 0xc420ea57b0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3682 [semacquire]:
sync.runtime_notifyListWait(0xc420ee8210, 0xc400000005)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420ee8200)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066c1b0, 0x2408700, 0xc420ed5500)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420ed5500)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420ea57e0, 0xc4222245a0, 0xc420ea57d0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3683 [semacquire]:
sync.runtime_notifyListWait(0xc420ee8210, 0xc400000006)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420ee8200)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066c1b0, 0x2408700, 0xc420ed5530)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420ed5530)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420ea5800, 0xc4222245a0, 0xc420ea57f0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3684 [semacquire]:
sync.runtime_notifyListWait(0xc420ee8210, 0xc400000007)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420ee8200)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066c1b0, 0x2408700, 0xc420ed5560)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420ed5560)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420ea5820, 0xc4222245a0, 0xc420ea5810)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3685 [semacquire]:
sync.runtime_notifyListWait(0xc420ee8210, 0xc400000008)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420ee8200)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066c1b0, 0x2408700, 0xc420ed5590)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420ed5590)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420ea5840, 0xc4222245a0, 0xc420ea5830)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3950 [semacquire]:
sync.runtime_notifyListWait(0xc421b0ddd0, 0xc40000003b)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421b0ddc0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420c282d0, 0x2408700, 0xc420ed4540)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420ed4540)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4205a14c0, 0xc420cae090, 0xc4205a14b0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3262 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc42279e510)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421d7c250, 0xc420cae7e0, 0xc421d7c240)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3379 [semacquire]:
sync.runtime_notifyListWait(0xc4204ead50, 0x49)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4204ead40)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066cb40, 0x2408700, 0xc421a5abd0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc421a5abd0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420542840, 0xc420cae7e0, 0xc420542820)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3380 [semacquire]:
sync.runtime_notifyListWait(0xc4204ead50, 0x4a)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4204ead40)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066cb40, 0x2408700, 0xc421a5ac00)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc421a5ac00)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420542870, 0xc420cae7e0, 0xc420542850)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3416 [semacquire]:
sync.runtime_notifyListWait(0xc4204ead50, 0x6e)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4204ead40)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066cb40, 0x2408700, 0xc421a5b2f0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc421a5b2f0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420543140, 0xc420cae7e0, 0xc420543130)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3417 [semacquire]:
sync.runtime_notifyListWait(0xc4204ead50, 0x6f)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4204ead40)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066cb40, 0x2408700, 0xc421a5b320)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc421a5b320)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420543160, 0xc420cae7e0, 0xc420543150)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3376 [semacquire]:
sync.runtime_notifyListWait(0xc4204ead50, 0x46)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4204ead40)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066cb40, 0x2408700, 0xc421a5ab40)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc421a5ab40)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420542740, 0xc420cae7e0, 0xc420542720)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3263 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc420c4eea0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab
goroutine 2945 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc42279e390)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421d7c090, 0xc420cae7e0, 0xc421d7c080)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3261 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc42279e4b0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421d7c210, 0xc420cae7e0, 0xc421d7c200)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3414 [semacquire]:
sync.runtime_notifyListWait(0xc4204ead50, 0x6c)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4204ead40)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066cb40, 0x2408700, 0xc421a5b290)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc421a5b290)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4205430d0, 0xc420cae7e0, 0xc4205430b0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3809 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x2408700, 0xc4205a5620)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x64
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4200f9a10, 0xc420cae090, 0xc42041a6a0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3250 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc42279e3c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421d7c0b0, 0xc420cae7e0, 0xc421d7c0a0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3253 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc42279e540)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421d7c110, 0xc420cae7e0, 0xc421d7c100)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3252 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc42279e570)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421d7c0f0, 0xc420cae7e0, 0xc421d7c0e0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3251 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc42279e330)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421d7c0d0, 0xc420cae7e0, 0xc421d7c0c0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3678 [semacquire]:
sync.runtime_notifyListWait(0xc420ee8210, 0xc40000003f)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420ee8200)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066c1b0, 0x2408700, 0xc42103dc80)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc42103dc80)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420ea5760, 0xc4222245a0, 0xc420ea5750)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3643 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func2(0x2408700, 0xc4209d4ba0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:112 +0x2af
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420558cd0, 0xc4222245a0, 0xc421dc8f00)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3945 [semacquire]:
sync.runtime_notifyListWait(0xc421b0ddd0, 0xc400000036)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421b0ddc0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420c282d0, 0x2408700, 0xc420ed4450)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420ed4450)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4205a1420, 0xc420cae090, 0xc4205a1410)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3372 [semacquire]:
sync.runtime_notifyListWait(0xc4204ead50, 0x82)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4204ead40)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066cb40, 0x2408700, 0xc421a5aa80)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc421a5aa80)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421d7dfe0, 0xc420cae7e0, 0xc421d7dfd0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3807 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*LeaseManager).RefreshLeases.func1(0x2408700, 0xc421fa4ae0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/lease.go:1479 +0x1d1
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4200f9920, 0xc420cae090, 0xc4211052c0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3947 [semacquire]:
sync.runtime_notifyListWait(0xc421b0ddd0, 0xc400000038)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421b0ddc0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420c282d0, 0x2408700, 0xc420ed44b0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420ed44b0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4205a1460, 0xc420cae090, 0xc4205a1450)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3679 [semacquire]:
sync.runtime_notifyListWait(0xc420ee8210, 0xc400000002)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420ee8200)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066c1b0, 0x2408700, 0xc420ed5470)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420ed5470)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420ea5780, 0xc4222245a0, 0xc420ea5770)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3687 [semacquire]:
sync.runtime_notifyListWait(0xc420ee8210, 0xc40000000a)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420ee8200)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066c1b0, 0x2408700, 0xc420ed55f0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420ed55f0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420ea5880, 0xc4222245a0, 0xc420ea5870)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3691 [semacquire]:
sync.runtime_notifyListWait(0xc420ee8210, 0xc40000002c)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420ee8200)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066c1b0, 0x2408700, 0xc420e832c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420e832c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420ea5920, 0xc4222245a0, 0xc420ea5910)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3449 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func2(0x2408700, 0xc421fa52f0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:112 +0x2af
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420543af0, 0xc420cae7e0, 0xc4203bb660)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3688 [semacquire]:
sync.runtime_notifyListWait(0xc420ee8210, 0xc400000001)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420ee8200)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066c1b0, 0x2408700, 0xc420ed5440)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420ed5440)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420ea58a0, 0xc4222245a0, 0xc420ea5890)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3500 [semacquire]:
sync.runtime_notifyListWait(0xc4224c42c8, 0xc400000000)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4224c42b8)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc4224c41a0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:647 +0x9c
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:634 +0x418
goroutine 3444 [select]:
github.com/cockroachdb/cockroach/pkg/storage/compactor.(*Compactor).Start.func1(0x2408700, 0xc42279f5c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/compactor/compactor.go:108 +0x1b0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4205436c0, 0xc420cae7e0, 0xc421e79650)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3260 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc42279e4e0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421d7c1f0, 0xc420cae7e0, 0xc421d7c1e0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3259 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc42279e3f0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421d7c1d0, 0xc420cae7e0, 0xc421d7c1c0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3258 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc42279e480)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421d7c1b0, 0xc420cae7e0, 0xc421d7c1a0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3257 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc42279e450)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421d7c190, 0xc420cae7e0, 0xc421d7c180)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3256 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc42279e420)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421d7c170, 0xc420cae7e0, 0xc421d7c160)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3926 [semacquire]:
sync.runtime_notifyListWait(0xc421b0ddd0, 0xc400000024)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421b0ddc0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420c282d0, 0x2408700, 0xc420ed40c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc420ed40c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4205a11c0, 0xc420cae090, 0xc4205a11b0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3886 [select]:
github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).Start.func2(0x2408700, 0xc422ca9560)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:269 +0x127
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420558f20, 0xc420cae090, 0xc422ca90b0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2942 [semacquire]:
sync.runtime_notifyListWait(0xc420c3a608, 0xc400000000)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c3a5f8)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc420c3a4e0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:647 +0x9c
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:634 +0x418
goroutine 3814 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6(0x2408700, 0xc421fa4db0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1112 +0xcf
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4200f9ae0, 0xc420cae090, 0xc4200d3e80)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3798 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc4205a5350)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420407e50, 0xc420cae090, 0xc420407e20)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3559 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4209c39e0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab
goroutine 3558 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc4210f9740)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4205a1950, 0xc4222245a0, 0xc4205a1940)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2943 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2408700, 0xc42279e300)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421d7c050, 0xc420cae7e0, 0xc421d7c040)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3415 [semacquire]:
sync.runtime_notifyListWait(0xc4204ead50, 0x6d)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4204ead40)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42066cb40, 0x2408700, 0xc421a5b2c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2408700, 0xc421a5b2c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420543120, 0xc420cae7e0, 0xc4205430e0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3561 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*LeaseManager).RefreshLeases.func1(0x2408700, 0xc420f12000)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/lease.go:1479 +0x1d1
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4205a19b0, 0xc4222245a0, 0xc4210539e0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 3381 [semacquire]:
sync.runtime_notifyListWait(0xc4204ead50, 0x4b)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4204ead40)
/usr/local/go/src/sync/cond.go:56 +0x80