sql/distsqlrun: TestSyncFlowAfterDrain failed under stress #25603
Description
SHA: https://github.com/cockroachdb/cockroach/commits/172a7b3aba77dc21e1bdebda2945f1e99fe967af
Parameters:
TAGS=
GOFLAGS=
Stress build found a failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=662436&tab=buildLog
I180517 05:39:35.305892 2646 server/config.go:539 [n?] 1 storage engine initialized
I180517 05:39:35.305941 2646 server/config.go:542 [n?] RocksDB cache size: 128 MiB
I180517 05:39:35.305962 2646 server/config.go:542 [n?] store 0: in-memory, size 0 B
I180517 05:39:35.309753 2646 server/node.go:376 [n?] **** cluster 004b645f-dffb-4513-a10c-dfc0e2f33ca5 has been created
I180517 05:39:35.309783 2646 server/server.go:1356 [n?] **** add additional nodes by specifying --join=127.0.0.1:43113
I180517 05:39:35.310446 2646 storage/store.go:1440 [n1,s1] [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I180517 05:39:35.310542 2646 server/node.go:506 [n1] initialized store [n1,s1]: disk (capacity=512 MiB, available=512 MiB, used=0 B, logicalBytes=6.9 KiB), ranges=1, leases=0, writes=0.00, bytesPerReplica={p10=7043.00 p25=7043.00 p50=7043.00 p75=7043.00 p90=7043.00 pMax=7043.00}, writesPerReplica={p10=0.00 p25=0.00 p50=0.00 p75=0.00 p90=0.00 pMax=0.00}
I180517 05:39:35.310592 2646 server/node.go:354 [n1] node ID 1 initialized
I180517 05:39:35.310667 2646 gossip/gossip.go:333 [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:43113" > attrs:<> locality:<> ServerVersion:<major_val:2 minor_val:0 patch:0 unstable:3 >
I180517 05:39:35.310910 2646 storage/stores.go:222 [n1] read 0 node addresses from persistent storage
I180517 05:39:35.310998 2646 server/node.go:647 [n1] connecting to gossip network to verify cluster ID...
I180517 05:39:35.311022 2646 server/node.go:672 [n1] node connected via gossip and verified as part of cluster "004b645f-dffb-4513-a10c-dfc0e2f33ca5"
I180517 05:39:35.311043 2646 server/node.go:440 [n1] node=1: started with [<no-attributes>=<in-mem>] engine(s) and attributes []
I180517 05:39:35.311316 2646 server/server.go:1483 [n1] starting https server at 127.0.0.1:37013
I180517 05:39:35.311338 2646 server/server.go:1484 [n1] starting grpc/postgres server at 127.0.0.1:43113
I180517 05:39:35.311363 2646 server/server.go:1485 [n1] advertising CockroachDB node at 127.0.0.1:43113
I180517 05:39:35.315363 2928 storage/replica_command.go:863 [split,n1,s1,r1/1:/M{in-ax}] initiating a split of this range at key /System/"" [r2]
I180517 05:39:35.324726 2985 storage/replica_command.go:863 [split,n1,s1,r2/1:/{System/-Max}] initiating a split of this range at key /System/NodeLiveness [r3]
I180517 05:39:35.330704 2991 storage/replica_command.go:863 [split,n1,s1,r3/1:/{System/NodeL…-Max}] initiating a split of this range at key /System/NodeLivenessMax [r4]
I180517 05:39:35.343739 2766 storage/replica_command.go:863 [split,n1,s1,r4/1:/{System/NodeL…-Max}] initiating a split of this range at key /System/tsd [r5]
I180517 05:39:35.365983 2996 storage/replica_command.go:863 [split,n1,s1,r5/1:/{System/tsd-Max}] initiating a split of this range at key /System/"tse" [r6]
I180517 05:39:35.392444 2974 storage/replica_command.go:863 [split,n1,s1,r6/1:/{System/tse-Max}] initiating a split of this range at key /Table/SystemConfigSpan/Start [r7]
I180517 05:39:35.400463 3026 storage/replica_command.go:863 [split,n1,s1,r7/1:/{Table/System…-Max}] initiating a split of this range at key /Table/11 [r8]
W180517 05:39:35.403737 2719 storage/intent_resolver.go:320 [n1,s1] failed to push during intent resolution: failed to push "sql txn" id=b8813997 key=/Table/SystemConfigSpan/Start rw=true pri=0.03288891 iso=SERIALIZABLE stat=PENDING epo=0 ts=1526535575.393946503,0 orig=1526535575.393946503,0 max=1526535575.393946503,0 wto=false rop=false seq=7
I180517 05:39:35.405673 2999 sql/event_log.go:124 [n1,intExec=optInToDiagnosticsStatReporting] Event: "set_cluster_setting", target: 0, info: {SettingName:diagnostics.reporting.enabled Value:true User:root}
I180517 05:39:35.412557 3004 storage/replica_command.go:863 [split,n1,s1,r8/1:/{Table/11-Max}] initiating a split of this range at key /Table/12 [r9]
I180517 05:39:35.419827 3025 storage/replica_command.go:863 [split,n1,s1,r9/1:/{Table/12-Max}] initiating a split of this range at key /Table/13 [r10]
I180517 05:39:35.423024 3023 sql/event_log.go:124 [n1,intExec=set-setting] Event: "set_cluster_setting", target: 0, info: {SettingName:version Value:$1 User:root}
I180517 05:39:35.427703 3077 storage/replica_command.go:863 [split,n1,s1,r10/1:/{Table/13-Max}] initiating a split of this range at key /Table/14 [r11]
I180517 05:39:35.435411 2931 storage/replica_command.go:863 [split,n1,s1,r11/1:/{Table/14-Max}] initiating a split of this range at key /Table/15 [r12]
I180517 05:39:35.435542 3074 sql/event_log.go:124 [n1,intExec=disableNetTrace] Event: "set_cluster_setting", target: 0, info: {SettingName:trace.debug.enable Value:false User:root}
W180517 05:39:35.443574 3089 storage/intent_resolver.go:320 [n1,s1] failed to push during intent resolution: failed to push "split" id=cdc9703d key=/Local/Range/Table/14/RangeDescriptor rw=true pri=0.04735170 iso=SERIALIZABLE stat=PENDING epo=0 ts=1526535575.435430478,0 orig=1526535575.435430478,0 max=1526535575.435430478,0 wto=false rop=false seq=3
I180517 05:39:35.445340 3107 storage/replica_command.go:863 [split,n1,s1,r12/1:/{Table/15-Max}] initiating a split of this range at key /Table/16 [r13]
I180517 05:39:35.452599 2937 storage/replica_command.go:863 [split,n1,s1,r13/1:/{Table/16-Max}] initiating a split of this range at key /Table/17 [r14]
W180517 05:39:35.458670 3104 storage/intent_resolver.go:320 [n1,s1] failed to push during intent resolution: failed to push "split" id=c18a39c9 key=/Local/Range/Table/16/RangeDescriptor rw=true pri=0.02283566 iso=SERIALIZABLE stat=PENDING epo=0 ts=1526535575.452611246,0 orig=1526535575.452611246,0 max=1526535575.452611246,0 wto=false rop=false seq=3
W180517 05:39:35.464572 3122 storage/intent_resolver.go:320 [n1,s1] failed to push during intent resolution: failed to push "split" id=c18a39c9 key=/Local/Range/Table/16/RangeDescriptor rw=true pri=0.02283566 iso=SERIALIZABLE stat=PENDING epo=0 ts=1526535575.452611246,0 orig=1526535575.452611246,0 max=1526535575.452611246,0 wto=false rop=false seq=3
I180517 05:39:35.467452 3115 storage/replica_command.go:863 [split,n1,s1,r14/1:/{Table/17-Max}] initiating a split of this range at key /Table/18 [r15]
I180517 05:39:35.472320 3141 storage/replica_command.go:863 [split,n1,s1,r15/1:/{Table/18-Max}] initiating a split of this range at key /Table/19 [r16]
I180517 05:39:35.480079 3161 storage/replica_command.go:863 [split,n1,s1,r16/1:/{Table/19-Max}] initiating a split of this range at key /Table/20 [r17]
I180517 05:39:35.487681 3178 storage/replica_command.go:863 [split,n1,s1,r17/1:/{Table/20-Max}] initiating a split of this range at key /Table/21 [r18]
I180517 05:39:35.503202 2558 storage/replica_command.go:863 [split,n1,s1,r18/1:/{Table/21-Max}] initiating a split of this range at key /Table/22 [r19]
I180517 05:39:35.510851 3188 sql/event_log.go:124 [n1,intExec=initializeClusterSecret] Event: "set_cluster_setting", target: 0, info: {SettingName:cluster.secret Value:gen_random_uuid()::STRING User:root}
I180517 05:39:35.513891 3034 storage/replica_command.go:863 [split,n1,s1,r19/1:/{Table/22-Max}] initiating a split of this range at key /Table/23 [r20]
I180517 05:39:35.524244 2646 server/server.go:1562 [n1] done ensuring all necessary migrations have run
I180517 05:39:35.524320 2646 server/server.go:1565 [n1] serving sql connections
I180517 05:39:35.525445 2646 util/stop/stopper.go:471 quiescing; tasks left:
1 [async] storage.replica consistency checker: processing replica
1 [async] storage.Replica: checking consistency
1 [async] auto-upgrade
W180517 05:39:35.526137 3252 server/node.go:857 [n1] node=1: unable to log node_join event: log-event: node unavailable; try another peer
I180517 05:39:35.526405 2646 util/stop/stopper.go:471 quiescing; tasks left:
1 [async] storage.replica consistency checker: processing replica
1 [async] auto-upgrade
E180517 05:39:35.526618 3123 storage/consistency_queue.go:125 [replica consistency checker,n1,s1,r1/1:/{Min-System/}] computing own checksum: could not dial node ID 1 address 127.0.0.1:43113: initial connection heartbeat failed: rpc error: code = Canceled desc = context canceled
E180517 05:39:35.526684 3123 storage/queue.go:778 [replica consistency checker,n1,s1,r1/1:/{Min-System/}] computing own checksum: could not dial node ID 1 address 127.0.0.1:43113: initial connection heartbeat failed: rpc error: code = Canceled desc = context canceled
I180517 05:39:35.526750 2646 util/stop/stopper.go:471 quiescing; tasks left:
1 [async] auto-upgrade
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x8 pc=0x101d6c5]
goroutine 3040 [running]:
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.(*DString).Size(0x0, 0xc42237e3d8)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/sem/tree/datum.go:933 +0x5
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowContainer).rowSize(0xc421495c20, 0xc42237e4b8, 0x1, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/row_container.go:247 +0x66
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowContainer).AddRow(0xc421495c20, 0x2510ce0, 0xc420b410e0, 0xc42237e4b8, 0x1, 0x1, 0x7, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/row_container.go:272 +0x7c
github.com/cockroachdb/cockroach/pkg/sql.(*planner).ShowClusterSetting.func1(0x2510ce0, 0xc420b410e0, 0xc42079f720, 0x0, 0x0, 0xfda700, 0xc4207ca368)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/show_cluster_setting.go:161 +0x186
github.com/cockroachdb/cockroach/pkg/sql.doExpandPlan(0x2510ce0, 0xc420b410e0, 0xc42079f720, 0x7fffffffffffffff, 0x0, 0x0, 0x0, 0x0, 0x25070a0, 0xc421e6ac00, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/expand_plan.go:326 +0x2df2
github.com/cockroachdb/cockroach/pkg/sql.(*planner).expandPlan(0xc42079f720, 0x2510ce0, 0xc420b410e0, 0x25070a0, 0xc421e6ac00, 0x25070a0, 0xc421e6ac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/expand_plan.go:35 +0xce
github.com/cockroachdb/cockroach/pkg/sql.(*planner).optimizePlan(0xc42079f720, 0x2510ce0, 0xc420b410e0, 0x25070a0, 0xc421e6ac00, 0xc4208d10fa, 0x1, 0x1, 0x25070a0, 0xc421e6ac00, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/optimize.go:47 +0xeb
github.com/cockroachdb/cockroach/pkg/sql.(*planner).makePlan(0xc42079f720, 0x2510ce0, 0xc420b410e0, 0x2513320, 0xc4214acf80, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:325 +0x2ea
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc42079f300, 0x2510ce0, 0xc420b410e0, 0x2513320, 0xc4214acf80, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:579 +0xbe9
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc42079f300, 0x2510ce0, 0xc420b410e0, 0x2513320, 0xc4214acf80, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:398 +0xb33
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc42079f300, 0x2510ce0, 0xc420b410e0, 0x2513320, 0xc4214acf80, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:92 +0x358
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc42079f300, 0x2510ce0, 0xc420b40de0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1051 +0x2039
github.com/cockroachdb/cockroach/pkg/sql.(*internalExecutorImpl).initConnEx.func1(0xc42079f300, 0x2510ce0, 0xc420b40de0, 0xc420419380, 0xc4208d1100)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/internal.go:211 +0x86
created by github.com/cockroachdb/cockroach/pkg/sql.(*internalExecutorImpl).initConnEx
/go/src/github.com/cockroachdb/cockroach/pkg/sql/internal.go:200 +0x268
goroutine 1 [chan receive]:
testing.(*T).Run(0xc421c8a000, 0x21cb827, 0x16, 0x22ceb40, 0x870e01)
/usr/local/go/src/testing/testing.go:825 +0x301
testing.runTests.func1(0xc42085c000)
/usr/local/go/src/testing/testing.go:1063 +0x64
testing.tRunner(0xc42085c000, 0xc420297dd8)
/usr/local/go/src/testing/testing.go:777 +0xd0
testing.runTests(0xc420832040, 0x323a580, 0x3a, 0x3a, 0xbeb7634535268885)
/usr/local/go/src/testing/testing.go:1061 +0x2c4
testing.(*M).Run(0xc420848100, 0x0)
/usr/local/go/src/testing/testing.go:978 +0x171
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun_test.TestMain(0xc420848100)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/main_test.go:36 +0xb8
main.main()
_testmain.go:204 +0x151
goroutine 20 [syscall]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.0
/usr/local/go/src/os/signal/signal_unix.go:28 +0x41
goroutine 29 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1158 +0xf1
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:591 +0x110
goroutine 30 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:598 +0xab
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:592 +0x128
goroutine 34 [select, locked to thread]:
runtime.gopark(0x22d2938, 0x0, 0x21b2753, 0x6, 0x18, 0x1)
/usr/local/go/src/runtime/proc.go:291 +0x11a
runtime.selectgo(0xc420357f50, 0xc4200b4180)
/usr/local/go/src/runtime/select.go:392 +0xe50
runtime.ensureSigM.func1()
/usr/local/go/src/runtime/signal_unix.go:549 +0x1f4
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1
goroutine 2856 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x114)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc4207455c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc4207455c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a618b0, 0xc421572090, 0xc420a618a0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2854 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x100)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc4207451d0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc4207451d0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61870, 0xc421572090, 0xc420a61860)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2852 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x10e)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745500)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745500)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61830, 0xc421572090, 0xc420a61820)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2855 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x116)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745590)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745590)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61890, 0xc421572090, 0xc420a61880)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2853 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x10f)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745530)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745530)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61850, 0xc421572090, 0xc420a61840)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2908 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x2510ce0, 0xc4205e5d10)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1546 +0x3ae
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61f50, 0xc421572090, 0xc420df70e0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2860 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x115)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745680)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745680)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61940, 0xc421572090, 0xc420a61920)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2859 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x113)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745650)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745650)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61910, 0xc421572090, 0xc420a61900)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 100 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc420a3afc0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab
goroutine 101 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc420a3b0e0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab
goroutine 2883 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xea)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc4207459e0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc4207459e0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61c40, 0xc421572090, 0xc420a61c30)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2877 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xe6)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc4207458c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc4207458c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61b80, 0xc421572090, 0xc420a61b70)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2876 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x11f)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745890)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745890)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61b60, 0xc421572090, 0xc420a61b50)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2861 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x117)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc4207456b0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc4207456b0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61960, 0xc421572090, 0xc420a61950)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2726 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2510ce0, 0xc420744480)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4214bb760, 0xc421572090, 0xc4214bb750)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2731 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4222b1d40)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab
goroutine 2730 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4222b1c20)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab
goroutine 2729 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2510ce0, 0xc420e622d0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4214bb7c0, 0xc421572090, 0xc4214bb7b0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2728 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2510ce0, 0xc420e622a0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4214bb7a0, 0xc421572090, 0xc4214bb790)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2727 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2510ce0, 0xc420744450)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4214bb780, 0xc421572090, 0xc4214bb770)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2733 [select]:
github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps.func1(0x22d2cb0, 0xc4217f2050, 0xc420f0e000, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:185 +0x13e
created by github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps
/go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:178 +0xaf
goroutine 2739 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6(0x2510ce0, 0xc420e62360)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1113 +0xcf
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4214bb970, 0xc421572090, 0xc420f8c500)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2734 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x2510ce0, 0xc4207b2060)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x64
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4214bb8b0, 0xc421572090, 0xc420475400)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2647 [select]:
github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1(0xc421572090, 0xc422842150)
/go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:122 +0x108
created by github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler
/go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:119 +0x49
goroutine 3253 [select]:
database/sql.(*DB).connectionOpener(0xc421846c80, 0x2510c20, 0xc422393800)
/usr/local/go/src/database/sql/sql.go:935 +0x119
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:634 +0x178
goroutine 3254 [select]:
database/sql.(*DB).connectionResetter(0xc421846c80, 0x2510c20, 0xc422393800)
/usr/local/go/src/database/sql/sql.go:948 +0x12a
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:635 +0x1ae
goroutine 1849 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc421434d80)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab
goroutine 1850 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc421434ea0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab
goroutine 2865 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x11b)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745770)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745770)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a619e0, 0xc421572090, 0xc420a619d0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2862 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x118)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc4207456e0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc4207456e0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61980, 0xc421572090, 0xc420a61970)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2886 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xeb)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745aa0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745aa0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61ca0, 0xc421572090, 0xc420a61c90)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2894 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xf5)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745c20)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745c20)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61da0, 0xc421572090, 0xc420a61d90)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2878 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xe2)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc4207458f0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc4207458f0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61ba0, 0xc421572090, 0xc420a61b90)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2841 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x11c)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc4207452c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc4207452c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61660, 0xc421572090, 0xc420a61650)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2867 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x11a)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc4207457d0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc4207457d0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61a20, 0xc421572090, 0xc420a61a10)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2952 [select]:
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*flowScheduler).Start.func1(0x2510ce0, 0xc4212fa900)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow_scheduler.go:125 +0x14c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227ea330, 0xc421572090, 0xc4227ea320)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2955 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1(0x2510ce0, 0xc4212fa9c0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:473 +0x264
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227ea3e0, 0xc421572090, 0xc4203fcc30)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2903 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop(0xc420e89c00, 0x2510ce0, 0xc420745f50)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3671 +0x1e7
github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.raftTickLoop)-fm(0x2510ce0, 0xc420745f50)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3657 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61ec0, 0xc421572090, 0xc420a61eb0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2863 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x110)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745560)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745560)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a619a0, 0xc421572090, 0xc420a61990)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2898 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xf8)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745ce0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745ce0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61e20, 0xc421572090, 0xc420a61e10)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2864 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x120)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745740)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745740)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a619c0, 0xc421572090, 0xc420a619b0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2896 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xf7)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745c80)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745c80)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61de0, 0xc421572090, 0xc420a61dd0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2851 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x10b)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc4207454d0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc4207454d0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61810, 0xc421572090, 0xc420a61800)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2897 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xf9)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745cb0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745cb0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61e00, 0xc421572090, 0xc420a61df0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2871 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xfe)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc4205e5f50)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc4205e5f50)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61ac0, 0xc421572090, 0xc420a61ab0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2899 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xfd)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745d10)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745d10)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61e40, 0xc421572090, 0xc420a61e30)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2900 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xfa)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745d70)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745d70)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61e60, 0xc421572090, 0xc420a61e50)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2902 [semacquire]:
sync.runtime_Semacquire(0xc420a68f24)
/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc420a68f18)
/usr/local/go/src/sync/waitgroup.go:129 +0x72
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc420a68ea0, 0x2510ce0, 0xc420745f20)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:172 +0x31
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait-fm(0x2510ce0, 0xc420745f20)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3655 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61ea0, 0xc421572090, 0xc420a61e90)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2901 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xf1)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745bc0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745bc0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61e80, 0xc421572090, 0xc420a61e70)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2879 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xe3)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745920)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745920)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61bc0, 0xc421572090, 0xc420a61bb0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2891 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xf0)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745b90)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745b90)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61d40, 0xc421572090, 0xc420a61d30)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2957 [select]:
github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).Start.func2(0x2510ce0, 0xc420e62120)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:269 +0x127
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227ea400, 0xc421572090, 0xc4212fa000)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2884 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xe8)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745a10)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745a10)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61c60, 0xc421572090, 0xc420a61c50)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2875 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x11e)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745860)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745860)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61b40, 0xc421572090, 0xc420a61b30)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 421 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4213ef200)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab
goroutine 422 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4213ef320)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab
goroutine 2857 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x112)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc4207455f0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc4207455f0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a618d0, 0xc421572090, 0xc420a618c0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2839 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x102)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745260)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745260)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61620, 0xc421572090, 0xc420a61610)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2872 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x104)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc4205e5f80)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc4205e5f80)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61ae0, 0xc421572090, 0xc420a61ad0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2885 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xe9)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745a70)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745a70)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61c80, 0xc421572090, 0xc420a61c70)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2880 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xe4)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745950)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745950)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61be0, 0xc421572090, 0xc420a61bd0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2881 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xec)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745980)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745980)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61c00, 0xc421572090, 0xc420a61bf0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2953 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*Server).Start.func1(0x2510ce0, 0xc4212fa930)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:296 +0x11f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227ea340, 0xc421572090, 0xc4204a8120)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2840 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0x103)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745290)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745290)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61640, 0xc421572090, 0xc420a61630)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2887 [semacquire]:
sync.runtime_notifyListWait(0xc420c874d0, 0xf2)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420c874c0)
/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420a68ea0, 0x2510ce0, 0xc420745ad0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2510ce0, 0xc420745ad0)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420a61cc0, 0xc421572090, 0xc420a61cb0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2724 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2510ce0, 0xc4207443f0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4214bb720, 0xc421572090, 0xc4214bb710)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2723 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2510ce0, 0xc4207443c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4214bb700, 0xc421572090, 0xc4214bb6f0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2722 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2510ce0, 0xc420744390)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4214bb6e0, 0xc421572090, 0xc4214bb6d0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2657 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2510ce0, 0xc420e62210)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4214bb6c0, 0xc421572090, 0xc4214bb6b0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2656 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2510ce0, 0xc420e62270)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4214bb6a0, 0xc421572090, 0xc4214bb690)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad
goroutine 2655 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x2510ce0, 0xc420744420)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4214bb680, 0xc421572090, 0xc4214bb670)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker