kv: TestBadRequest failed under stress

SHA: https://github.com/cockroachdb/cockroach/commits/9376ca50f874e475189b1fe1f7ef8f4e0f6c18b4

Parameters:
```
TAGS=deadlock
GOFLAGS=
```

Stress build found a failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=658672&tab=buildLog

```

I180515 08:59:16.708598 3310578 server/config.go:539  [n?] 1 storage engine initialized
I180515 08:59:16.708748 3310578 server/config.go:542  [n?] RocksDB cache size: 128 MiB
I180515 08:59:16.708840 3310578 server/config.go:542  [n?] store 0: in-memory, size 0 B
I180515 08:59:16.719735 3310578 server/node.go:376  [n?] **** cluster 9fbd12e4-634c-4e2d-9e38-290745f44c2d has been created
I180515 08:59:16.719842 3310578 server/server.go:1351  [n?] **** add additional nodes by specifying --join=127.0.0.1:39039
I180515 08:59:16.735216 3310578 storage/store.go:1431  [n1,s1] [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I180515 08:59:16.735427 3310578 server/node.go:506  [n1] initialized store [n1,s1]: disk (capacity=512 MiB, available=512 MiB, used=0 B, logicalBytes=6.9 KiB), ranges=1, leases=0, writes=0.00, bytesPerReplica={p10=7043.00 p25=7043.00 p50=7043.00 p75=7043.00 p90=7043.00 pMax=7043.00}, writesPerReplica={p10=0.00 p25=0.00 p50=0.00 p75=0.00 p90=0.00 pMax=0.00}
I180515 08:59:16.735575 3310578 server/node.go:354  [n1] node ID 1 initialized
I180515 08:59:16.735729 3310578 gossip/gossip.go:333  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:39039" > attrs:<> locality:<> ServerVersion:<major_val:2 minor_val:0 patch:0 unstable:3 > 
I180515 08:59:16.736112 3310578 storage/stores.go:222  [n1] read 0 node addresses from persistent storage
I180515 08:59:16.736315 3310578 server/node.go:647  [n1] connecting to gossip network to verify cluster ID...
I180515 08:59:16.743760 3310578 server/node.go:672  [n1] node connected via gossip and verified as part of cluster "9fbd12e4-634c-4e2d-9e38-290745f44c2d"
I180515 08:59:16.743888 3310578 server/node.go:440  [n1] node=1: started with [<no-attributes>=<in-mem>] engine(s) and attributes []
I180515 08:59:16.744713 3310578 server/server.go:1476  [n1] starting https server at 127.0.0.1:46561
I180515 08:59:16.744818 3310578 server/server.go:1477  [n1] starting grpc/postgres server at 127.0.0.1:39039
I180515 08:59:16.744941 3310578 server/server.go:1478  [n1] advertising CockroachDB node at 127.0.0.1:39039
I180515 08:59:16.835710 3312995 sql/event_log.go:124  [n1,intExec=optInToDiagnosticsStatReporting] Event: "set_cluster_setting", target: 0, info: {SettingName:diagnostics.reporting.enabled Value:true User:root}
I180515 08:59:16.928762 3314941 sql/event_log.go:124  [n1,intExec=set-setting] Event: "set_cluster_setting", target: 0, info: {SettingName:version Value:$1 User:root}
I180515 08:59:16.974246 3317300 storage/replica_consistency.go:129  [replica consistency checker,n1,s1,r1/1:/M{in-ax}] triggering stats recomputation to resolve delta of {ContainsEstimates:true LastUpdateNanos:1526374756929452155 IntentAge:0 GCBytesAge:0 LiveBytes:-20388 LiveCount:-431 KeyBytes:-19987 KeyCount:-431 ValBytes:-401 ValCount:-431 IntentBytes:0 IntentCount:0 SysBytes:0 SysCount:0}
I180515 08:59:16.983126 3316800 sql/event_log.go:124  [n1,intExec=disableNetTrace] Event: "set_cluster_setting", target: 0, info: {SettingName:trace.debug.enable Value:false User:root}
I180515 08:59:17.119570 3320472 sql/event_log.go:124  [n1,intExec=initializeClusterSecret] Event: "set_cluster_setting", target: 0, info: {SettingName:cluster.secret Value:gen_random_uuid()::STRING User:root}
I180515 08:59:17.145967 3310578 server/server.go:1555  [n1] done ensuring all necessary migrations have run
I180515 08:59:17.146054 3310578 server/server.go:1558  [n1] serving sql connections
I180515 08:59:17.155014 3310578 util/stop/stopper.go:471  quiescing; tasks left:
1      [async] auto-upgrade
W180515 08:59:17.156149 3322473 server/node.go:857  [n1] node=1: unable to log node_join event: log-event: node unavailable; try another peer
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x8 pc=0x1040015]

goroutine 3322522 [running]:
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.(*DString).Size(0x0, 0xc4341c43e8)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/sem/tree/datum.go:933 +0x5
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowContainer).rowSize(0xc42a0b8fc0, 0xc4341c44c8, 0x1, 0x1, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/row_container.go:247 +0x66
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowContainer).AddRow(0xc42a0b8fc0, 0x24dd840, 0xc4266aa750, 0xc4341c44c8, 0x1, 0x1, 0x7, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/row_container.go:272 +0x7c
github.com/cockroachdb/cockroach/pkg/sql.(*planner).ShowClusterSetting.func1(0x24dd840, 0xc4266aa750, 0xc433bb8418, 0x0, 0x0, 0xc42cfc9100, 0x3209730)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/show_cluster_setting.go:161 +0x186
github.com/cockroachdb/cockroach/pkg/sql.doExpandPlan(0x24dd840, 0xc4266aa750, 0xc433bb8418, 0x7fffffffffffffff, 0x0, 0x0, 0x0, 0x0, 0x24d2e80, 0xc4264a7580, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/expand_plan.go:320 +0x2c51
github.com/cockroachdb/cockroach/pkg/sql.(*planner).expandPlan(0xc433bb8418, 0x24dd840, 0xc4266aa750, 0x24d2e80, 0xc4264a7580, 0x24d2e80, 0xc4264a7580, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/expand_plan.go:35 +0xce
github.com/cockroachdb/cockroach/pkg/sql.(*planner).optimizePlan(0xc433bb8418, 0x24dd840, 0xc4266aa750, 0x24d2e80, 0xc4264a7580, 0xc429b26ff9, 0x1, 0x1, 0x24d2e80, 0xc4264a7580, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/optimize.go:47 +0xeb
github.com/cockroachdb/cockroach/pkg/sql.(*planner).makePlan(0xc433bb8418, 0x24dd840, 0xc4266aa750, 0x24dfe00, 0xc4246a46a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:324 +0x2ea
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc433bb8000, 0x24dd840, 0xc4266aa750, 0x24dfe00, 0xc4246a46a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:578 +0xb28
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc433bb8000, 0x24dd840, 0xc4266aa750, 0x24dfe00, 0xc4246a46a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:397 +0xb33
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc433bb8000, 0x24dd840, 0xc4266aa750, 0x24dfe00, 0xc4246a46a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:91 +0x358
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc433bb8000, 0x24dd840, 0xc424c3f8c0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1051 +0x2039
github.com/cockroachdb/cockroach/pkg/sql.(*internalExecutorImpl).initConnEx.func1(0xc433bb8000, 0x24dd840, 0xc424c3f8c0, 0xc42506ac40, 0xc429b26e70)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/internal.go:211 +0x86
created by github.com/cockroachdb/cockroach/pkg/sql.(*internalExecutorImpl).initConnEx
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/internal.go:200 +0x268

goroutine 1 [chan receive]:
testing.(*T).Run(0xc425ab6e10, 0x21b45c0, 0xe, 0x22c3ec0, 0x883a01)
	/usr/local/go/src/testing/testing.go:825 +0x301
testing.runTests.func1(0xc420514870)
	/usr/local/go/src/testing/testing.go:1063 +0x64
testing.tRunner(0xc420514870, 0xc42061ddd8)
	/usr/local/go/src/testing/testing.go:777 +0xd0
testing.runTests(0xc4202fe5e0, 0x321a160, 0x80, 0x80, 0x0)
	/usr/local/go/src/testing/testing.go:1061 +0x2c4
testing.(*M).Run(0xc420258480, 0x0)
	/usr/local/go/src/testing/testing.go:978 +0x171
github.com/cockroachdb/cockroach/pkg/kv_test.TestMain(0xc420258480)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/main_test.go:51 +0x51
main.main()
	_testmain.go:300 +0x151

goroutine 11 [syscall, 1 minutes]:
os/signal.signal_recv(0xc42006e7c0)
	/usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.0
	/usr/local/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 28 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1158 +0xf1
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:591 +0x110

goroutine 29 [chan receive, 1 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:598 +0xab
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:592 +0x128

goroutine 30 [select, 1 minutes, locked to thread]:
runtime.gopark(0x22c8d00, 0x0, 0x21a7e5a, 0x6, 0x18, 0x1)
	/usr/local/go/src/runtime/proc.go:291 +0x11a
runtime.selectgo(0xc42052d750, 0xc4204a4780)
	/usr/local/go/src/runtime/select.go:392 +0xe50
runtime.ensureSigM.func1()
	/usr/local/go/src/runtime/signal_unix.go:549 +0x1f4
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1

goroutine 3311664 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x41)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385500)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385500)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cca40, 0xc424260510, 0xc4227cca30)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3310578 [semacquire]:
sync.runtime_notifyListWait(0xc421e9cad0, 0xc400000000)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421e9cac0)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Quiesce(0xc424260510, 0x24dd7c0, 0xc42003e128)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:473 +0x18a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Stop(0xc424260510, 0x24dd7c0, 0xc42003e128)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:416 +0x1ff
github.com/cockroachdb/cockroach/pkg/kv_test.TestBadRequest(0xc425ab6e10)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender_server_test.go:1636 +0x42a
testing.tRunner(0xc425ab6e10, 0x22c3ec0)
	/usr/local/go/src/testing/testing.go:777 +0xd0
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:824 +0x2e0

goroutine 3311666 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0xc400000043)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385560)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385560)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cca90, 0xc424260510, 0xc4227cca80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3312067 [select]:
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*flowScheduler).Start.func1(0x24dd840, 0xc429da3530)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow_scheduler.go:125 +0x14c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc422574000, 0xc424260510, 0xc423a57fe0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311680 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x51)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385830)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385830)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227ccc80, 0xc424260510, 0xc4227ccc70)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 2985703 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc42a5c46c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3125483 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc42a26eea0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3310639 [select]:
github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1(0xc424260510, 0xc42352db20)
	/go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:122 +0x108
created by github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler
	/go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:119 +0x49

goroutine 3310738 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24dd840, 0xc4267870e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a57250, 0xc424260510, 0xc423a57240)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311690 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x5b)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385a70)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385a70)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cce80, 0xc424260510, 0xc4227cce70)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311687 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x58)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a3859b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a3859b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cce20, 0xc424260510, 0xc4227cce10)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3310739 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24dd840, 0xc426787140)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a57270, 0xc424260510, 0xc423a57260)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3109828 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc42998fd40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3311929 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x24dd840, 0xc42a3fce70)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:580 +0x194
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc422827c30, 0xc424260510, 0xc4252233c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311708 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0xc400000033)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a3fcab0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a3fcab0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cd1a0, 0xc424260510, 0xc4227cd190)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311715 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0xc40000003a)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a3fcc00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a3fcc00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cd360, 0xc424260510, 0xc4227cd350)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3310741 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24dd840, 0xc426787200)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a572d0, 0xc424260510, 0xc423a572c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311703 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0xc400000028)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385d10)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385d10)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cd090, 0xc424260510, 0xc4227cd080)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3310817 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6(0x24dd840, 0xc429908570)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1112 +0xcf
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a57a70, 0xc424260510, 0xc4218f1c00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311712 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0xc40000002f)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a3fc990)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a3fc990)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cd2e0, 0xc424260510, 0xc4227cd2d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3312069 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*Server).Start.func1(0x24dd840, 0xc429da3560)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:296 +0x11f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc422574180, 0xc424260510, 0xc4270f4300)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311970 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossip.func1(0x24dd840, 0xc42a3fd0e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:700 +0x2b8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42a8e1390, 0xc424260510, 0xc42b399a80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3310744 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24dd840, 0xc426787050)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a57330, 0xc424260510, 0xc423a57320)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311673 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x4a)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a3856b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a3856b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227ccba0, 0xc424260510, 0xc4227ccb90)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3042360 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc427d58ea0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3311696 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x61)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385bc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385bc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227ccf60, 0xc424260510, 0xc4227ccf50)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311732 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func2(0x24dd840, 0xc426c1b020)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1401 +0x10e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cd600, 0xc424260510, 0xc42553e880)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311987 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*SchemaChangeManager).Start.func1(0x24dd840, 0xc426c1b2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/schema_changer.go:1129 +0x498
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42a8e15d0, 0xc424260510, 0xc42b399e20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311735 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x24dd840, 0xc42a3fccf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1535 +0x3ae
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cd640, 0xc424260510, 0xc4244cbc20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 2899219 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc428364d80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3310752 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24dd840, 0xc426787410)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a57460, 0xc424260510, 0xc423a57450)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311701 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x66)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385cb0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385cb0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cd050, 0xc424260510, 0xc4227cd040)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 2942097 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc428c479e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3214989 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc43081c120)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3311725 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0xc40000002d)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc426c1a750)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc426c1a750)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cd4d0, 0xc424260510, 0xc4227cd4c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 2912063 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4296f19e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3322474 [select]:
database/sql.(*DB).connectionOpener(0xc425a0cc80, 0x24dd780, 0xc4264a6f80)
	/usr/local/go/src/database/sql/sql.go:935 +0x119
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:634 +0x178

goroutine 2964065 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4299fdd40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3310801 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*LeaseManager).RefreshLeases.func1(0x24dd840, 0xc426787440)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/lease.go:1479 +0x1d1
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a577f0, 0xc424260510, 0xc429da23c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3310812 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x24dd840, 0xc426787470)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x64
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a57990, 0xc424260510, 0xc425e2d260)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3322473 [select]:
github.com/cockroachdb/cockroach/pkg/util/retry.(*Retry).Next(0xc433f79f30, 0xc4266aa060)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/retry/retry.go:119 +0x16b
github.com/cockroachdb/cockroach/pkg/server.(*Node).recordJoinEvent.func1(0x24dd840, 0xc424c3ff80)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:841 +0x2b0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4246a4280, 0xc424260510, 0xc424c3f050)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311718 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0xc40000003b)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a3fcc30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a3fcc30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cd3e0, 0xc424260510, 0xc4227cd3d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3250400 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc428070120)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3125479 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc42a26ec60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3311984 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).startSampleEnvironment.func1(0x24dd840, 0xc426c1b2f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1763 +0x158
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42a8e1590, 0xc424260510, 0xc42b399de0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311717 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0xc40000003c)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a3fcc60)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a3fcc60)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cd3c0, 0xc424260510, 0xc4227cd3b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311681 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x52)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385860)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385860)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cccb0, 0xc424260510, 0xc4227ccc90)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3310805 [select]:
github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps.func1(0x22c9088, 0xc421c1b2c0, 0xc424495b20, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:185 +0x13e
created by github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps
	/go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:178 +0xaf

goroutine 3311705 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0xc400000030)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a3fc9c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a3fc9c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cd140, 0xc424260510, 0xc4227cd120)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3291158 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc431af37a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3312112 [select]:
github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).Start.func2(0x24dd840, 0xc429d5c240)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:269 +0x127
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42a8e1a30, 0xc424260510, 0xc426c1b470)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311986 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteNodeStatus.func1(0x24dd840, 0xc426c1b290)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:775 +0x1e1
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42a8e15c0, 0xc424260510, 0xc42b399e00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3185955 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc429c3fc20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3311704 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0xc400000040)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385d40)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385d40)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cd0d0, 0xc424260510, 0xc4227cd0c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311691 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x5c)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385aa0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385aa0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227ccea0, 0xc424260510, 0xc4227cce90)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311719 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0xc40000003d)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc429da3500)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc429da3500)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cd400, 0xc424260510, 0xc4227cd3f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3042364 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc427694ea0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3311697 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x62)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385bf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385bf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227ccf80, 0xc424260510, 0xc4227ccf70)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311684 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x55)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a3858f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a3858f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227ccd40, 0xc424260510, 0xc4227ccd30)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3291162 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc431af38c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3311672 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x49)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385680)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385680)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227ccb80, 0xc424260510, 0xc4227ccb70)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 2899215 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc428364a20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3311692 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x5d)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385ad0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385ad0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227ccee0, 0xc424260510, 0xc4227cceb0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3310745 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24dd840, 0xc4267872c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a57360, 0xc424260510, 0xc423a57350)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311667 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x44)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385590)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385590)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227ccab0, 0xc424260510, 0xc4227ccaa0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3185959 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4299fc5a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3311676 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x4d)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385740)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385740)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227ccc00, 0xc424260510, 0xc4227ccbf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311933 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x24dd840, 0xc42a3fcf30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:580 +0x194
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc422827c70, 0xc424260510, 0xc425223440)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311724 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0xc40000002c)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc426c1a720)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc426c1a720)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cd4b0, 0xc424260510, 0xc4227cd490)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311675 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x4c)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385710)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385710)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227ccbe0, 0xc424260510, 0xc4227ccbd0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311985 [select]:
github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1(0x24dd840, 0xc42a3fd140)
	/go/src/github.com/cockroachdb/cockroach/pkg/ts/db.go:126 +0x142
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42a8e15b0, 0xc424260510, 0xc42a8e15a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311709 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0xc400000034)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a3fcae0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a3fcae0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cd1d0, 0xc424260510, 0xc4227cd1b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3322475 [select]:
database/sql.(*DB).connectionResetter(0xc425a0cc80, 0x24dd780, 0xc4264a6f80)
	/usr/local/go/src/database/sql/sql.go:948 +0x12a
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:635 +0x1ae

goroutine 3311685 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x56)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385950)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385950)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227ccd80, 0xc424260510, 0xc4227ccd50)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3310737 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24dd840, 0xc4267870b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a57230, 0xc424260510, 0xc423a57220)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3140678 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc428db97a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3311688 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x59)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a3859e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a3859e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cce40, 0xc424260510, 0xc4227cce30)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311930 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x24dd840, 0xc42a3fcea0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:580 +0x194
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc422827c40, 0xc424260510, 0xc4252233e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3310743 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24dd840, 0xc426787260)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a57310, 0xc424260510, 0xc423a57300)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3310462 [semacquire]:
sync.runtime_notifyListWait(0xc421b4fa98, 0xc40000002c)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421b4fa88)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc421b4f980)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:647 +0x9c
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:634 +0x418

goroutine 3312111 [select]:
github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).Start.func1(0x24dd840, 0xc429d5c210)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:258 +0x151
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42a8e1a20, 0xc424260510, 0xc426c1b440)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/s1 runs completed, 1 failures, over 1m40s
topper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3250404 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc428071e60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3311682 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x53)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385890)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385890)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cccd0, 0xc424260510, 0xc4227cccc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311693 [semacquire]:
sync.runtime_notifyListWait(0xc42173ea10, 0x5e)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42173ea00)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420259580, 0x24dd840, 0xc42a385b30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x24dd840, 0xc42a385b30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227ccf00, 0xc424260510, 0xc4227ccef0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3232661 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc42af7d680)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x2ab

goroutine 3311733 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x24dd840, 0xc426c1b050)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1535 +0x3ae
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cd610, 0xc424260510, 0xc4244cbb80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3310740 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x24dd840, 0xc4267871a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x162
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a57290, 0xc424260510, 0xc423a57280)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xad

goroutine 3311729 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).coalescedHeartbeatsLoop(0xc428ade400, 0x24dd840, 0xc426c1aff0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3638 +0x159
github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.coalescedHeartbeatsLoop)-fm(0x24dd840, 0xc426c1aff0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3584 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4227cd5a0, 0xc424260510, 0xc4227cd590)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0xe9
```

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

kv: TestBadRequest failed under stress #25485

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

kv: TestBadRequest failed under stress #25485

Description

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions