Skip to content

storage: TestLogGrowthWhenRefreshingPendingCommands failed under stress #27878

Closed
#27888
Closed
storage: TestLogGrowthWhenRefreshingPendingCommands failed under stress#27878
#27888
Assignees
nvb
Labels
A-kv-replicationRelating to Raft, consensus, and coordination.C-test-failureBroken test (automatically or manually discovered).O-robotOriginated from a bot.
Milestone
2.1
@cockroach-teamcity

Description

@cockroach-teamcity
cockroach-teamcity
opened on Jul 24, 2018

SHA: https://github.com/cockroachdb/cockroach/commits/eb028e6e962b864b79c9d0c610d2c44b26a7d0c4

Parameters:

TAGS=
GOFLAGS=

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=793599&tab=buildLog


I180724 06:34:06.701812 176430 gossip/client.go:129  [n5] started gossip client to 127.0.0.1:39261
I180724 06:34:06.702135 176430 gossip/client.go:134  [n5] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:06.714864 176627 gossip/client.go:129  [n3] started gossip client to 127.0.0.1:39261
I180724 06:34:06.715191 176627 gossip/client.go:134  [n3] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:06.775297 176622 gossip/client.go:129  [n4] started gossip client to 127.0.0.1:39261
I180724 06:34:06.775772 176622 gossip/client.go:134  [n4] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:07.702045 176558 gossip/client.go:129  [n5] started gossip client to 127.0.0.1:39261
I180724 06:34:07.702882 176558 gossip/client.go:134  [n5] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:07.715348 176524 gossip/client.go:129  [n3] started gossip client to 127.0.0.1:39261
I180724 06:34:07.715868 176524 gossip/client.go:134  [n3] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:07.775476 176672 gossip/client.go:129  [n4] started gossip client to 127.0.0.1:42787
I180724 06:34:07.776215 176672 gossip/client.go:134  [n4] closing client to node 2 (127.0.0.1:42787): stopping outgoing client to node 2 (127.0.0.1:42787); already have incoming
I180724 06:34:08.702169 176585 gossip/client.go:129  [n5] started gossip client to 127.0.0.1:45547
I180724 06:34:08.702500 176585 gossip/client.go:134  [n5] closing client to node 3 (127.0.0.1:45547): stopping outgoing client to node 3 (127.0.0.1:45547); already have incoming
I180724 06:34:08.715532 176739 gossip/client.go:129  [n3] started gossip client to 127.0.0.1:42787
I180724 06:34:08.715882 176739 gossip/client.go:134  [n3] closing client to node 2 (127.0.0.1:42787): stopping outgoing client to node 2 (127.0.0.1:42787); already have incoming
I180724 06:34:08.775692 176710 gossip/client.go:129  [n4] started gossip client to 127.0.0.1:45547
I180724 06:34:08.776112 176710 gossip/client.go:134  [n4] closing client to node 3 (127.0.0.1:45547): stopping outgoing client to node 3 (127.0.0.1:45547); already have incoming
I180724 06:34:09.702349 176758 gossip/client.go:129  [n5] started gossip client to 127.0.0.1:44815
I180724 06:34:09.702580 176687 gossip/server.go:269  [n4] refusing gossip from node 5 (max 3 conns); forwarding to 3 ({tcp 127.0.0.1:45547})
I180724 06:34:09.702955 176758 gossip/client.go:134  [n5] closing client to node 4 (127.0.0.1:44815): received forward from node 4 to 3 (127.0.0.1:45547); already have active connection, skipping
I180724 06:34:09.715671 176774 gossip/client.go:129  [n3] started gossip client to 127.0.0.1:39261
I180724 06:34:09.716330 176774 gossip/client.go:134  [n3] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:09.775820 176803 gossip/client.go:129  [n4] started gossip client to 127.0.0.1:39261
I180724 06:34:09.776474 176803 gossip/client.go:134  [n4] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:10.702450 176689 gossip/client.go:129  [n5] started gossip client to 127.0.0.1:39261
I180724 06:34:10.702784 176689 gossip/client.go:134  [n5] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:10.715839 176792 gossip/client.go:129  [n3] started gossip client to 127.0.0.1:39261
I180724 06:34:10.716267 176792 gossip/client.go:134  [n3] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:10.782598 176856 gossip/client.go:129  [n4] started gossip client to 127.0.0.1:39261
I180724 06:34:10.783470 176856 gossip/client.go:134  [n4] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:11.702622 176822 gossip/client.go:129  [n5] started gossip client to 127.0.0.1:39261
I180724 06:34:11.703388 176822 gossip/client.go:134  [n5] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:11.716041 176886 gossip/client.go:129  [n3] started gossip client to 127.0.0.1:42787
I180724 06:34:11.716487 176886 gossip/client.go:134  [n3] closing client to node 2 (127.0.0.1:42787): stopping outgoing client to node 2 (127.0.0.1:42787); already have incoming
I180724 06:34:11.782754 176850 gossip/client.go:129  [n4] started gossip client to 127.0.0.1:42787
I180724 06:34:11.783094 176850 gossip/client.go:134  [n4] closing client to node 2 (127.0.0.1:42787): stopping outgoing client to node 2 (127.0.0.1:42787); already have incoming
I180724 06:34:12.702846 176918 gossip/client.go:129  [n5] started gossip client to 127.0.0.1:45547
I180724 06:34:12.703292 176918 gossip/client.go:134  [n5] closing client to node 3 (127.0.0.1:45547): stopping outgoing client to node 3 (127.0.0.1:45547); already have incoming
I180724 06:34:12.716234 176946 gossip/client.go:129  [n3] started gossip client to 127.0.0.1:39261
I180724 06:34:12.716750 176946 gossip/client.go:134  [n3] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:12.782992 176950 gossip/client.go:129  [n4] started gossip client to 127.0.0.1:45547
I180724 06:34:12.783493 176950 gossip/client.go:134  [n4] closing client to node 3 (127.0.0.1:45547): stopping outgoing client to node 3 (127.0.0.1:45547); already have incoming
I180724 06:34:13.703065 176879 gossip/client.go:129  [n5] started gossip client to 127.0.0.1:44815
I180724 06:34:13.703530 176905 gossip/server.go:269  [n4] refusing gossip from node 5 (max 3 conns); forwarding to 2 ({tcp 127.0.0.1:42787})
I180724 06:34:13.703878 176879 gossip/client.go:134  [n5] closing client to node 4 (127.0.0.1:44815): received forward from node 4 to 2 (127.0.0.1:42787); already have active connection, skipping
I180724 06:34:13.716487 177007 gossip/client.go:129  [n3] started gossip client to 127.0.0.1:39261
I180724 06:34:13.717031 177007 gossip/client.go:134  [n3] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:13.780184 176984 gossip/client.go:129  [n2] started gossip client to 127.0.0.1:44815
I180724 06:34:13.783236 176908 gossip/client.go:129  [n4] started gossip client to 127.0.0.1:39261
I180724 06:34:13.784099 176908 gossip/client.go:134  [n4] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:14.703196 177025 gossip/client.go:129  [n5] started gossip client to 127.0.0.1:39261
I180724 06:34:14.703574 177025 gossip/client.go:134  [n5] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:14.716630 177036 gossip/client.go:129  [n3] started gossip client to 127.0.0.1:42787
I180724 06:34:14.717258 177036 gossip/client.go:134  [n3] closing client to node 2 (127.0.0.1:42787): stopping outgoing client to node 2 (127.0.0.1:42787); already have incoming
I180724 06:34:14.780364 177091 gossip/client.go:129  [n2] started gossip client to 127.0.0.1:38803
I180724 06:34:14.780568 177074 gossip/server.go:269  [n5] refusing gossip from node 2 (max 3 conns); forwarding to 4 ({tcp 127.0.0.1:44815})
I180724 06:34:14.780810 177091 gossip/client.go:134  [n2] closing client to node 5 (127.0.0.1:38803): received forward from node 5 to 4 (127.0.0.1:44815); already have active connection, skipping
I180724 06:34:14.783344 177086 gossip/client.go:129  [n4] started gossip client to 127.0.0.1:39261
I180724 06:34:14.783654 177086 gossip/client.go:134  [n4] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:15.703417 177118 gossip/client.go:129  [n5] started gossip client to 127.0.0.1:39261
I180724 06:34:15.703931 177118 gossip/client.go:134  [n5] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:15.716806 177134 gossip/client.go:129  [n3] started gossip client to 127.0.0.1:39261
I180724 06:34:15.717110 177134 gossip/client.go:134  [n3] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:15.780534 177055 gossip/client.go:129  [n2] started gossip client to 127.0.0.1:38803
I180724 06:34:15.780727 177143 gossip/server.go:269  [n5] refusing gossip from node 2 (max 3 conns); forwarding to 4 ({tcp 127.0.0.1:44815})
I180724 06:34:15.780915 177055 gossip/client.go:134  [n2] closing client to node 5 (127.0.0.1:38803): received forward from node 5 to 4 (127.0.0.1:44815); already have active connection, skipping
I180724 06:34:15.783524 177183 gossip/client.go:129  [n4] started gossip client to 127.0.0.1:42787
I180724 06:34:15.784227 177183 gossip/client.go:134  [n4] closing client to node 2 (127.0.0.1:42787): stopping outgoing client to node 2 (127.0.0.1:42787); already have incoming
I180724 06:34:16.703621 177144 gossip/client.go:129  [n5] started gossip client to 127.0.0.1:45547
I180724 06:34:16.703948 177144 gossip/client.go:134  [n5] closing client to node 3 (127.0.0.1:45547): stopping outgoing client to node 3 (127.0.0.1:45547); already have incoming
I180724 06:34:16.716942 177209 gossip/client.go:129  [n3] started gossip client to 127.0.0.1:39261
I180724 06:34:16.717229 177209 gossip/client.go:134  [n3] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:16.780713 177225 gossip/client.go:129  [n2] started gossip client to 127.0.0.1:38803
I180724 06:34:16.780889 176738 gossip/server.go:269  [n5] refusing gossip from node 2 (max 3 conns); forwarding to 1 ({tcp 127.0.0.1:39261})
I180724 06:34:16.781150 177225 gossip/client.go:134  [n2] closing client to node 5 (127.0.0.1:38803): received forward from node 5 to 1 (127.0.0.1:39261); already have active connection, skipping
I180724 06:34:16.783674 177201 gossip/client.go:129  [n4] started gossip client to 127.0.0.1:45547
I180724 06:34:16.784099 177201 gossip/client.go:134  [n4] closing client to node 3 (127.0.0.1:45547): stopping outgoing client to node 3 (127.0.0.1:45547); already have incoming
I180724 06:34:17.703734 177269 gossip/client.go:129  [n5] started gossip client to 127.0.0.1:44815
I180724 06:34:17.703989 177315 gossip/server.go:269  [n4] refusing gossip from node 5 (max 3 conns); forwarding to 2 ({tcp 127.0.0.1:42787})
I180724 06:34:17.704302 177269 gossip/client.go:134  [n5] closing client to node 4 (127.0.0.1:44815): received forward from node 4 to 2 (127.0.0.1:42787); already have active connection, skipping
I180724 06:34:17.717121 177331 gossip/client.go:129  [n3] started gossip client to 127.0.0.1:42787
I180724 06:34:17.717557 177331 gossip/client.go:134  [n3] closing client to node 2 (127.0.0.1:42787): stopping outgoing client to node 2 (127.0.0.1:42787); already have incoming
I180724 06:34:17.780867 177347 gossip/client.go:129  [n2] started gossip client to 127.0.0.1:38803
I180724 06:34:17.781047 177302 gossip/server.go:269  [n5] refusing gossip from node 2 (max 3 conns); forwarding to 1 ({tcp 127.0.0.1:39261})
I180724 06:34:17.781333 177347 gossip/client.go:134  [n2] closing client to node 5 (127.0.0.1:38803): received forward from node 5 to 1 (127.0.0.1:39261); already have active connection, skipping
I180724 06:34:17.783825 177318 gossip/client.go:129  [n4] started gossip client to 127.0.0.1:39261
I180724 06:34:17.784201 177318 gossip/client.go:134  [n4] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:18.703984 177358 gossip/client.go:129  [n5] started gossip client to 127.0.0.1:39261
I180724 06:34:18.704336 177358 gossip/client.go:134  [n5] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:18.717335 177244 gossip/client.go:129  [n3] started gossip client to 127.0.0.1:39261
I180724 06:34:18.717819 177244 gossip/client.go:134  [n3] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
I180724 06:34:18.781079 175771 gossip/client.go:129  [n2] started gossip client to 127.0.0.1:38803
I180724 06:34:18.781528 177436 gossip/server.go:269  [n5] refusing gossip from node 2 (max 3 conns); forwarding to 1 ({tcp 127.0.0.1:39261})
I180724 06:34:18.781840 175771 gossip/client.go:134  [n2] closing client to node 5 (127.0.0.1:38803): received forward from node 5 to 1 (127.0.0.1:39261); already have active connection, skipping
I180724 06:34:18.784797 177437 gossip/client.go:129  [n4] started gossip client to 127.0.0.1:39261
I180724 06:34:18.785321 177437 gossip/client.go:134  [n4] closing client to node 1 (127.0.0.1:39261): stopping outgoing client to node 1 (127.0.0.1:39261); already have incoming
panic: test timed out after 30m0s

goroutine 177390 [running]:
testing.(*M).startAlarm.func1()
	/usr/local/go/src/testing/testing.go:1240 +0xfc
created by time.goFunc
	/usr/local/go/src/time/sleep.go:172 +0x44

goroutine 1 [chan receive, 29 minutes]:
testing.(*T).Run(0xc420662690, 0x2588262, 0x2a, 0x2670af8, 0x8b2b01)
	/usr/local/go/src/testing/testing.go:825 +0x301
testing.runTests.func1(0xc4205573b0)
	/usr/local/go/src/testing/testing.go:1063 +0x64
testing.tRunner(0xc4205573b0, 0xc420745b90)
	/usr/local/go/src/testing/testing.go:777 +0xd0
testing.runTests(0xc4202c0900, 0x38e02a0, 0x202, 0x202, 0x152da0f)
	/usr/local/go/src/testing/testing.go:1061 +0x2c4
testing.(*M).Run(0xc4205e0300, 0x0)
	/usr/local/go/src/testing/testing.go:978 +0x171
github.com/cockroachdb/cockroach/pkg/storage_test.TestMain(0xc4205e0300)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/main_test.go:57 +0x1e3
main.main()
	_testmain.go:1084 +0x151

goroutine 20 [syscall, 30 minutes]:
os/signal.signal_recv(0x0)
	/usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.0
	/usr/local/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 61209 [semacquire]:
sync.runtime_notifyListWait(0xc420dd6590, 0x5ed8d)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420dd6580)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4216d67e0, 0x2a61120, 0xc420dca2a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2a61120, 0xc420dca2a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420546890, 0xc4216d6480, 0xc420546870)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 21 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1171 +0xf1
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:590 +0xf4

goroutine 22 [chan receive, 30 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:597 +0xab
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:591 +0x10c

goroutine 4 [select, 30 minutes, locked to thread]:
runtime.gopark(0x2673490, 0x0, 0x2536d42, 0x6, 0x18, 0x1)
	/usr/local/go/src/runtime/proc.go:291 +0x11a
runtime.selectgo(0xc420491f50, 0xc42041afc0)
	/usr/local/go/src/runtime/select.go:392 +0xe50
runtime.ensureSigM.func1()
	/usr/local/go/src/runtime/signal_unix.go:549 +0x1f4
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1

goroutine 61037 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc42095a400)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:869 +0x241
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:281 +0xdce

goroutine 61092 [semacquire]:
sync.runtime_notifyListWait(0xc420f46610, 0x5d803)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420f46600)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420799d40, 0x2a61120, 0xc420e46150)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2a61120, 0xc420e46150)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204b6440, 0xc4207997a0, 0xc4204b6430)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61385 [semacquire, 29 minutes]:
sync.runtime_notifyListWait(0xc420be1e68, 0xc400000014)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420be1e58)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc420be1d40)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:648 +0x9c
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:635 +0x418

goroutine 61721 [select, 29 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc42132fb00, 0x1, 0xc420dac4e0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:240 +0xff
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).RecvCompress(0xc42132fb00, 0xc420dac4e0, 0xc420c2a650)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:251 +0x2b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc42092f2b0, 0x23eba00, 0xc420dca900, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:525 +0x639
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc420ca5580, 0x23eba00, 0xc420dca900, 0x30, 0x2405760)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:405 +0x43
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc4203be8c0, 0x2405760, 0xc420dca000, 0x227a900)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:366 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x2a76ca0, 0xc4203be8c0, 0xc420417880, 0xc42054a780, 0x2a61120, 0xc420dca000, 0x84f0d9, 0xc400000008)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:421 +0x35
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1(0x2a61120, 0xc420dca000)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:436 +0x66
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203be8f0, 0xc420798870, 0xc42159f3b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 177031 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc420e3cfa0, 0xc421ca5210, 0x5, 0x5, 0xc420b169d0, 0xc4210dfbd0, 0x135ec2d)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:142 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420e3cfa0, 0xc421ca5210, 0x5, 0x5, 0xc42205e000, 0x4, 0x12ace)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:131 +0x67
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc420aa26f0, 0xc421ca5210, 0x5, 0x5, 0xc4210dfb40, 0xc4210dfb00, 0xceff48)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:394 +0x55
io.ReadAtLeast(0x2a3ca20, 0xc420aa26f0, 0xc421ca5210, 0x5, 0x5, 0x5, 0xc420d7e240, 0xc42132e700, 0x5)
	/usr/local/go/src/io/io.go:309 +0x86
io.ReadFull(0x2a3ca20, 0xc420aa26f0, 0xc421ca5210, 0x5, 0x5, 0xc4210dfc40, 0xc4210dfcd8, 0x135adbe)
	/usr/local/go/src/io/io.go:327 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc42132e700, 0xc421ca5210, 0x5, 0x5, 0xc420879200, 0x4e, 0x600)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:378 +0xbf
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc421ca5200, 0x7fffffff, 0xc4210dfcd8, 0xc4210dfd50, 0x8358b9, 0xc420fa6420, 0x60, 0x58)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:452 +0x65
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc421ca5200, 0x2aef763643c8, 0x3b579f8, 0xc42132e700, 0x0, 0x0, 0x23dd4c0, 0xc421e33040, 0x7fffffff, 0xc420fa6420, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:578 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc421f385b0, 0x23dd4c0, 0xc421e33040, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:539 +0x132
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc420b06280, 0x23dd4c0, 0xc421e33040, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:405 +0x43
github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc420414fe0, 0x2a61120, 0xc420aa2750, 0xc420e4b200)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:191 +0x62
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0x2a764c0, 0xc420414fe0, 0xc421f38340, 0x2a61120, 0xc420aa2750, 0xc420e4b200, 0xc3dbaa, 0x2a61120)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:311 +0x35
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2(0x2a61120, 0xc420aa2750)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:319 +0xc0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420415350, 0xc420798870, 0xc421014e00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61414 [chan receive, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3(0x2a61120, 0xc420fb22a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:371 +0x5a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204b6540, 0xc420798870, 0xc42040d0a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61160 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x2a61120, 0xc42101aed0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1604 +0x3ae
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204b7300, 0xc4207997a0, 0xc4209cd1d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61178 [chan receive, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3(0x2a61120, 0xc421163ec0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:371 +0x5a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420546670, 0xc420798870, 0xc42040de00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61207 [semacquire]:
sync.runtime_notifyListWait(0xc420dd6590, 0x5ed8b)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420dd6580)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4216d67e0, 0x2a61120, 0xc420dca240)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2a61120, 0xc420dca240)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420546830, 0xc4216d6480, 0xc420546820)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61000 [semacquire]:
sync.runtime_notifyListWait(0xc4213e8c88, 0xc40002b1f9)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4213e8c78)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc4213e8b60)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:648 +0x9c
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:635 +0x418

goroutine 61204 [semacquire]:
sync.runtime_notifyListWait(0xc420dd6590, 0x5ed89)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420dd6580)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4216d67e0, 0x2a61120, 0xc420dca1b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2a61120, 0xc420dca1b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4205467c0, 0xc4216d6480, 0xc4205467b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61075 [IO wait]:
internal/poll.runtime_pollWait(0x2aef741d9c90, 0x72, 0xc420289b30)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4205e0118, 0x72, 0xffffffffffffff00, 0x2a3ec80, 0x38e40c8)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4205e0118, 0xc420c96000, 0x8000, 0x8000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4205e0100, 0xc420c96000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4205e0100, 0xc420c96000, 0x8000, 0x8000, 0xc420a8e120, 0xc420289d18, 0xc420289c70)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4210ec000, 0xc420c96000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
bufio.(*Reader).Read(0xc4209461e0, 0xc4203a0038, 0x9, 0x9, 0xc420b1a808, 0xc420289d18, 0x1)
	/usr/local/go/src/bufio/bufio.go:216 +0x238
io.ReadAtLeast(0x2a38e60, 0xc4209461e0, 0xc4203a0038, 0x9, 0x9, 0x9, 0x0, 0xc420289d80, 0xd097a6)
	/usr/local/go/src/io/io.go:309 +0x86
io.ReadFull(0x2a38e60, 0xc4209461e0, 0xc4203a0038, 0x9, 0x9, 0xd009db, 0xc42116c000, 0xc420d79a00)
	/usr/local/go/src/io/io.go:327 +0x58
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4203a0038, 0x9, 0x9, 0x2a38e60, 0xc4209461e0, 0x0, 0x100000000, 0xb000001e1, 0x1e100000001)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4203a0000, 0xc420d541e0, 0xc420d541e0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc42058e000, 0xc420e46030, 0x26728a8)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:427 +0x7c
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc420444000, 0x2a7ad20, 0xc42058e000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:676 +0xeb
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2()
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:631 +0x3c
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc42050f220, 0xc420444000, 0x2aef763644f8, 0xc42058e000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:639 +0x27
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x5f9

goroutine 61105 [semacquire]:
sync.runtime_notifyListWait(0xc420f46610, 0x5d7d5)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420f46600)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420799d40, 0x2a61120, 0xc420e463f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2a61120, 0xc420e463f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204b6620, 0xc4207997a0, 0xc4204b6610)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61532 [select, 29 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc421645400)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x14a
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x14f

goroutine 61215 [semacquire]:
sync.runtime_notifyListWait(0xc420dd6590, 0x5ed93)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420dd6580)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4216d67e0, 0x2a61120, 0xc420dca3f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2a61120, 0xc420dca3f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4205469d0, 0xc4216d6480, 0xc4205469c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 169859 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip(0xc4202544b0, 0x2a76520, 0xc4211d3e10, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:185 +0x5d2
github.com/cockroachdb/cockroach/pkg/gossip._Gossip_Gossip_Handler(0x24b8aa0, 0xc4202544b0, 0x2a71180, 0xc421c78790, 0xc420203708, 0xc420b63200)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:208 +0xb2
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc4211ae380, 0x2a7ad20, 0xc42095a000, 0xc42116c200, 0xc42181f080, 0x38b8b20, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1160 +0xa2d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4211ae380, 0x2a7ad20, 0xc42095a000, 0xc42116c200, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1253 +0x12b1
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203d6080, 0xc4211ae380, 0x2a7ad20, 0xc42095a000, 0xc42116c200)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:680 +0x9f
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:678 +0xa1

goroutine 61117 [semacquire]:
sync.runtime_notifyListWait(0xc420f46610, 0x5d7da)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420f46600)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420799d40, 0x2a61120, 0xc420e464b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2a61120, 0xc420e464b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204b68f0, 0xc4207997a0, 0xc4204b68c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61109 [semacquire]:
sync.runtime_notifyListWait(0xc420f46610, 0x5d7e4)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420f46600)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420799d40, 0x2a61120, 0xc420d54450)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2a61120, 0xc420d54450)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204b6710, 0xc4207997a0, 0xc4204b66f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61253 [semacquire]:
sync.runtime_notifyListWait(0xc420dd6590, 0x5ed68)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420dd6580)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4216d67e0, 0x2a61120, 0xc420dca780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2a61120, 0xc420dca780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420546fc0, 0xc4216d6480, 0xc420546fb0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61111 [semacquire]:
sync.runtime_notifyListWait(0xc420f46610, 0x5d7ce)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420f46600)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420799d40, 0x2a61120, 0xc42101ab40)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2a61120, 0xc42101ab40)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204b6760, 0xc4207997a0, 0xc4204b6750)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61218 [semacquire]:
sync.runtime_notifyListWait(0xc420dd6590, 0x5ed95)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420dd6580)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4216d67e0, 0x2a61120, 0xc420dca480)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2a61120, 0xc420dca480)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420546a30, 0xc4216d6480, 0xc420546a20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 159716 [select, 4 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func5(0xc420624300, 0xc420cb8380, 0x2a61120, 0xc4221671a0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:321 +0xfe
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:320 +0xa76

goroutine 61205 [semacquire]:
sync.runtime_notifyListWait(0xc420dd6590, 0x5ed86)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420dd6580)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4216d67e0, 0x2a61120, 0xc420dca1e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2a61120, 0xc420dca1e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4205467e0, 0xc4216d6480, 0xc4205467d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61154 [semacquire, 29 minutes]:
sync.runtime_Semacquire(0xc420799dc4)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc420799db8)
	/usr/local/go/src/sync/waitgroup.go:129 +0x72
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc420799d40, 0x2a61120, 0xc420e46840)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:172 +0x31
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait-fm(0x2a61120, 0xc420e46840)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3822 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204b7240, 0xc4207997a0, 0xc4204b7230)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61296 [semacquire, 29 minutes]:
sync.runtime_notifyListWait(0xc420588608, 0xc400000019)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4205885f8)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc4205884e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:648 +0x9c
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:635 +0x418

goroutine 61437 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc420fa4280, 0xc420378110, 0x5, 0x5, 0x24, 0x0, 0xc4213ae800)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:142 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420fa4280, 0xc420378110, 0x5, 0x5, 0xc420083800, 0x1, 0x2aef741771e8)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:131 +0x67
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc4216f4570, 0xc420378110, 0x5, 0x5, 0x0, 0xc420cf5b48, 0xceff48)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:394 +0x55
io.ReadAtLeast(0x2a3ca20, 0xc4216f4570, 0xc420378110, 0x5, 0x5, 0x5, 0xc42058e600, 0xc421446000, 0x5)
	/usr/local/go/src/io/io.go:309 +0x86
io.ReadFull(0x2a3ca20, 0xc4216f4570, 0xc420378110, 0x5, 0x5, 0x0, 0x0, 0x0)
	/usr/local/go/src/io/io.go:327 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc421446000, 0xc420378110, 0x5, 0x5, 0x0, 0x1, 0x100000001)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:378 +0xbf
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc420378100, 0x7fffffff, 0x0, 0xc420cf5d98, 0x8358b9, 0xc420b0fe60, 0x60, 0x58)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:452 +0x65
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc420378100, 0x2aef763643c8, 0x3b579f8, 0xc421446000, 0x0, 0x0, 0x23eb920, 0xc4203fac60, 0x7fffffff, 0xc420b0fe60, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:578 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc4211b8000, 0x23eb920, 0xc4203fac60, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:757 +0x124
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc4204b6060, 0x2a61120, 0xc4210a6180, 0xc4217ce000)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:434 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x2a76d00, 0xc4204b6060, 0xc42054a780, 0x2a61120, 0xc4210a6180, 0x84f0d9, 0xc400000008)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:318 +0x83
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1(0x2a61120, 0xc4210a6180)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:341 +0x5d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204b6080, 0xc420798870, 0xc4210a6150)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 62985 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc420f81f90, 0xc421f9da30, 0x5, 0x5, 0xc420b169a0, 0xc42221bbd0, 0x135ec2d)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:142 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420f81f90, 0xc421f9da30, 0x5, 0x5, 0xc421a10000, 0x4, 0x12ace)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:131 +0x67
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc4215a2c60, 0xc421f9da30, 0x5, 0x5, 0xc42221bb40, 0xc42221bb00, 0xceff48)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:394 +0x55
io.ReadAtLeast(0x2a3ca20, 0xc4215a2c60, 0xc421f9da30, 0x5, 0x5, 0x5, 0xc420b64000, 0xc421596300, 0x5)
	/usr/local/go/src/io/io.go:309 +0x86
io.ReadFull(0x2a3ca20, 0xc4215a2c60, 0xc421f9da30, 0x5, 0x5, 0xc42221bc40, 0xc42221bcd8, 0x135adbe)
	/usr/local/go/src/io/io.go:327 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc421596300, 0xc421f9da30, 0x5, 0x5, 0xc4221e0c00, 0x4e, 0x600)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:378 +0xbf
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc421f9da20, 0x7fffffff, 0xc42221bcd8, 0xc42221bd50, 0x8358b9, 0xc420c649c0, 0x60, 0x58)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:452 +0x65
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc421f9da20, 0x2aef763643c8, 0x3b579f8, 0xc421596300, 0x0, 0x0, 0x23dd4c0, 0xc421eda3c0, 0x7fffffff, 0xc420c649c0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:578 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc4214e84e0, 0x23dd4c0, 0xc421eda3c0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:539 +0x132
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc420b06300, 0x23dd4c0, 0xc421eda3c0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:405 +0x43
github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc4203618a0, 0x2a61120, 0xc4215a2d50, 0xc421e50c00)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:191 +0x62
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0x2a764c0, 0xc4203618a0, 0xc4214e8410, 0x2a61120, 0xc4215a2d50, 0xc421e50c00, 0xc3dbaa, 0x2a61120)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:311 +0x35
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2(0x2a61120, 0xc4215a2d50)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:319 +0xc0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203619f0, 0xc420798870, 0xc42082b580)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61185 [semacquire]:
sync.runtime_notifyListWait(0xc420dd6590, 0x5ed85)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420dd6580)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4216d67e0, 0x2a61120, 0xc420dca120)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2a61120, 0xc420dca120)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420546760, 0xc4216d6480, 0xc420546750)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 62632 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc420a536d0, 0xc4203c76f0, 0x5, 0x5, 0xc420835e40, 0xc4214f2bd0, 0x135ec2d)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:142 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420a536d0, 0xc4203c76f0, 0x5, 0x5, 0xc420b48000, 0x4, 0x12ace)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:131 +0x67
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc4205b3ef0, 0xc4203c76f0, 0x5, 0x5, 0xc4214f2b40, 0xc4214f2b00, 0xceff48)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:394 +0x55
io.ReadAtLeast(0x2a3ca20, 0xc4205b3ef0, 0xc4203c76f0, 0x5, 0x5, 0x5, 0xc420b64000, 0xc420256900, 0x5)
	/usr/local/go/src/io/io.go:309 +0x86
io.ReadFull(0x2a3ca20, 0xc4205b3ef0, 0xc4203c76f0, 0x5, 0x5, 0xc4214f2c40, 0xc4214f2cd8, 0x135adbe)
	/usr/local/go/src/io/io.go:327 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc420256900, 0xc4203c76f0, 0x5, 0x5, 0xc421a24000, 0x4e, 0x600)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:378 +0xbf
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4203c76e0, 0x7fffffff, 0xc4214f2cd8, 0xc4214f2d50, 0x8358b9, 0xc42163e2a0, 0x60, 0x58)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:452 +0x65
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4203c76e0, 0x2aef763643c8, 0x3b579f8, 0xc420256900, 0x0, 0x0, 0x23dd4c0, 0xc420fb8500, 0x7fffffff, 0xc42163e2a0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:578 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc421126c30, 0x23dd4c0, 0xc420fb8500, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:539 +0x132
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc4203d1380, 0x23dd4c0, 0xc420fb8500, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:405 +0x43
github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc42030e830, 0x2a61120, 0xc4205b3f50, 0xc4208b7e00)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:191 +0x62
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0x2a764c0, 0xc42030e830, 0xc421126b60, 0x2a61120, 0xc4205b3f50, 0xc4208b7e00, 0xc3dbaa, 0x2a61120)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:311 +0x35
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2(0x2a61120, 0xc4205b3f50)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:319 +0xc0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42030ee50, 0xc420798870, 0xc4211dcfc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61439 [select, 29 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func5(0xc420034600, 0xc4205e0380, 0x2a61120, 0xc4210a65a0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:321 +0xfe
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:320 +0xa76

goroutine 39821 [chan receive, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc42037c6c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1223 +0x68
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:80 +0x288

goroutine 151426 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc420805a90, 0xc420332210, 0x5, 0x5, 0xc420b167c0, 0x830264, 0x253dbe9)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:142 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420805a90, 0xc420332210, 0x5, 0x5, 0xc420423a28, 0x57, 0x16)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:131 +0x67
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc421a2f260, 0xc420332210, 0x5, 0x5, 0xc421051b40, 0xc421051b00, 0xceff48)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:394 +0x55
io.ReadAtLeast(0x2a3ca20, 0xc421a2f260, 0xc420332210, 0x5, 0x5, 0x5, 0xc420d7e240, 0xc42132fa00, 0xc400000005)
	/usr/local/go/src/io/io.go:309 +0x86
io.ReadFull(0x2a3ca20, 0xc421a2f260, 0xc420332210, 0x5, 0x5, 0x87ba00, 0xc4208b6d80, 0x4)
	/usr/local/go/src/io/io.go:327 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc42132fa00, 0xc420332210, 0x5, 0x5, 0xc4202edf00, 0xc421051c78, 0x829097)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:378 +0xbf
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc420332200, 0x7fffffff, 0x1, 0xc421051d50, 0x8358b9, 0xc420c64780, 0x60, 0x58)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:452 +0x65
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc420332200, 0x2aef763643c8, 0x3b579f8, 0xc42132fa00, 0x0, 0x0, 0x23dd4c0, 0xc421eda230, 0x7fffffff, 0xc420c64780, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:578 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc420b5d6c0, 0x23dd4c0, 0xc421eda230, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:539 +0x132
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc420b06900, 0x23dd4c0, 0xc421eda230, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:405 +0x43
github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc4220c14e0, 0x2a61120, 0xc421a2f2c0, 0xc421e52a80)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:191 +0x62
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0x2a764c0, 0xc4220c14e0, 0xc420b5d5f0, 0x2a61120, 0xc421a2f2c0, 0xc421e52a80, 0xc3dbaa, 0x2a61120)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:311 +0x35
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2(0x2a61120, 0xc421a2f2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:319 +0xc0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220c1710, 0xc420798870, 0xc421015000)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61181 [chan receive, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1(0x2a61120, 0xc420dca060)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:156 +0x49
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4205466e0, 0xc4216d6480, 0xc42040de60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61387 [IO wait, 29 minutes]:
internal/poll.runtime_pollWait(0x2aef741d93a0, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4203d0f18, 0x72, 0xc420bf4e00, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4203d0f18, 0xffffffffffffff00, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Accept(0xc4203d0f00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:372 +0x1a8
net.(*netFD).accept(0xc4203d0f00, 0xc420828048, 0x0, 0x0)
	/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc4210ec110, 0x87c240, 0xc4212e7e18, 0xc4212e7e20)
	/usr/local/go/src/net/tcpsock_posix.go:136 +0x2e
net.(*TCPListener).Accept(0xc4210ec110, 0x2672850, 0xc420444c40, 0x2a76f40, 0xc420828048)
	/usr/local/go/src/net/tcpsock.go:259 +0x49
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve(0xc420444c40, 0x2a5df20, 0xc4210ec110, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:544 +0x21a
github.com/cockroachdb/cockroach/pkg/util/netutil.ListenAndServeGRPC.func2(0x2a61120, 0xc42101a2a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:57 +0x3c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42038b360, 0xc420798870, 0xc4203b9120)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61223 [semacquire]:
sync.runtime_notifyListWait(0xc420dd6590, 0x5ed9b)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420dd6580)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4216d67e0, 0x2a61120, 0xc420dca570)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2a61120, 0xc420dca570)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420546ad0, 0xc4216d6480, 0xc420546ac0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61288 [select, 29 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x2a61120, 0xc4208349f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x194
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420414400, 0xc4216d6480, 0xc4203865e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61242 [semacquire]:
sync.runtime_notifyListWait(0xc420dd6590, 0x5ed7a)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420dd6580)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4216d67e0, 0x2a61120, 0xc420d54ae0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2a61120, 0xc420d54ae0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420546d90, 0xc4216d6480, 0xc420546d80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 176241 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip(0xc420557b30, 0x2a76520, 0xc42030ea60, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:185 +0x5d2
github.com/cockroachdb/cockroach/pkg/gossip._Gossip_Gossip_Handler(0x24b8aa0, 0xc420557b30, 0x2a71180, 0xc420fc1970, 0xc420203708, 0xc4223a3e60)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:208 +0xb2
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc420444e00, 0x2a7ad20, 0xc42058e600, 0xc421b40200, 0xc421162780, 0x38b8b20, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1160 +0xa2d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc420444e00, 0x2a7ad20, 0xc42058e600, 0xc421b40200, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1253 +0x12b1
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4205258a0, 0xc420444e00, 0x2a7ad20, 0xc42058e600, 0xc421b40200)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:680 +0x9f
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:678 +0xa1

goroutine 61251 [semacquire]:
sync.runtime_notifyListWait(0xc420dd6590, 0x5eda2)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc420dd6580)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4216d67e0, 0x2a61120, 0xc420e46d20)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x2a61120, 0xc420e46d20)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420546f80, 0xc4216d6480, 0xc420546f70)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xe9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad

goroutine 61188 [semacquire]:
sync.runtime_notifyListWait(0xc4213e8e28, 0xc400033bac)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4213e8e18)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc4213e8d00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:648 +0x9c
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:635 +0x418

goroutine 61888 [semacquire, 29 minutes]:
sync.runtime_Semacquire(0xc42184214c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc421842140)
	/usr/local/go/src/sync/waitgroup.go:129 +0x72
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Stop(0xc421842120, 0x2a610a0, 0xc4200e4058)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:420 +0x229
github.com/cockroachdb/cockroach/pkg/storage_test.(*multiTestContext).stopStore(0xc4213e8000, 0x4)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/client_test.go:884 +0x9a
github.com/cockroachdb/cockroach/pkg/storage_test.TestLogGrowthWhenRefreshingPendingCommands.func1(0xc42142d3b0, 0x8131a9501)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/client_raft_test.go:1215 +0x204
github.com/cockroachdb/cockroach/pkg/testutils.RunTrueAndFalse.func1(0xc42142d3b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/testutils/subtest.go:27 +0x3d
testing.tRunner(0xc42142d3b0, 0xc4203198e0)
	/usr/local/go/src/testing/testing.go:777 +0xd0
created by testing.(*T).Run

Metadata

Metadata

Assignees

Labels

A-kv-replicationRelating to Raft, consensus, and coordination.C-test-failureBroken test (automatically or manually discovered).O-robotOriginated from a bot.

Type

No type

Projects

No projects

Milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions