Skip to content

streamclient: panic in TestIdleExit #67592

@knz

Description

@knz

From
https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_UnitTests/3181970?

== RUN   TestIdleExit
    test_log_scope.go:73: test logs captured to: /go/src/github.com/cockroachdb/cockroach/artifacts/logTestIdleExit161562313
    test_log_scope.go:74: use -show-logs to present logs inline
    replication_helpers.go:129: test timed out
panic: close of closed channel

goroutine 1818 [running]:
github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/streamclient.(*sinklessReplicationClient).ConsumePartition.func1(0xc001d24d20, 0xc00191c9c0, 0xc0014be680, 0xc002568e00, 0x56d6f10, 0xc001105100)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/streamclient/cockroach_sinkless_replication_client.go:128 +0x6f5
created by github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/streamclient.(*sinklessReplicationClient).ConsumePartition
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/streamclient/cockroach_sinkless_replication_client.go:88 +0x3f0

goroutine 1 [chan receive]:
testing.(*T).Run(0xc000186f00, 0x43630e7, 0x1d, 0x4968070, 0x4b4806)
	/usr/local/go/src/testing/testing.go:1239 +0x2da
testing.runTests.func1(0xc000186480)
	/usr/local/go/src/testing/testing.go:1511 +0x78
testing.tRunner(0xc000186480, 0xc000be3d60)
	/usr/local/go/src/testing/testing.go:1193 +0xef
testing.runTests(0xc0001b1a58, 0x73b84a0, 0x1, 0x1, 0xc033d483228e8571, 0x1a322c83180, 0x77176a0, 0xc0009c31c0)
	/usr/local/go/src/testing/testing.go:1509 +0x2fe
testing.(*M).Run(0xc0009b8e80, 0x0)
	/usr/local/go/src/testing/testing.go:1417 +0x1eb
github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/streamclient.TestMain(0xc0009b8e80)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/streamclient/main_test.go:31 +0xde
main.main()
	_testmain.go:47 +0x165

goroutine 5 [chan receive]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0000ff040)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173

goroutine 1359 [IO wait]:
internal/poll.runtime_pollWait(0x7fa017219240, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc002658f18, 0x72, 0x800, 0x8b5, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc002658f00, 0xc002b4c000, 0x8b5, 0x8b5, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc002658f00, 0xc002b4c000, 0x8b5, 0x8b5, 0x8b0, 0xc002b4c000, 0x5)
	/usr/local/go/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc0012146c0, 0xc002b4c000, 0x8b5, 0x8b5, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:183 +0x91
crypto/tls.(*atLeastReader).Read(0xc001dce210, 0xc002b4c000, 0x8b5, 0x8b5, 0x8b0, 0xc0006b7000, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:776 +0x63
bytes.(*Buffer).ReadFrom(0xc0020e4978, 0x5655980, 0xc001dce210, 0x427f45, 0x3d74fe0, 0x424d0c0)
	/usr/local/go/src/bytes/buffer.go:204 +0xbe
crypto/tls.(*Conn).readFromUntil(0xc0020e4700, 0x565d860, 0xc0012146c0, 0x5, 0xc0012146c0, 0xc001920960)
	/usr/local/go/src/crypto/tls/conn.go:798 +0xf3
crypto/tls.(*Conn).readRecordOrCCS(0xc0020e4700, 0x0, 0x0, 0xc001920b90)
	/usr/local/go/src/crypto/tls/conn.go:605 +0x115
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:573
crypto/tls.(*Conn).Read(0xc0020e4700, 0xc002b58000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1276 +0x165
bufio.(*Reader).Read(0xc001b03c80, 0xc0025201f8, 0x9, 0x9, 0x5000000000000, 0x101920c98, 0x0)
	/usr/local/go/src/bufio/bufio.go:227 +0x222
io.ReadAtLeast(0x56557a0, 0xc001b03c80, 0xc0025201f8, 0x9, 0x9, 0x9, 0xc001212558, 0x0, 0x0)
	/usr/local/go/src/io/io.go:328 +0x87
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader(0xc0025201f8, 0x9, 0x9, 0x56557a0, 0xc001b03c80, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0025201c0, 0xc002266420, 0x77176a0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc002306000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1347 +0x1a5
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1

goroutine 6 [chan receive]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0000ff110)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173

goroutine 7 [chan receive]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0000ff1e0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173

goroutine 8 [chan receive]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0000ff2b0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173

goroutine 9 [chan receive]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0000ff380)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173

goroutine 10 [chan receive]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0000ff450)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173

goroutine 11 [chan receive]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0000ff520)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173

goroutine 12 [chan receive]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0000ff5f0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173

goroutine 13 [chan receive]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0000ff6c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173

goroutine 14 [chan receive]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0000ff790)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173

goroutine 15 [chan receive]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0000ff860)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173

goroutine 16 [chan receive]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0000ff930)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173

goroutine 50 [chan receive]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0000ffa00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173

goroutine 51 [chan receive]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0000ffad0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173

goroutine 52 [chan receive]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0000ffba0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173

goroutine 53 [chan receive]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0000ffc70)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173

goroutine 54 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:75 +0x74
created by github.com/cockroachdb/cockroach/pkg/util/log.init.6
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:41 +0x35

goroutine 55 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:98 +0x12c
created by github.com/cockroachdb/cockroach/pkg/util/log.init.6
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:42 +0x4d

goroutine 19 [syscall]:
os/signal.signal_recv(0x0)
	/usr/local/go/src/runtime/sigqueue.go:168 +0xa5
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:23 +0x25
created by os/signal.Notify.func1.1
	/usr/local/go/src/os/signal/signal.go:151 +0x45

goroutine 44 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:154 +0x189
created by github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:142 +0x35

goroutine 46 [sleep]:
time.Sleep(0x4c4b40)
	/usr/local/go/src/runtime/time.go:193 +0xd2
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Quiesce(0xc0009b8280, 0x56d6f48, 0xc00019e010)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:524 +0xf4
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Stop(0xc0009b8280, 0x56d6f48, 0xc00019e010)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:469 +0x1e7
github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/streamingtest.NewReplicationHelper.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/streamingtest/replication_helpers.go:210 +0xfe
github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/streamclient.TestSinklessReplicationClient(0xc000186f00)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/streamclient/cockroach_sinkless_replication_client_test.go:113 +0x768
testing.tRunner(0xc000186f00, 0x4968070)
	/usr/local/go/src/testing/testing.go:1193 +0xef
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:1238 +0x2b3

goroutine 82 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ab6150)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ab6150, 0xc0001ecee0, 0x1, 0x1, 0xc0002e3fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000592400)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 83 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ab6330)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ab6330, 0xc0001ecee0, 0x1, 0x1, 0xc0002e57b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000592500)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 84 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ab6420)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ab6420, 0xc0001ecee0, 0x1, 0x1, 0xc0002e5fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000593a00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 85 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ab64b0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ab64b0, 0xc0001ecee0, 0x1, 0x1, 0xc0002e67b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000593e00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 86 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ab6570)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ab6570, 0xc0001ecee0, 0x1, 0x1, 0xc0002e6fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000214300)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 87 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ab65d0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ab65d0, 0xc0001ecee0, 0x1, 0x1, 0xc0002e77b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000214400)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 88 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ab6660)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ab6660, 0xc0001ecee0, 0x1, 0x1, 0xc0002e7fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000214500)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 89 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ab66c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ab66c0, 0xc0001ecee0, 0x1, 0x1, 0xc000357fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000214700)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 90 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ab6060)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ab6060, 0xc0001ecee0, 0x1, 0x1, 0xc00012c7b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000214800)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 91 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ab67b0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ab67b0, 0xc0001ecee0, 0x1, 0x1, 0xc00012cfb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000302d00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 92 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ab6960)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ab6960, 0xc0001ecee0, 0x1, 0x1, 0xc00012d7b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061a000)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 93 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ab6a80)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ab6a80, 0xc0001ecee0, 0x1, 0x1, 0xc00012ffb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061a200)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 94 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ab6720)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ab6720, 0xc0001ecee0, 0x1, 0x1, 0xc0001307b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061a300)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 95 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ab6c60)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ab6c60, 0xc0001ecee0, 0x1, 0x1, 0xc000130fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061a400)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 96 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ab6b70)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ab6b70, 0xc0001ecee0, 0x1, 0x1, 0xc0001317b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061a600)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 97 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ab6cc0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ab6cc0, 0xc0001ecee0, 0x1, 0x1, 0xc000131fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061a700)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 2219 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc000386fe0, 0x46)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000386fd0)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:193
github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc000386f00, 0x56d6fb8, 0xc00218c450)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:396 +0x10b
runtime/pprof.Do(0x56d6f48, 0xc00218c450, 0xc0001ecde0, 0x1, 0x1, 0xc001885fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble/record.NewLogWriter.func2(0xc000386f00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:314 +0x89
created by github.com/cockroachdb/pebble/record.NewLogWriter
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:313 +0x2ea

goroutine 103 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc0009f9ae0, 0xc000ae4480)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:180 +0x9c
created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue
	/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:176 +0x18f

goroutine 104 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc0009f9b80, 0xc000ae46c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:180 +0x9c
created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue
	/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:176 +0x18f

goroutine 105 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc0009f9c20, 0xc000ae4ba0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:180 +0x9c
created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue
	/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:176 +0x18f

goroutine 106 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc0009f9cc0, 0xc000ae5260)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:180 +0x9c
created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue
	/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:176 +0x18f

goroutine 107 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc0009f9d60, 0xc000ae5560)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:180 +0x9c
created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue
	/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:176 +0x18f

goroutine 108 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ffed80)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ffed80, 0xc0001ecee0, 0x1, 0x1, 0xc0005f77b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061b000)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 109 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ffede0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ffede0, 0xc0001ecee0, 0x1, 0x1, 0xc0005f7fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061b100)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 110 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ffee40)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ffee40, 0xc0001ecee0, 0x1, 0x1, 0xc0005f87b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061b300)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 111 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ffeea0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ffeea0, 0xc0001ecee0, 0x1, 0x1, 0xc0005f8fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061b400)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 112 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ffef00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ffef00, 0xc0001ecee0, 0x1, 0x1, 0xc0005f97b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061b500)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 113 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ffef60)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ffef60, 0xc0001ecee0, 0x1, 0x1, 0xc0005f9fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061b600)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 114 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ffefc0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ffefc0, 0xc0001ecee0, 0x1, 0x1, 0xc0011fe7b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061b700)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 115 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000fff020)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000fff020, 0xc0001ecee0, 0x1, 0x1, 0xc0011fefb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061b800)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 116 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000fff080)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000fff080, 0xc0001ecee0, 0x1, 0x1, 0xc0011ff7b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061b900)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 117 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000fff0e0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000fff0e0, 0xc0001ecee0, 0x1, 0x1, 0xc0011fffb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061ba00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 118 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000fff140)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000fff140, 0xc0001ecee0, 0x1, 0x1, 0xc0005f47b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061bb00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 119 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000fff1a0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000fff1a0, 0xc0001ecee0, 0x1, 0x1, 0xc0005f4fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061bc00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 120 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000fff200)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000fff200, 0xc0001ecee0, 0x1, 0x1, 0xc0000c1fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061bd00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 121 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000fff260)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000fff260, 0xc0001ecee0, 0x1, 0x1, 0xc0005f5fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061be00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 122 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000fff2c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000fff2c0, 0xc0001ecee0, 0x1, 0x1, 0xc0011fa7b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00061bf00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 123 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc000ffed20)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc000ffed20, 0xc0001ecee0, 0x1, 0x1, 0xc0011fafb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0006e0000)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 47 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc0003800e0, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc0003800d0)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:193
github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc000380000, 0x56d6fb8, 0xc000a0e930)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:396 +0x10b
runtime/pprof.Do(0x56d6f48, 0xc000a0e930, 0xc0001ecde0, 0x1, 0x1, 0xc00012f7b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble/record.NewLogWriter.func2(0xc000380000)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:314 +0x89
created by github.com/cockroachdb/pebble/record.NewLogWriter
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:313 +0x2ea

goroutine 151 [select]:
github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps.func1(0x4970f70, 0xc0009e43c0, 0xc0007c6460, 0x56d6fb8, 0xc0009f1e60, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:239 +0xff
created by github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps
	/go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:232 +0xd0

goroutine 319 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.(*ioLoadListener).start.func2(0x496d008, 0xc00133d050, 0xc000ad09a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/granter.go:1345 +0xe8
created by github.com/cockroachdb/cockroach/pkg/util/admission.(*ioLoadListener).start
	/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/granter.go:1339 +0x154

goroutine 1536 [semacquire]:
sync.runtime_Semacquire(0xc00053b9c8)
	/usr/local/go/src/runtime/sema.go:56 +0x45
sync.(*WaitGroup).Wait(0xc00053b9c0)
	/usr/local/go/src/sync/waitgroup.go:130 +0x65
google.golang.org/grpc.(*Server).serveStreams(0xc000345500, 0x5733778, 0xc002582780)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:944 +0x111
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000345500, 0x5733778, 0xc002582780, 0xc0009d4010, 0xf)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:868 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:867 +0x555

goroutine 2275 [runnable]:
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/schemafeed.doNothingSchemaFeed.Run(0x56d6f10, 0xc001539800, 0x6, 0x56a4530)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/schemafeed/schema_feed.go:675 +0x48
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0xc0009b8280, 0x496d2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc002100de0, 0xc002100e10)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 1528 [runnable]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc001538480)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:69 +0xac
created by google.golang.org/grpc.newCCBalancerWrapper
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:60 +0x172

goroutine 1767 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc00205b2c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc00205b2c0, 0xc0001ecee0, 0x1, 0x1, 0xc0018b2fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d6b600)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 1762 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc00205b3e0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc00205b3e0, 0xc0001ecee0, 0x1, 0x1, 0xc0018b77b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d6b000)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 2258 [runnable]:
runtime/debug.Stack(0xc001a29a40, 0x3da8c40, 0xc00321cf00)
	/usr/local/go/src/runtime/debug/stack.go:28 +0x73
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError.func1(0xc001a29e90)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:41 +0x70
panic(0x3da8c40, 0xc00321cf00)
	/usr/local/go/src/runtime/panic.go:965 +0x1b9
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.ExpectedError(...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:200
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Next(0xc002626800, 0xc001a29d00, 0x4b4806)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:227 +0x851
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*InvariantsChecker).Next(0xc0015390c0, 0xc001a29e00, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/invariants_checker.go:79 +0x69
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*batchInfoCollector).Next(0xc000d76d00, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/stats.go:98 +0x49
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:239
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc001a29f50, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:243
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc0005ad080, 0x56d6f10, 0xc001539080)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:282 +0x197
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc00266b4f0, 0x56d6f10, 0xc001539080, 0x49691a8)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:255 +0x234
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc001632b40, 0xc001388c40, 0xc002181900, 0xc002658500, 0xc001ccf180, 0xc0026147a8, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:525 +0x4d7
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc001632b40, 0x56f1818, 0xc0004ca570, 0xc0026147a8, 0xc001388c40, 0xc002181900, 0x56d8598, 0xc002658280, 0x0, 0xc001ccf180, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1262 +0x1cb
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc002614000, 0x56f1818, 0xc0004ca570, 0xc002614578, 0x3, 0x7f9feb08b390, 0xc0005ddc98, 0xc001a2a900, 0xc002c15678, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1101 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc002614000, 0x56f1818, 0xc0004ca570, 0xc002614578, 0x7f9feb08b390, 0xc0005ddc98, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:963 +0x7bd
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc002614000, 0x56f1818, 0xc0004ca570, 0x56fc258, 0xc0005acf20, 0xc0018e3032, 0x54, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:681 +0xfc6
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc002614000, 0x56d6f10, 0xc001538e80, 0x56fc258, 0xc0005acf20, 0xc0018e3032, 0x54, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:125 +0xb8f
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1(0x56fc258, 0xc0005acf20, 0xc0018e3032, 0x54, 0x0, 0x0, 0x2d552023, 0xed880ca85, 0x0, 0x2d552db5, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1603 +0x233
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc002614000, 0x56d6f10, 0xc001538e80, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1605 +0x1691
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc002614000, 0x56d6f10, 0xc001105540, 0xc00134e140, 0x5400, 0x15000, 0xc00134e320, 0xc000db7470, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1527 +0x225
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc000a3a580, 0x56d6f10, 0xc001105540, 0xc002614000, 0x5400, 0x15000, 0xc00134e320, 0xc000db7470, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:615 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc00053b48e, 0xc002b084e0, 0x56d6f10, 0xc001105540, 0xc000db7470, 0xc000a3a580, 0xc0005dd600, 0x57303e0, 0xc002720100, 0xc0004a44e0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:645 +0x3f5
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:573 +0x185

goroutine 1360 [runnable]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0027da640, 0x1, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:407 +0xff
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001b03d40, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:527 +0x1dd
google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc002306000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae

goroutine 2289 [runnable]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00218a4c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:69 +0xac
created by google.golang.org/grpc.newCCBalancerWrapper
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:60 +0x172

goroutine 2161 [runnable]:
database/sql.(*DB).connectionOpener(0xc0014be680, 0x56d6f10, 0xc001105140)
	/usr/local/go/src/database/sql/sql.go:1133 +0xb5
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:740 +0x12a

goroutine 2277 [runnable]:
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvevent.(*blockingBuffer).Get(0xc002c15aa0, 0x56d6f10, 0xc001539880, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvevent/blocking_buffer.go:82 +0x10b
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed.copyFromSourceToSinkUntilTableEvent(0x56d6f10, 0xc001539880, 0x7f9fecd0cb00, 0xc0012145a8, 0x7f9fecce73a0, 0xc001e80020, 0xc002100cc0, 0x1, 0x1, 0x1691a6e781cabc0d, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed/kv_feed.go:483 +0x317
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed.(*kvFeed).runUntilTableEvent.func1(0x56d6f10, 0xc001539880, 0xc000c77fa8, 0xb3f26b)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed/kv_feed.go:367 +0x155
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0xc002582780, 0xc0015db440)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc002100ea0, 0xc002100ed0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 2252 [chan receive]:
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0005dd600, 0x56d6fb8, 0xc002c32f30, 0xc000db7430, 0xc000a3a580, 0x5400, 0x15000, 0xc00134e320, 0x0, 0x4, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:505 +0x65f
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00070c200, 0x56d6fb8, 0xc002c32f30, 0x572d418, 0xc0020bfc00, 0xc000d95009, 0x4, 0xc002c32e40, 0x569bf20, 0xc0020f15c0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:160 +0x1d8
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00070c200, 0x56d6fb8, 0xc002c32f30, 0x5731460, 0xc0020bfc00, 0x56d6f01, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:691 +0xe25
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x5731460, 0xc003134d48)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2274 +0xf7
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc0009b8280, 0x56d6fb8, 0xc0009f1e60, 0xc000634c58, 0x5731460, 0xc003134d48, 0xc001ae2960)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x256

goroutine 1524 [runnable]:
google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0018ea000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1421 +0x17a
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b

goroutine 2225 [semacquire]:
sync.runtime_Semacquire(0xc002100df0)
	/usr/local/go/src/runtime/sema.go:56 +0x45
sync.(*WaitGroup).Wait(0xc002100de8)
	/usr/local/go/src/sync/waitgroup.go:130 +0x65
golang.org/x/sync/errgroup.(*Group).Wait(0xc002100de0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait(0xc002100de0, 0x56d6f10, 0xc001539800, 0xc001e80010, 0xc001539800)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:142 +0x57
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed.Run(0x56f1818, 0xc0004cb0b0, 0xc000526000, 0xc0009b8e00, 0xc0006ad728, 0xc0006ad728, 0x0, 0xc0009e43c0, 0x42fd500, 0xc000332340, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed/kv_feed.go:104 +0x630
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changeAggregator).startKVFeed.func1(0x56f1818, 0xc0004cb0b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:295 +0xfe
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc0009b8280, 0x56f1818, 0xc0004cb0b0, 0xc0004caf00, 0xc0028d3d28)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:347 +0xb9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:342 +0xfc

goroutine 1764 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc00205b4a0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc00205b4a0, 0xc0001ecee0, 0x1, 0x1, 0xc000c73fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d6b200)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 2274 [runnable]:
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changeFrontier).Start.func1(0x56f1818, 0xc0004cacf0, 0xc001e8e900)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:1069 +0x48
created by github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changeFrontier).Start
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:1068 +0x446

goroutine 1775 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc001f35e00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc001f35e00, 0xc0001ecee0, 0x1, 0x1, 0xc00189d7b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d6bf00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 1899 [runnable]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc001942620, 0x56d6f10, 0xc0018c7080, 0xc0018c7080, 0xc000cf6f80)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:254 +0x1b8
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc001942620, 0x56d6f10, 0xc0018c7080)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:271 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1(0x56d6f10, 0xc0029a4840)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:259 +0x65
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc0009b8280, 0x56d6f10, 0xc0029a4840, 0x0, 0xc000a41b78)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:347 +0xb9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:342 +0xfc

goroutine 1774 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc00205b7a0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc00205b7a0, 0xc0001ecee0, 0x1, 0x1, 0xc001898fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d6bd00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 1769 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc00205b680)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc00205b680, 0xc0001ecee0, 0x1, 0x1, 0xc0018b67b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d6b800)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 1766 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc00205b560)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc00205b560, 0xc0001ecee0, 0x1, 0x1, 0xc000c777b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d6b500)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 847 [select]:
database/sql.(*DB).connectionOpener(0xc000bc3ba0, 0x56d6f10, 0xc0020ea640)
	/usr/local/go/src/database/sql/sql.go:1133 +0xb5
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:740 +0x12a

goroutine 1681 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc00205b380)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc00205b380, 0xc0001ecee0, 0x1, 0x1, 0xc0018927b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d6af00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 1817 [runnable]:
database/sql.(*Rows).awaitDone(0xc002568e00, 0x56d6f10, 0xc0032b16c0, 0x0, 0x0)
	/usr/local/go/src/database/sql/sql.go:2827 +0x9f
created by database/sql.(*Rows).initContextClose
	/usr/local/go/src/database/sql/sql.go:2815 +0xc9

goroutine 2280 [semacquire]:
sync.runtime_Semacquire(0xc002101060)
	/usr/local/go/src/runtime/sema.go:56 +0x45
sync.(*WaitGroup).Wait(0xc002101058)
	/usr/local/go/src/sync/waitgroup.go:130 +0x65
golang.org/x/sync/errgroup.(*Group).Wait(0xc002101050, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait(0xc002101050, 0x56d6f10, 0xc001539a00, 0xc002c15d40, 0xc001539a00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:142 +0x57
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc0006d3400, 0x56d6f10, 0xc0004cb290, 0xc0004bde00, 0x2, 0x8, 0xc0004bde08, 0x2, 0x2, 0x1691a6e781cabc0d, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:83 +0x3ca
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed.rangefeedFactory.Run.func1(0x56d6f10, 0xc001539980, 0xc0009b8280, 0x70000000496d2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed/physical_kv_feed.go:77 +0xca
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0xc0009b8280, 0x496d2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc002100f60, 0xc002100fc0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 1768 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc00205b620)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc00205b620, 0xc0001ecee0, 0x1, 0x1, 0xc0018b37b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d6b700)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 1773 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc001fda6f0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc001fda6f0, 0xc0001ecee0, 0x1, 0x1, 0xc0018997b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d6bc00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 1772 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc00205b740)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc00205b740, 0xc0001ecee0, 0x1, 0x1, 0xc0018987b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d6bb00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 1771 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc00205b5c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc00205b5c0, 0xc0001ecee0, 0x1, 0x1, 0xc00189cfb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d6ba00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 1776 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc00038b9e0, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00038b9d0)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:193
github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc00038b900, 0x56d6fb8, 0xc001fda750)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:396 +0x10b
runtime/pprof.Do(0x56d6f48, 0xc001fda750, 0xc0001ecde0, 0x1, 0x1, 0xc0011f9fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble/record.NewLogWriter.func2(0xc00038b900)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:314 +0x89
created by github.com/cockroachdb/pebble/record.NewLogWriter
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:313 +0x2ea

goroutine 1898 [runnable]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(0xc00212b500, 0xc0009add40, 0x5720688, 0xc000cf6f60, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:242 +0x670
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc000688000, 0xc0009add40, 0x5720688, 0xc000cf6f60, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2520 +0x12d
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc000148ee0, 0xc0009add40, 0x5720688, 0xc000cf6f60, 0x10)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:216 +0xce
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc0003a3200, 0xc0009add40, 0x5720688, 0xc000cf6f60, 0xc0003a3200, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:1064 +0x54
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_RangeFeed_Handler(0x42b26c0, 0xc0003a3200, 0x5718698, 0xc001d76e10, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:7858 +0x113
github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1(0x42b26c0, 0xc0003a3200, 0x5718698, 0xc001d76e10, 0xc000a41b00, 0x4968b40, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:170 +0x5f7
google.golang.org/grpc.getChainStreamHandler.func1(0x42b26c0, 0xc0003a3200, 0x5718698, 0xc001d76e10, 0x4087900, 0xc0018c6f40)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1408 +0xdd
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2(0x42b26c0, 0xc0003a3200, 0x5718698, 0xc001d76e10, 0xc000a41b00, 0xc0018c6f40, 0xc0018c6f40, 0x42a918)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:182 +0x99
google.golang.org/grpc.getChainStreamHandler.func1(0x42b26c0, 0xc0003a3200, 0x5718698, 0xc001d76e10, 0xc001d76de0, 0x7fa0172cc3c8)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1408 +0xdd
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor(0x42b26c0, 0xc0003a3200, 0x571ae40, 0xc001dea480, 0xc000a41b00, 0xc0018c6f00, 0x4087900, 0xc0018c6f00)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:97 +0xb1
google.golang.org/grpc.chainStreamServerInterceptors.func1(0x42b26c0, 0xc0003a3200, 0x571ae40, 0xc001dea480, 0xc000a41b00, 0x4968b40, 0x56d6fb8, 0xc001d76d20)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1394 +0xbd
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000345500, 0x5733778, 0xc002582780, 0xc0015f8a20, 0xc0006bab70, 0x73c59a0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1540 +0x535
google.golang.org/grpc.(*Server).handleStream(0xc000345500, 0x5733778, 0xc002582780, 0xc0015f8a20, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1613 +0xca5
google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00053b9c0, 0xc000345500, 0x5733778, 0xc002582780, 0xc0015f8a20)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:934 +0xab
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:932 +0x1fd

goroutine 1770 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc00205b6e0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc00205b6e0, 0xc0001ecee0, 0x1, 0x1, 0xc00189e7b8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d6b900)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 2306 [chan receive]:
google.golang.org/grpc.(*addrConn).resetTransport(0xc002fd3080)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1214 +0x465
created by google.golang.org/grpc.(*addrConn).connect
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:844 +0x12a

goroutine 1756 [runnable]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc000e88000, 0x56d6f10, 0xc0029a4980, 0xc0029a4980, 0xc000c3b2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:254 +0x1b8
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc000e88000, 0x56d6f10, 0xc0029a4980)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:271 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1(0x56d6f10, 0xc0029a4840)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:259 +0x65
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc0009b8280, 0x56d6f10, 0xc0029a4840, 0x0, 0xc000efe900)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:347 +0xb9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:342 +0xfc

goroutine 2281 [runnable]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1(0x56d6f10, 0xc001539a00, 0xc0009b8280, 0x70000000496d2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:66 +0x12f
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0xc0009b8280, 0x496d2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc002101050, 0xc002101080)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 2278 [semacquire]:
sync.runtime_Semacquire(0xc002100f70)
	/usr/local/go/src/runtime/sema.go:56 +0x45
sync.(*WaitGroup).Wait(0xc002100f68)
	/usr/local/go/src/sync/waitgroup.go:130 +0x65
golang.org/x/sync/errgroup.(*Group).Wait(0xc002100f60, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait(0xc002100f60, 0x56d6f10, 0xc001539980, 0xc002658800, 0xc001539980)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:142 +0x57
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed.rangefeedFactory.Run(0xc0028335a0, 0x56d6f10, 0xc001539880, 0x7f9fecd0cb50, 0xc001e80020, 0xc002100cc0, 0x1, 0x1, 0x1691a6e781cabc0d, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed/physical_kv_feed.go:80 +0x355
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed.(*kvFeed).runUntilTableEvent.func2(0x56d6f10, 0xc001539880, 0xc000c797a8, 0x700000000b3f26b)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed/kv_feed.go:370 +0x123
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0xc0009b8280, 0x496d2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc002100ea0, 0xc002100f00)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 2279 [runnable]:
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed.(*rangefeed).addEventsToBuffer(0xc0027033b0, 0x56d6f10, 0xc001539980, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed/physical_kv_feed.go:86 +0xdd
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0xc0009b8280, 0x496d2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc002100f60, 0xc002100f90)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 1361 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0027dab40, 0x1, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:407 +0xff
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0028c4360, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:527 +0x1dd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc001a80f00)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:292 +0xd7
created by google.golang.org/grpc/internal/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:289 +0x1110

goroutine 2323 [IO wait]:
internal/poll.runtime_pollWait(0x7fa017219158, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc002677318, 0x72, 0x700, 0x773, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc002677300, 0xc001d58000, 0x773, 0x773, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc002677300, 0xc001d58000, 0x773, 0x773, 0xc001d58005, 0xc00152ec98, 0xc001974828)
	/usr/local/go/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc003134d70, 0xc001d58000, 0x773, 0x773, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:183 +0x91
github.com/cockroachdb/cmux.(*MuxConn).Read(0xc001cf59e0, 0xc001d58000, 0x773, 0x773, 0x42a1bb, 0xc00200c5a0, 0x18)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c
crypto/tls.(*atLeastReader).Read(0xc00200c5a0, 0xc001d58000, 0x773, 0x773, 0xc003193900, 0xc000207800, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:776 +0x63
bytes.(*Buffer).ReadFrom(0xc0020e4cf8, 0x5655980, 0xc00200c5a0, 0x427f45, 0x3d74fe0, 0x4092460)
	/usr/local/go/src/bytes/buffer.go:204 +0xbe
crypto/tls.(*Conn).readFromUntil(0xc0020e4a80, 0x7f9feb158a38, 0xc001cf59e0, 0x5, 0xc001cf59e0, 0xc0019749e8)
	/usr/local/go/src/crypto/tls/conn.go:798 +0xf3
crypto/tls.(*Conn).readRecordOrCCS(0xc0020e4a80, 0x0, 0x0, 0xab87be)
	/usr/local/go/src/crypto/tls/conn.go:605 +0x115
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:573
crypto/tls.(*Conn).Read(0xc0020e4a80, 0xc002b70000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1276 +0x165
bufio.(*Reader).Read(0xc0028c4300, 0xc0025203b8, 0x9, 0x9, 0xc00200c588, 0xc001974ce0, 0x1)
	/usr/local/go/src/bufio/bufio.go:227 +0x222
io.ReadAtLeast(0x56557a0, 0xc0028c4300, 0xc0025203b8, 0x9, 0x9, 0x9, 0x1, 0xc00200c588, 0x0)
	/usr/local/go/src/io/io.go:328 +0x87
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader(0xc0025203b8, 0x9, 0x9, 0x56557a0, 0xc0028c4300, 0x0, 0xc000000000, 0x31, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89
golang.org/x/net/http2.(*Framer).ReadFrame(0xc002520380, 0xc001ddcf90, 0x77176a0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001a80f00, 0xc001ddd0e0, 0x496ed98)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:473 +0x9b
google.golang.org/grpc.(*Server).serveStreams(0xc000345500, 0x5733778, 0xc001a80f00)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:918 +0x103
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000345500, 0x5733778, 0xc001a80f00, 0xc002ba2650, 0xf)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:868 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:867 +0x555

goroutine 2322 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001a80f00)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:993 +0x265
created by google.golang.org/grpc/internal/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:300 +0x1135

goroutine 1680 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc00205b320)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc00205b320, 0xc0001ecee0, 0x1, 0x1, 0xc0018b6fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d6ae00)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 2223 [runnable]:
github.com/lib/pq.(*conn).watchCancel.func1(0xc002067740, 0xc0013b02c0, 0xc000046ae0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:94 +0x85
created by github.com/lib/pq.(*conn).watchCancel
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:93 +0xa5

goroutine 1763 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc00205b440)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc00205b440, 0xc0001ecee0, 0x1, 0x1, 0xc0018b7fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d6b100)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 792 [runnable]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00218a380)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:69 +0xac
created by google.golang.org/grpc.newCCBalancerWrapper
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:60 +0x172

goroutine 1765 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x56d6fb8, 0xc00205b500)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x56d6f48, 0xc00205b500, 0xc0001ecee0, 0x1, 0x1, 0xc001897fb8)
	/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d6b400)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5

goroutine 2276 [semacquire]:
sync.runtime_Semacquire(0xc002100eb0)
	/usr/local/go/src/runtime/sema.go:56 +0x45
sync.(*WaitGroup).Wait(0xc002100ea8)
	/usr/local/go/src/sync/waitgroup.go:130 +0x65
golang.org/x/sync/errgroup.(*Group).Wait(0xc002100ea0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait(0xc002100ea0, 0x56d6f10, 0xc001539880, 0xc002c15c20, 0xc001539880)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:142 +0x57
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed.(*kvFeed).runUntilTableEvent(0xc000d77040, 0x56d6f10, 0xc001539800, 0x1691a6e781cabc0d, 0x1691a60000000000, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed/kv_feed.go:377 +0x3d1
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed.(*kvFeed).run(0xc000d77040, 0x56d6f10, 0xc001539800, 0xc001e53850, 0x4970cc8)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvfeed/kv_feed.go:201 +0x116
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0xc002582780, 0xc0015db440)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc002100de0, 0xc002100e40)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 2224 [runnable]:
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvevent.(*chanBuffer).Get(0xc0012145a8, 0x56d6f10, 0xc001539600, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/kvevent/chan_buffer.go:69 +0x11f
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*bufEventProducer).GetEvent(0xc002833570, 0x56d6f10, 0xc001539600, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:590 +0xd1
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changeAggregator).tick(0xc00316ce00, 0x0, 0x100000001)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:477 +0xa5
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changeAggregator).Next(0xc00316ce00, 0xc002101200, 0x3, 0x3, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:452 +0x85
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changeFrontier).Next(0xc001e8e900, 0xc001e8e9d8, 0x3, 0x3, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:1147 +0xec
github.com/cockroachdb/cockroach/pkg/sql/execinfra.Run(0x56f1818, 0xc0004cacf0, 0x56f4898, 0xc001e8e900, 0x569a260, 0xc001939880)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/base.go:186 +0x35
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc001e8e900, 0x56d6f10, 0xc0015394c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:731 +0x90
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc001106500, 0x56d6f10, 0xc0015394c0, 0x49691a8)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:393 +0x2c4
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc001632b40, 0xc001388ee0, 0x0, 0xc001a69b28, 0xc002c13180, 0xc0026d0f00, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:525 +0x4d7
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeeddist.StartDistChangefeed(0x56f1818, 0xc0004ca930, 0x572daa0, 0xc002614578, 0x0, 0x0, 0x0, 0x0, 0xc002100a80, 0x1691a6e781cabc0d, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeeddist/distflow.go:144 +0xac6
github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/streamproducer.streamKVs(0x56f1818, 0xc0004ca930, 0x5815f70, 0xc002614578, 0x1691a6e781cabc0d, 0x0, 0xc002100a50, 0x1, 0x1, 0xc0000470e0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/streamproducer/replication_stream_planning.go:101 +0x368
github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/streamproducer.doCreateReplicationStream(0x56f1818, 0xc0004ca930, 0x5815f70, 0xc002614578, 0xc002832900, 0xc0000470e0, 0xc001884f8c, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/streamproducer/replication_stream_planning.go:158 +0x3b3
github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/streamproducer.createReplicationStreamHook.func1(0x56f1818, 0xc0004ca930, 0x0, 0x0, 0x0, 0xc0000470e0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/streamproducer/replication_stream_planning.go:196 +0x65
github.com/cockroachdb/cockroach/pkg/sql.(*hookFnNode).startExec.func1(0xc002658280, 0x56f1818, 0xc0004ca930, 0xc0026d0c00, 0xc002614578)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/planhook.go:142 +0x7d
created by github.com/cockroachdb/cockroach/pkg/sql.(*hookFnNode).startExec
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/planhook.go:141 +0xcc
FAIL	github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/streamclient	11.461s

Epic CRDB-7778

Metadata

Metadata

Assignees

No one assigned

    Type

    No type

    Projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions