changefeedccl: TestChangefeedNemeses/webhook - 887.84s #69175
Description
TestChangefeedNemeses/webhook is getting stuck and timing out the package.
Example Build - https://teamcity.cockroachdb.com//viewLog.html?buildId=3327837&buildTypeId=Cockroach_MergeToMaster
Stacks:
panic: test timed out after 45m0s
goroutine 683736 [running]:
testing.(*M).startAlarm.func1()
/usr/local/go/src/testing/testing.go:1700 +0xe5
created by time.goFunc
/usr/local/go/src/time/sleep.go:180 +0x45
goroutine 1 [chan receive, 35 minutes]:
testing.(*T).Run(0xc00716ec00, 0x4edb1c0, 0x15, 0x553f200, 0x4b3401)
/usr/local/go/src/testing/testing.go:1239 +0x2da
testing.runTests.func1(0xc000138a80)
/usr/local/go/src/testing/testing.go:1511 +0x78
testing.tRunner(0xc000138a80, 0xc000d7fd60)
/usr/local/go/src/testing/testing.go:1193 +0xef
testing.runTests(0xc000450618, 0x87226a0, 0x59, 0x59, 0xc03fd13e992347ac, 0x274b7406d2d, 0x8aa6700, 0xc000a666c0)
/usr/local/go/src/testing/testing.go:1509 +0x2fe
testing.(*M).Run(0xc000647480, 0x0)
/usr/local/go/src/testing/testing.go:1417 +0x1eb
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.TestMain(0xc000647480)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/main_test.go:31 +0xde
main.main()
_testmain.go:273 +0x165
goroutine 29 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc0003d1780)
/go/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:276 +0xcd
created by go.opencensus.io/stats/view.init.0
/go/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:34 +0x68
goroutine 25 [syscall, 45 minutes]:
os/signal.signal_recv(0x5548fa8)
/usr/local/go/src/runtime/sigqueue.go:168 +0xa5
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:23 +0x25
created by os/signal.Notify.func1.1
/usr/local/go/src/os/signal/signal.go:151 +0x45
goroutine 14 [chan receive, 45 minutes]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0002b8a90)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173
goroutine 15 [chan receive, 45 minutes]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0002b8b60)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173
goroutine 16 [chan receive, 45 minutes]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0002b8c30)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173
goroutine 50 [chan receive, 45 minutes]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0002b8d00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173
goroutine 51 [chan receive, 45 minutes]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0002b8dd0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173
goroutine 52 [chan receive, 45 minutes]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0002b8ea0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173
goroutine 53 [chan receive, 45 minutes]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0002b8f70)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173
goroutine 54 [chan receive, 45 minutes]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0002b9040)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173
goroutine 55 [chan receive, 45 minutes]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0002b9110)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173
goroutine 56 [chan receive, 45 minutes]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0002b91e0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173
goroutine 57 [chan receive, 45 minutes]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0002b92b0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173
goroutine 58 [chan receive, 45 minutes]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0002b9380)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173
goroutine 59 [chan receive, 45 minutes]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0002b9450)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173
goroutine 60 [chan receive, 45 minutes]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0002b9520)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173
goroutine 61 [chan receive, 45 minutes]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0002b95f0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173
goroutine 62 [chan receive, 45 minutes]:
github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0002b96c0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149
created by github.com/klauspost/compress/zstd.newBlockDec
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173
goroutine 63 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:75 +0x74
created by github.com/cockroachdb/cockroach/pkg/util/log.init.6
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:41 +0x35
goroutine 64 [chan receive, 45 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:98 +0x12c
created by github.com/cockroachdb/cockroach/pkg/util/log.init.6
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:42 +0x4d
goroutine 28 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0.func1()
/go/src/github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:154 +0x189
created by github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0
/go/src/github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:142 +0x35
goroutine 675538 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc005226240)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc005226240, 0xc0006b4300, 0x1, 0x1, 0xc00249b7b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008b1e500)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675512 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc003a5e2d0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc003a5e2d0, 0xc0006b4300, 0x1, 0x1, 0xc00206d7b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc009015e00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675516 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0007cd9b0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0007cd9b0, 0xc0006b4300, 0x1, 0x1, 0xc002a2c7b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006bc2b00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675523 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0007cde30)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0007cde30, 0xc0006b4300, 0x1, 0x1, 0xc0002c67b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006bc3800)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675519 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0007cdad0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0007cdad0, 0xc0006b4300, 0x1, 0x1, 0xc002022fb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006bc3000)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675541 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0007cdef0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0007cdef0, 0xc0006b4300, 0x1, 0x1, 0xc0024fb7b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008b1e900)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675509 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0007cd3b0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0007cd3b0, 0xc0006b4300, 0x1, 0x1, 0xc0002cafb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc009015b00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675548 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0052265a0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0052265a0, 0xc0006b4300, 0x1, 0x1, 0xc0024947b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008b1f000)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675514 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0007cd890)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0007cd890, 0xc0006b4300, 0x1, 0x1, 0xc00309bfb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006bc2400)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675546 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc005226540)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc005226540, 0xc0006b4300, 0x1, 0x1, 0xc0024997b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008b1ee00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675550 [sync.Cond.Wait, 34 minutes]:
sync.runtime_notifyListWait(0xc0045de5e0, 0x0)
/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc0045de5d0)
/usr/local/go/src/sync/cond.go:56 +0x99
github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:193
github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc0045de500, 0x64d9350, 0xc003a5e510)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:396 +0x10b
runtime/pprof.Do(0x64d92e0, 0xc003a5e510, 0xc0006b4200, 0x1, 0x1, 0xc00c5d37b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble/record.NewLogWriter.func2(0xc0045de500)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:314 +0x89
created by github.com/cockroachdb/pebble/record.NewLogWriter
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:313 +0x2ea
goroutine 675531 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc00b0dc460, 0xc00a14d080)
/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:217 +0x9c
created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue
/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:213 +0x185
goroutine 675727 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider.func1(0xc0009f4000)
/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/granter.go:1126 +0xda
created by github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider
/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/granter.go:1121 +0x24e
goroutine 675517 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0007cda10)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0007cda10, 0xc0006b4300, 0x1, 0x1, 0xc0020237b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006bc2d00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675544 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc005226480)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc005226480, 0xc0006b4300, 0x1, 0x1, 0xc002499fb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008b1ec00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675533 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc00b0dc5a0, 0xc00a14d140)
/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:217 +0x9c
created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue
/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:213 +0x185
goroutine 675539 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0052262a0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0052262a0, 0xc0006b4300, 0x1, 0x1, 0xc0024fcfb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008b1e600)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675508 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0007ccf60)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0007ccf60, 0xc0006b4300, 0x1, 0x1, 0xc005552fb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc009015a00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675543 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc005226420)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc005226420, 0xc0006b4300, 0x1, 0x1, 0xc002494fb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008b1eb00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 676585 [select, 34 minutes]:
database/sql.(*DB).connectionOpener(0xc00c5ec000, 0x64d92a8, 0xc0040e2140)
/usr/local/go/src/database/sql/sql.go:1133 +0xb5
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:740 +0x12a
goroutine 675535 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0052260c0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0052260c0, 0xc0006b4300, 0x1, 0x1, 0xc002495fb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008b1e200)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675515 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0007ccdb0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0007ccdb0, 0xc0006b4300, 0x1, 0x1, 0xc00309ffb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006bc2a00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675532 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc00b0dc500, 0xc00a14d0e0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:217 +0x9c
created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue
/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:213 +0x185
goroutine 675510 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0007cd7a0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0007cd7a0, 0xc0006b4300, 0x1, 0x1, 0xc00206afb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc009015c00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 655238 [select, 35 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc01094d6c0, 0x64d92a8, 0xc007e5c140)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x8e
created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/flags.go:277 +0xb66
goroutine 675537 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0052261e0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0052261e0, 0xc0006b4300, 0x1, 0x1, 0xc0024987b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008b1e400)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 682740 [semacquire, 34 minutes]:
sync.runtime_Semacquire(0xc00da241a8)
/usr/local/go/src/runtime/sema.go:56 +0x45
sync.(*WaitGroup).Wait(0xc00da241a0)
/usr/local/go/src/sync/waitgroup.go:130 +0x65
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*inflightTracker).Wait(0xc00da241a0, 0x64d92a8, 0xc00f76a280, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/sink_webhook.go:556 +0x92
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*webhookSink).Flush(0xc00d72d260, 0x64d92a8, 0xc00f76a280, 0x8efa3ca1ea, 0x8aa6700)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/sink_webhook.go:624 +0x45
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*notifyFlushSink).Flush(0xc0095b4c00, 0x64d92a8, 0xc00f76a280, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/testfeed_test.go:415 +0x7a
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*metricsSink).Flush(0xc0095b4db0, 0x64d92a8, 0xc00f76a280, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/metrics.go:78 +0x6e
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.errorWrapperSink.Flush(0x64f97b8, 0xc0095b4db0, 0x64d92a8, 0xc00f76a280, 0x611ec554, 0xb69763b)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/sink.go:218 +0x49
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changeAggregator).flushFrontier(0xc0015f9c00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:527 +0x50
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changeAggregator).noteResolvedSpan(0xc0015f9c00, 0xc0010dc140, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:515 +0x1b4
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changeAggregator).tick(0xc0015f9c00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:489 +0x20c
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changeAggregator).Next(0xc0015f9c00, 0x649f700, 0xc0015c8f78, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:445 +0x85
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changeFrontier).Next(0xc002a6ea80, 0xc000492660, 0x28, 0x649f720, 0xc0015c8f78)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:1200 +0xec
github.com/cockroachdb/cockroach/pkg/sql/execinfra.Run(0x64d92a8, 0xc00f76a1c0, 0x64f9638, 0xc002a6ea80, 0x6491890, 0xc00427d180)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/base.go:186 +0x35
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc002a6ea80, 0x64d92a8, 0xc00f76a1c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:731 +0x90
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc003e2ec60, 0x64d92a8, 0xc00f76a1c0, 0x5540c18)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:418 +0x2c4
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0012ddc20, 0xc00d72d180, 0x0, 0xc001823148, 0xc0055b1500, 0xc00427ca80, 0xc001822fe8, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:525 +0x4d7
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeeddist.StartDistChangefeed(0x64d9350, 0xc004b35c50, 0x653ee58, 0xc0077f0910, 0x984fe5242b60001, 0xc0076f9680, 0xc00cc30b00, 0x3d, 0xc0076f9650, 0x169cd09e1e543726, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeeddist/distflow.go:144 +0xac6
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.distChangefeedFlow(0x64d9350, 0xc004b35c50, 0x653ee58, 0xc0077f0910, 0x984fe5242b60001, 0xc0076f9680, 0xc00cc30b00, 0x3d, 0xc0076f9650, 0x169cd09e1e543726, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_dist.go:117 +0x387
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changefeedResumer).resumeWithRetries(0xc0077f08c0, 0x64d9350, 0xc004b35c50, 0x653ee58, 0xc0077f0910, 0x984fe5242b60001, 0xc0076f9680, 0xc00cc30b00, 0x3d, 0xc0076f9650, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_stmt.go:692 +0x215
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changefeedResumer).Resume(0xc0077f08c0, 0x64d9350, 0xc004b35c50, 0x4b8d680, 0xc0077f0910, 0xc001823810, 0xc001823830)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_stmt.go:624 +0x247
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*reportErrorResumer).Resume(0xc00dd39b78, 0x64d9350, 0xc004b35c50, 0x4b8d680, 0xc0077f0910, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/testfeed_test.go:209 +0x5b
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine.func1(0xc00c582740, 0x6490350, 0xc00dd39b78, 0x64d9350, 0xc004b35c50, 0x4b8d680, 0xc0077f0910, 0xc0032799a8)
/go/src/github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1187 +0xae
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine(0xc009775c20, 0x64d9350, 0xc004b35bc0, 0x4b8d680, 0xc0077f0910, 0x6490350, 0xc00dd39b78, 0xc00c02f900, 0xc00d704518, 0x7, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1188 +0x8d2
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).runJob(0xc009775c20, 0x64d92a8, 0xc0046ff580, 0x6490350, 0xc00dd39b78, 0xc00c02f900, 0xc00d704518, 0x7, 0xc00e0c69f0, 0x16, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:419 +0x46c
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).resumeJob.func1(0x64d92a8, 0xc001898140)
/go/src/github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:348 +0x165
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00a8a8700, 0x64d92a8, 0xc001898140, 0x0, 0x0, 0xc00baa6000)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b
goroutine 675521 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0007cddd0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0007cddd0, 0xc0006b4300, 0x1, 0x1, 0xc0030b77b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006bc3200)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675726 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc00ccac280, 0xc00483a7e0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:217 +0x9c
created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue
/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:213 +0x185
goroutine 675520 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0007cd920)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0007cd920, 0xc0006b4300, 0x1, 0x1, 0xc0030b87b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006bc3100)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675542 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0052263c0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0052263c0, 0xc0006b4300, 0x1, 0x1, 0xc002496fb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008b1ea00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675549 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc005226600)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc005226600, 0xc0006b4300, 0x1, 0x1, 0xc00249a7b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008b1f100)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675545 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc005226360)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc005226360, 0xc0006b4300, 0x1, 0x1, 0xc0011f4fb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008b1ed00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675522 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0007cdb90)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0007cdb90, 0xc0006b4300, 0x1, 0x1, 0xc0002ca7b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006bc3500)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 682840 [sync.Cond.Wait, 34 minutes]:
sync.runtime_notifyListWait(0xc0045e2fe0, 0xb3)
/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc0045e2fd0)
/usr/local/go/src/sync/cond.go:56 +0x99
github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:193
github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc0045e2f00, 0x64d9350, 0xc00aca0c90)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:396 +0x10b
runtime/pprof.Do(0x64d92e0, 0xc00aca0c90, 0xc0006b4200, 0x1, 0x1, 0xc00c5d0fb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble/record.NewLogWriter.func2(0xc0045e2f00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:314 +0x89
created by github.com/cockroachdb/pebble/record.NewLogWriter
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:313 +0x2ea
goroutine 675547 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0052264e0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0052264e0, 0xc0006b4300, 0x1, 0x1, 0xc0024977b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008b1ef00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675513 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0007cd800)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0007cd800, 0xc0006b4300, 0x1, 0x1, 0xc006f78fb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc009015f00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675534 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc005226030)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc005226030, 0xc0006b4300, 0x1, 0x1, 0xc00249afb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006bc3f00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675511 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc003a5e450)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc003a5e450, 0xc0006b4300, 0x1, 0x1, 0xc00206f7b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc009015d00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675507 [sleep]:
time.Sleep(0x4c4b40)
/usr/local/go/src/runtime/time.go:193 +0xd2
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Quiesce(0xc00a8a8700, 0x64d92e0, 0xc000072108)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:560 +0xf4
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Stop(0xc00a8a8700, 0x64d92e0, 0xc000072108)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:505 +0x1e7
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.startTestFullServer.func1()
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/helpers_test.go:377 +0x6f
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.webhookTestWithOptions.func1(0xc009f79980)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/helpers_test.go:639 +0x1d7
testing.tRunner(0xc009f79980, 0xc00783cc00)
/usr/local/go/src/testing/testing.go:1193 +0xef
created by testing.(*T).Run
/usr/local/go/src/testing/testing.go:1238 +0x2b3
goroutine 675536 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc005226120)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc005226120, 0xc0006b4300, 0x1, 0x1, 0xc002498fb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008b1e300)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675574 [select, 34 minutes]:
github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps.func1(0x5549230, 0xc0078661e0, 0xc00fdba620, 0x64d9350, 0xc013139170, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:247 +0xff
created by github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps
/go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:240 +0xd0
goroutine 675529 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc00b0dc320, 0xc00a14cfc0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:217 +0x9c
created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue
/go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:213 +0x185
goroutine 675540 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc005226300)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc005226300, 0xc0006b4300, 0x1, 0x1, 0xc0024fffb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008b1e700)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 675518 [chan receive, 34 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x64d9350, 0xc0007cda70)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76
runtime/pprof.Do(0x64d92e0, 0xc0007cda70, 0xc0006b4300, 0x1, 0x1, 0xc00201e7b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006bc2f00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5
goroutine 655352 [chan receive, 34 minutes]:
testing.(*T).Run(0xc009f79980, 0x4eabc86, 0x7, 0xc00783cc00, 0xc00783cc00)
/usr/local/go/src/testing/testing.go:1239 +0x2da
github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.TestChangefeedNemeses(0xc00716ec00)
/go/src/github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/nemeses_test.go:50 +0x2d5
testing.tRunner(0xc00716ec00, 0x553f200)
/usr/local/go/src/testing/testing.go:1193 +0xef
created by testing.(*T).Run
/usr/local/go/src/testing/testing.go:1238 +0x2b3