Skip to content

cli: Example_demo_locality fails with "pebble: closed" panic #58959

@RaduBerinde

Description

@RaduBerinde

https://teamcity.cockroachdb.com//viewLog.html?buildId=2580313&buildTypeId=Cockroach_MergeToMaster

* ERROR: [n5] a panic has occurred!
* pebble: closed
* (1) attached stack trace
*   -- stack trace:
*   | runtime.gopanic
*   | 	/usr/local/go/src/runtime/panic.go:969
*   | github.com/cockroachdb/pebble.(*DB).Apply
*   | 	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/db.go:545
*   | github.com/cockroachdb/pebble.(*Batch).Commit
*   | 	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/batch.go:732
*   | github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:519
*   | github.com/cockroachdb/cockroach/pkg/storage.WriteSyncNoop
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine.go:845
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).updateLiveness
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:1213
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).heartbeatInternal
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:889
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).Start.func1.1
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:704
*   | github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:140
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).Start.func1
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:690
*   | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:222
* Wraps: (2) attached stack trace
*   -- stack trace:
*   | github.com/cockroachdb/pebble.(*DB).Close
*   | 	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/db.go:885
*   | github.com/cockroachdb/cockroach/pkg/storage.(*Pebble).Close
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/storage/pebble.go:634
*   | github.com/cockroachdb/cockroach/pkg/server.(*stickyInMemEnginesRegistryImpl).CloseAllStickyInMemEngines
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/server/sticky_engine.go:121
*   | github.com/cockroachdb/cockroach/pkg/cli.(*transientCluster).start.func3
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/cli/demo_cluster.go:216
*   | github.com/cockroachdb/cockroach/pkg/util/stop.CloserFn.Close
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:114
*   | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Stop
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:522
*   | github.com/cockroachdb/cockroach/pkg/cli.(*transientCluster).cleanup
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/cli/demo_cluster.go:368
*   | github.com/cockroachdb/cockroach/pkg/cli.runDemo
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/cli/demo.go:365
*   | github.com/cockroachdb/cockroach/pkg/cli.init.4.func1
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/cli/demo.go:57
*   | github.com/cockroachdb/cockroach/pkg/cli.MaybeDecorateGRPCError.func1
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/cli/error.go:194
*   | github.com/spf13/cobra.(*Command).execute
*   | 	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra/command.go:826
*   | github.com/spf13/cobra.(*Command).ExecuteC
*   | 	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra/command.go:914
*   | github.com/spf13/cobra.(*Command).Execute
*   | 	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra/command.go:864
*   | github.com/cockroachdb/cockroach/pkg/cli.Run
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/cli/cli.go:277
*   | github.com/cockroachdb/cockroach/pkg/cli.cliTest.RunWithArgs.func1
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/cli/cli_test.go:359
*   | github.com/cockroachdb/cockroach/pkg/cli.cliTest.RunWithArgs
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/cli/cli_test.go:360
*   | github.com/cockroachdb/cockroach/pkg/cli.Example_demo_locality
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/cli/demo_locality_test.go:43
*   | testing.runExample
*   | 	/usr/local/go/src/testing/run_example.go:62
*   | testing.runExamples
*   | 	/usr/local/go/src/testing/example.go:44
*   | testing.(*M).Run
*   | 	/usr/local/go/src/testing/testing.go:1346
*   | github.com/cockroachdb/cockroach/pkg/cli_test.TestMain
*   | 	/go/src/github.com/cockroachdb/cockroach/pkg/cli/main_test.go:34
*   | main.main
*   | 	_testmain.go:221
*   | [...repeated from below...]
* Wraps: (3) attached stack trace
*   -- stack trace:
*   | github.com/cockroachdb/pebble.init
*   | 	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/db.go:42
*   | runtime.doInit
*   | 	/usr/local/go/src/runtime/proc.go:5652
*   | runtime.doInit
*   | 	/usr/local/go/src/runtime/proc.go:5647
*   | runtime.doInit
*   | 	/usr/local/go/src/runtime/proc.go:5647
*   | runtime.doInit
*   | 	/usr/local/go/src/runtime/proc.go:5647
*   | runtime.doInit
*   | 	/usr/local/go/src/runtime/proc.go:5647
*   | runtime.main
*   | 	/usr/local/go/src/runtime/proc.go:191
*   | runtime.goexit
*   | 	/usr/local/go/src/runtime/asm_amd64.s:1374
* Wraps: (4) pebble: closed
* Error types: (1) *withstack.withStack (2) *withstack.withStack (3) *withstack.withStack (4) *errutil.leafError
*
panic: pebble: closed [recovered]
	panic: pebble: closed

goroutine 917444 [running]:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Recover(0xc0047b6480, 0x5149fc0, 0xc004ee2f30)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:207 +0x126
panic(0x41095e0, 0xc00fd6a420)
	/usr/local/go/src/runtime/panic.go:969 +0x1b9
github.com/cockroachdb/pebble.(*DB).Apply(0xc000be1400, 0xc011aa83c0, 0x70b9bf7, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/db.go:545 +0x413
github.com/cockroachdb/pebble.(*Batch).Commit(...)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/batch.go:732
github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc0093dc800, 0x1, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:519 +0x5e
github.com/cockroachdb/cockroach/pkg/storage.WriteSyncNoop(0x5149f80, 0xc00afae2a0, 0x529eee0, 0xc00aef2930, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine.go:845 +0xc6
github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).updateLiveness(0xc00a383680, 0x5149f80, 0xc00afae2a0, 0x5, 0x1, 0x1659d79a6f28d4b4, 0x0, 0x0, 0x0, 0x5, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:1213 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).heartbeatInternal(0xc00a383680, 0x5149f80, 0xc00afae2a0, 0x5, 0x1, 0x1659d79962e2174e, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:889 +0x708
github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).Start.func1.1(0x5149f80, 0xc00afae2a0, 0x10c388d00, 0x5149f80)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:704 +0x251
github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout(0x5149f80, 0xc00afae2a0, 0x44b2a8b, 0x17, 0x10c388d00, 0xc009727e28, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:140 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).Start.func1(0x5149fc0, 0xc004ee2f30)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:690 +0x33a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc005452900, 0xc0047b6480, 0xc0099420e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:222 +0xe4
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0xa8

goroutine 1 [sync.Cond.Wait]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:312
sync.runtime_notifyListWait(0xc001fc4e10, 0xc00000000d)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc001fc4e00)
	/usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Quiesce(0xc0049ec6c0, 0x5149f40, 0xc000072120)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:577 +0x1ee
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Stop(0xc0049ec6c0, 0x5149f40, 0xc000072120)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:510 +0x270
github.com/cockroachdb/cockroach/pkg/server.(*Server).Stop(...)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2225
github.com/cockroachdb/cockroach/pkg/util/stop.CloserFn.Close(0xc004eac5b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:114 +0x25
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Stop(0xc0050b8510, 0x5149f40, 0xc000072120)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:522 +0x3cf
github.com/cockroachdb/cockroach/pkg/cli.(*transientCluster).cleanup(0xc0050b8360, 0x5149f40, 0xc000072120)
	/go/src/github.com/cockroachdb/cockroach/pkg/cli/demo_cluster.go:368 +0x87
github.com/cockroachdb/cockroach/pkg/cli.runDemo(0x7093760, 0x50fe580, 0xc0002cfe60, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/cli/demo.go:365 +0x980
github.com/cockroachdb/cockroach/pkg/cli.init.4.func1(0x7093760, 0xc00511c440, 0x0, 0x4, 0xc009320830, 0xd)
	/go/src/github.com/cockroachdb/cockroach/pkg/cli/demo.go:57 +0x33
github.com/cockroachdb/cockroach/pkg/cli.MaybeDecorateGRPCError.func1(0x7093760, 0xc00511c440, 0x0, 0x4, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/cli/error.go:194 +0x79
github.com/spf13/cobra.(*Command).execute(0x7093760, 0xc00511c400, 0x4, 0x4, 0x7093760, 0xc00511c400)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra/command.go:826 +0x47c
github.com/spf13/cobra.(*Command).ExecuteC(0x70920e0, 0xc000b06758, 0xc0062899a0, 0x1)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra/command.go:914 +0x30b
github.com/spf13/cobra.(*Command).Execute(...)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra/command.go:864
github.com/cockroachdb/cockroach/pkg/cli.Run(...)
	/go/src/github.com/cockroachdb/cockroach/pkg/cli/cli.go:277
github.com/cockroachdb/cockroach/pkg/cli.cliTest.RunWithArgs.func1(0xc00a9b4a50, 0x5, 0x5, 0x0, 0xc00664f220, 0x44, 0x4b3db70, 0xc000010758, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/cli/cli_test.go:359 +0x2ab
github.com/cockroachdb/cockroach/pkg/cli.cliTest.RunWithArgs(0x0, 0xc00664f220, 0x44, 0x4b3db70, 0xc000010758, 0x0, 0x0, 0x0, 0xc00a9b4a50, 0x5, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/cli/cli_test.go:360 +0x85
github.com/cockroachdb/cockroach/pkg/cli.Example_demo_locality()
	/go/src/github.com/cockroachdb/cockroach/pkg/cli/demo_locality_test.go:43 +0x410
testing.runExample(0x44a8aac, 0x15, 0x4b3d5f8, 0x4962a59, 0x2dd, 0x0, 0x0)
	/usr/local/go/src/testing/run_example.go:62 +0x209
testing.runExamples(0xc00d089e48, 0x70a4660, 0x17, 0x17, 0xbff7e76791a536f1)
	/usr/local/go/src/testing/example.go:44 +0x1af
testing.(*M).Run(0xc000938a80, 0x0)
	/usr/local/go/src/testing/testing.go:1346 +0x273
github.com/cockroachdb/cockroach/pkg/cli_test.TestMain(0xc000938a80)
	/go/src/github.com/cockroachdb/cockroach/pkg/cli/main_test.go:34 +0x8e
main.main()
	_testmain.go:221 +0x165

Metadata

Metadata

Assignees

Labels

C-test-failureBroken test (automatically or manually discovered).branch-masterFailures and bugs on the master branch.

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions