Skip to content

kv/kvserver: TestReplicateQueueRebalanceMultiStore failed #112658

@cockroach-teamcity

Description

@cockroach-teamcity

kv/kvserver.TestReplicateQueueRebalanceMultiStore failed with artifacts on master @ d2c5bbe1aa7930d8e7267357f051ba7d2a79aec2:

Fatal error:

panic: test timed out after 14m55s
running tests:
	TestReplicateQueueRebalanceMultiStore (14m33s)
	TestReplicateQueueRebalanceMultiStore/multi-store (14m23s)

Stack:

goroutine 16028085 [running]:
testing.(*M).startAlarm.func1()
	GOROOT/src/testing/testing.go:2241 +0x31c
created by time.goFunc
	GOROOT/src/time/sleep.go:176 +0x38
Log preceding fatal error

* sync.runtime_SemacquireMutex(0x4012cf30c0?, 0x0?, 0x4013bcf398?)
* 	GOROOT/src/runtime/sema.go:77 +0x28
* sync.(*Mutex).lockSlow(0x400f8ded58)
* 	GOROOT/src/sync/mutex.go:171 +0x174
* sync.(*Mutex).Lock(...)
* 	GOROOT/src/sync/mutex.go:90
* github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(0x400f8dec80, 0x4018e68160, {0xffff7a1df8d8, 0x40042cb300}, 0x2?)
* 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:266 +0x2fc
* github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0x40066af500, 0x4018e68160, {0xffff7a1df8d8, 0x40042cb300})
* 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:3000 +0xe0
* github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0x75bed00?, 0x4018e68160, {0xffff7a1df8d8, 0x40042cb300})
* 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:227 +0xf0
* github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0x400bf47400, 0x4018e68160, {0x7603f40?, 0x40042cb300})
* 	github.com/cockroachdb/cockroach/pkg/server/node.go:1673 +0x1fc
* github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1({0x0?, 0x75bed00?}, {0x75f7140?, 0x4012cf3080?})
* 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1032 +0x78
* github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5cc8240, 0x400bf47400}, {0x75f3558?, 0x4000fb9560?}, 0xab7aa60, 0x40075e16c0)
* 	github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:165 +0x4e8
* github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5cc8240?, 0x400bf47400?}, {0x75f3558?, 0x4000fb9560?})
* 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:847 +0x60
* github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5cc8240, 0x400bf47400}, {0x75f3558, 0x4000fb9560}, 0x400c22c000?, 0x4010aedb40)
* 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:178 +0x78
* github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5cc8240?, 0x400bf47400?}, {0x75f3558?, 0x4000fb9560?})
* 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:850 +0x8c
* github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0x400c22c000?, {{0x75bec58?}, {0x75fdf50?, 0x4016831110?}}}, {0x5cc8240, 0x400bf47400}, {0x75f3558, 0x4000fb9560}, 0xab7aa60, 0x4010aedb40)
* 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x30c
* github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5cc8240?, 0x400bf47400?}, {0x75f3558?, 0x4000fb9560?})
* 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:850 +0x8c
* github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x4008ecadb8?, 0x463330?})
* 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:145 +0x34
* github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0x400c073320, {0x75bed00, 0x4000fb9530}, {0x75bec58?, 0x4008ecaea8?}, 0x4008ecae68)
* 	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:336 +0xac
* github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5cc8240, 0x400bf47400}, {0x75f3558?, 0x4000fb9560?}, 0xab7aa60, 0x4010aedb40)
* 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:144 +0xcc
* github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5cc8240?, 0x400bf47400?}, {0x75f3558?, 0x4000fb9560?})
* 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:850 +0x8c
* github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run({0x40050989a0, 0x4, 0x4}, {0x5cc8240, 0x400bf47400}, {0x75f3558, 0x4000fb9560}, 0xab7aa60, 0x40075e16c0)
* 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:852 +0x12c
* github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2()
* 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1037 +0xfc
* created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed
* 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1025 +0x2a0
* 
* goroutine 14276796 [select]:
* github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1()
* 	github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:171 +0x9c
* created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker
* 	github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:166 +0x64
* 
*

Help

See also: How To Investigate a Go Test Failure (internal)

/cc @cockroachdb/kv

This test on roachdash | Improve this report!

Jira issue: CRDB-32533

Metadata

Metadata

Assignees

Labels

C-bugCode not up to spec/doc, specs & docs deemed correct. Solution expected to change code/behavior.C-test-failureBroken test (automatically or manually discovered).O-robotOriginated from a bot.T-kvKV Teambranch-masterFailures and bugs on the master branch.release-blockerIndicates a release-blocker. Use with branch-release-2x.x label to denote which branch is blocked.

Type

No type

Projects

Relationships

None yet

Development

No branches or pull requests

Issue actions