Skip to content

ccl/importccl: TestImportPgDump failed #50995

Closed
#51395
Closed
ccl/importccl: TestImportPgDump failed#50995
#51395
Assignees
andreimatei
Labels
C-test-failureBroken test (automatically or manually discovered).O-robotOriginated from a bot.branch-masterFailures and bugs on the master branch.
Milestone
20.2
@cockroach-teamcity

Description

@cockroach-teamcity
cockroach-teamcity
opened on Jul 6, 2020

(ccl/importccl).TestImportPgDump failed on master@f61d73fe1e4773372f3800b715143305a3342ee7:

Fatal error:

panic: overlapping descriptors with same gen but different IDs: r59:/{Table/54/2/NULL/NULL/10-Max} [(n1,s1):1, (n3,s3):2, (n2,s2):3, next=4, gen=16] vs r57:/Table/5{4/3-5} [(n1,s1):1, (n3,s3):2, (n2,s2):3, next=4, gen=16] [recovered]
	panic: overlapping descriptors with same gen but different IDs: r59:/{Table/54/2/NULL/NULL/10-Max} [(n1,s1):1, (n3,s3):2, (n2,s2):3, next=4, gen=16] vs r57:/Table/5{4/3-5} [(n1,s1):1, (n3,s3):2, (n2,s2):3, next=4, gen=16] [recovered]
	panic: overlapping descriptors with same gen but different IDs: r59:/{Table/54/2/NULL/NULL/10-Max} [(n1,s1):1, (n3,s3):2, (n2,s2):3, next=4, gen=16] vs r57:/Table/5{4/3-5} [(n1,s1):1, (n3,s3):2, (n2,s2):3, next=4, gen=16]

Stack:

goroutine 330801 [running]:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Recover(0xc006e34870, 0x5037e60, 0xc002ad0870)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:183 +0x11f
panic(0x3f894a0, 0xc0057efcc0)
	/usr/local/go/src/runtime/panic.go:969 +0x166
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send.func1(0xc005109268, 0xc0051092f0, 0xc0051092e8, 0xc006415500)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:111 +0x1e6
panic(0x3f894a0, 0xc0057efcc0)
	/usr/local/go/src/runtime/panic.go:975 +0x3e3
github.com/cockroachdb/cockroach/pkg/kv/kvbase.(*RangeCacheEntry).NewerThan(0xc0002ab8c0, 0xc005eed5c0, 0x4fb8440)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvbase/range_cache.go:122 +0x137
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*RangeDescriptorCache).clearOlderOverlapping.func1(0xc0057e1ec0, 0x1)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/range_cache.go:974 +0x63
github.com/cockroachdb/cockroach/pkg/util/cache.(*OrderedCache).DoRangeEntry.func1(0x4fb5140, 0xc0057e1ec0, 0x4fb5140)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/cache/cache.go:458 +0x40
github.com/biogo/store/llrb.(*Node).doRange(0xc0057e1ef0, 0xc005105450, 0x4fb5140, 0xc0052ef680, 0x4fb5140, 0xc0052ef6b0, 0xc00c2b2e20)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/biogo/store/llrb/llrb.go:519 +0x13f
github.com/biogo/store/llrb.(*Node).doRange(0xc007600b10, 0xc005105450, 0x4fb5140, 0xc0052ef680, 0x4fb5140, 0xc0052ef6b0, 0xc00da06bc0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/biogo/store/llrb/llrb.go:525 +0x11e
github.com/biogo/store/llrb.(*Node).doRange(0xc00850fce0, 0xc005105450, 0x4fb5140, 0xc0052ef680, 0x4fb5140, 0xc0052ef6b0, 0xc0058c0540)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/biogo/store/llrb/llrb.go:525 +0x11e
github.com/biogo/store/llrb.(*Node).doRange(0xc004af0690, 0xc005105450, 0x4fb5140, 0xc0052ef680, 0x4fb5140, 0xc0052ef6b0, 0xc0052ef6b0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/biogo/store/llrb/llrb.go:525 +0x11e
github.com/biogo/store/llrb.(*Tree).DoRange(0xc00b6b4530, 0xc005105450, 0x4fb5140, 0xc0052ef680, 0x4fb5140, 0xc0052ef6b0, 0x20)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/biogo/store/llrb/llrb.go:507 +0xae
github.com/cockroachdb/cockroach/pkg/util/cache.(*OrderedCache).DoRangeEntry(0xc00b6b44d0, 0xc0051055c8, 0x40ed1c0, 0xc0058c0520, 0x40ed1c0, 0xc0058c0540, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/cache/cache.go:457 +0xea
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*RangeDescriptorCache).clearOlderOverlapping(0xc00d878660, 0x5037e60, 0xc002ad0960, 0xc0002ab8c0, 0x0, 0x4fb5140)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/range_cache.go:972 +0x2d9
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*RangeDescriptorCache).insertLockedInner(0xc00d878660, 0x5037e60, 0xc002ad0960, 0xc0099eb270, 0x1, 0x1, 0xedfc01, 0x437fb40, 0x4473e00)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/range_cache.go:894 +0x127
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*RangeDescriptorCache).insertLocked(0xc00d878660, 0x5037e60, 0xc002ad0960, 0xc005105ab8, 0x1, 0x1, 0x0, 0xedfc01, 0x437fb40)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/range_cache.go:866 +0x1ca
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.EvictionToken.EvictAndReplace(0xc00d878660, 0xc005eed500, 0xc0057dc3a8, 0x5037e60, 0xc002ad0960, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/range_cache.go:427 +0x219
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.EvictionToken.Evict(...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/range_cache.go:411
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc007ed2840, 0x5037e60, 0xc002ad0960, 0x161f1355ddf9f792, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1471 +0x4de
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc007ed2840, 0x5037e60, 0xc002ad0960, 0x161f1355ddf9f792, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1098 +0x18ef
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc007ed2840, 0x5037e60, 0xc002ad0960, 0x161f1355ddf9f792, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:738 +0x8e4
github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc006807fe0, 0x5037e60, 0xc002ad0960, 0x161f1355ddf9f792, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:225 +0x95
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc006807f80, 0x5037e60, 0xc002ad0960, 0x161f1355ddf9f792, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:748 +0x122
github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:730
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5037e60, 0xc002ad0960, 0xc004783448, 0xc0029f4680, 0x0, 0xc004783958)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:658 +0xf7
github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc006807f80, 0x5037e60, 0xc002ad0960, 0xc0029f4680, 0xc0047836b8, 0x1)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:681 +0x9c
github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).MaybePushTransactions(0xc003f708c0, 0x5037e60, 0xc002ad0960, 0xc004783958, 0x161f1355d8facdad, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:369 +0x6e2
github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).PushTransaction(0xc003f708c0, 0x5037e60, 0xc002ad0960, 0xc00617fbc0, 0x161f1355d8facdad, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:279 +0x151
github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc007c9bb80, 0x5037e60, 0xc002ad0960, 0xc003d927e0, 0x161f1355d8e7605f, 0x0, 0x0, 0x0, 0xc0064dfa40, 0x1, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:435 +0x315
github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc007c9bb80, 0x5037e60, 0xc002ad0960, 0xc003d927e0, 0x161f1355d8e7605f, 0x0, 0x0, 0x0, 0xc0064dfa40, 0x1, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:350 +0x2db
github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00b687200, 0x5037e60, 0xc002ad0960, 0xc0081ffb90, 0xc003d927e0, 0x161f1355d8e7605f, 0x0, 0x0, 0x0, 0xc0064dfa40, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:173 +0x3f3
github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0xc00b687200, 0x5037e60, 0xc002ad0960, 0x0, 0xc003d927e0, 0x161f1355d8e7605f, 0x0, 0x0, 0x0, 0xc0064dfa40, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:124 +0xe2
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc002003800, 0x5037e60, 0xc002ad0960, 0xc0008e6000, 0x4773510, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:224 +0x31e
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sendWithRangeID(0xc002003800, 0x5037e60, 0xc002ad0930, 0x1, 0xc0008e6000, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:94 +0x64b
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc002003800, 0x5037e60, 0xc002ad0930, 0x161f1355d8e7605f, 0x0, 0x100000001, 0x1, 0x0, 0x1, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:36 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc006415500, 0x5037e60, 0xc002ad08d0, 0x161f1355d8e7605f, 0x0, 0x100000001, 0x1, 0x0, 0x1, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:202 +0x647
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc002d12000, 0x5037e60, 0xc002ad08d0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x1, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:177 +0xed
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5037e60, 0xc002ad0870, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:846 +0x1f0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc006e34870, 0x5037e60, 0xc002ad0870, 0x460a1df, 0x10, 0xc0047856d0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:302 +0xf1
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc003c7ca00, 0x5037e60, 0xc002ad0870, 0xc00aa3ff00, 0xc002ad0870, 0xc00004e000, 0x410bfe0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:834 +0x194
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc003c7ca00, 0x5037e60, 0xc002ad0810, 0xc00aa3ff00, 0xc003c7ca00, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:872 +0x9f
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5037e60, 0xc002ad0810, 0x45bb560, 0xc00aa3ff00, 0x45bb560, 0xc00aa3ff00, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:10265 +0x86
github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x5037e60, 0xc002ad0810, 0x45bb560, 0xc00aa3ff00, 0xc00099d2e0, 0xc00099d300, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:44 +0x9e6
github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func2(0x5037e60, 0xc002ad0810, 0x45bb560, 0xc00aa3ff00, 0xc00099d2e0, 0xc00099d300, 0x718028, 0x20, 0x4177b60, 0xc000ea0101)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:225 +0xb6
github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func4(0x5037e60, 0xc002ad0810, 0x45bb560, 0xc00aa3ff00, 0xc00099d2e0, 0xc00099d300, 0xc004570ba0, 0x8172c8, 0x43a6040, 0xc002ad0810)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:255 +0xac
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x45681a0, 0xc003c7ca00, 0x5037e60, 0xc002ad0810, 0xc000ea01e0, 0xc0077df650, 0x5037e60, 0xc002ad0810, 0xc003dc0000, 0xda)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:10267 +0x14b
google.golang.org/grpc.(*Server).processUnaryRPC(0xc001d616c0, 0x5098880, 0xc008898180, 0xc006007c00, 0xc009830180, 0x7489a90, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x50a
google.golang.org/grpc.(*Server).handleStream(0xc001d616c0, 0x5098880, 0xc008898180, 0xc006007c00, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xccb
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0047f8d50, 0xc001d616c0, 0x5098880, 0xc008898180, 0xc006007c00)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa1
Log preceding fatal error

=== RUN   TestImportPgDump/simple_from_multi
I200706 05:40:52.571476 324629 kv/kvserver/store_remove_replica.go:122  [n2,s2,r56/3:/Table/54/2{-/NULL/N…}] removing replica r59/3
I200706 05:40:52.571774 323555 kv/kvserver/store_remove_replica.go:122  [n1,s1,r56/1:/Table/54/2{-/NULL/N…}] removing replica r59/1
I200706 05:40:52.572601 324957 kv/kvserver/store_remove_replica.go:122  [n3,s3,r56/2:/Table/54/2{-/NULL/N…}] removing replica r59/2
I200706 05:40:52.573440 330959 kv/kvserver/replica_command.go:671  [n2,merge,s2,r60/3:/Table/54{-/2}] initiating a merge of r56:/Table/54/{2-3} [(n1,s1):1, (n3,s3):2, (n2,s2):3, next=4, gen=16] into this range (lhs+rhs has (size=19 KiB+19 KiB=37 KiB qps=0.00+0.00=0.00qps) below threshold (size=128 MiB, qps=1250.00))
I200706 05:40:52.657894 324597 kv/kvserver/store_remove_replica.go:122  [n2,s2,r60/3:/Table/54{-/2}] removing replica r56/3
I200706 05:40:52.658030 323562 kv/kvserver/store_remove_replica.go:122  [n1,s1,r60/1:/Table/54{-/2}] removing replica r56/1
I200706 05:40:52.658257 324950 kv/kvserver/store_remove_replica.go:122  [n3,s3,r60/2:/Table/54{-/2}] removing replica r56/2

More

Parameters:

  • TAGS=
  • GOFLAGS=-parallel=4
make stressrace TESTS=TestImportPgDump PKG=./pkg/ccl/importccl TESTTIMEOUT=5m STRESSFLAGS='-timeout 5m' 2>&1

See this test on roachdash
powered by pkg/cmd/internal/issues

Metadata

Metadata

Assignees

Labels

C-test-failureBroken test (automatically or manually discovered).O-robotOriginated from a bot.branch-masterFailures and bugs on the master branch.

Type

No type

Projects

No projects

Milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions