Skip to content

Bug Report: vtctldclient Reshard panic when target shard has no PRIMARY tablet #16284

@mattlord

Description

@mattlord

Overview of the Issue

$ vtctldclient Reshard --target-keyspace customer --workflow cust2cust create --source-shards '0' --target-shards '-80,80-'
E0627 14:39:40.150176   31887 main.go:56] rpc error: code = Unknown desc = uncaught panic: runtime error: invalid memory address or nil pointer dereference from: goroutine 2125 [running]:
runtime/debug.Stack()
	runtime/debug/stack.go:24 +0x64
vitess.io/vitess/go/vt/vtctl/grpcvtctldserver.panicHandler(0x14000f458d8)
	vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/server.go:119 +0x3c
panic({0x104e81140?, 0x106901fb0?})
	runtime/panic.go:770 +0x124
vitess.io/vitess/go/vt/topo.(*Server).GetTablet(0x0?, {0x10523b1a0, 0x14001783710}, 0x0)
	vitess.io/vitess/go/vt/topo/tablet.go:177 +0x3c
vitess.io/vitess/go/vt/vtctl/workflow.(*Server).buildResharder(0x140008440f0, {0x10523b1a0, 0x14001783710}, {0x140017845b0, 0x8}, {0x140017845a0, 0x9}, {0x1400178e490, 0x1, 0x140005ad42a?}, ...)
	vitess.io/vitess/go/vt/vtctl/workflow/resharder.go:106 +0x4bc
vitess.io/vitess/go/vt/vtctl/workflow.(*Server).ReshardCreate(0x140008440f0, {0x10523b1a0?, 0x14001783710?}, 0x140013f7450)
	vitess.io/vitess/go/vt/vtctl/workflow/server.go:1732 +0x4c8
vitess.io/vitess/go/vt/vtctl/grpcvtctldserver.(*VtctldServer).ReshardCreate(0x14000b0c000, {0x10523b1a0?, 0x14001783710?}, 0x140013f7450)
	vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/server.go:3289 +0x298
vitess.io/vitess/go/vt/proto/vtctlservice._Vtctld_ReshardCreate_Handler({0x105202160, 0x14000b0c000}, {0x10523b1a0, 0x14001783710}, 0x14001207900, 0x0)
	vitess.io/vitess/go/vt/proto/vtctlservice/vtctlservice_grpc.pb.go:3772 +0x1c0
google.golang.org/grpc.(*Server).processUnaryRPC(0x1400090c400, {0x10523b1a0, 0x14001783650}, {0x10524c060, 0x140010ea000}, 0x140016fd320, 0x14000844180, 0x10695ee68, 0x0)
	google.golang.org/grpc@v1.64.0/server.go:1379 +0xb58
google.golang.org/grpc.(*Server).handleStream(0x1400090c400, {0x10524c060, 0x140010ea000}, 0x140016fd320)
	google.golang.org/grpc@v1.64.0/server.go:1790 +0xb20
google.golang.org/grpc.(*Server).serveStreams.func2.1()
	google.golang.org/grpc@v1.64.0/server.go:1029 +0x8c
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 2161
	google.golang.org/grpc@v1.64.0/server.go:1040 +0x13c

Reproduction Steps

git checkout main && make build

pushd examples/local

./101_initial_cluster.sh; mysql < ../common/insert_commerce_data.sql; ./201_customer_tablets.sh; ./202_move_tables.sh; ./203_switch_reads.sh; ./204_switch_writes.sh; ./205_clean_commerce.sh;  ./301_customer_sharded.sh

# So that vtorc does not elect a primary for the new shards
../common/scripts/vtorc-down.sh

## Add our new shards
source ../common/env.sh

for i in 300 301 302; do
        CELL=zone1 TABLET_UID=$i ../common/scripts/mysqlctl-up.sh
        SHARD=-80 CELL=zone1 KEYSPACE=customer TABLET_UID=$i ../common/scripts/vttablet-up.sh
done

for i in 400 401 402; do
        CELL=zone1 TABLET_UID=$i ../common/scripts/mysqlctl-up.sh
        SHARD=80- CELL=zone1 KEYSPACE=customer TABLET_UID=$i ../common/scripts/vttablet-up.sh
done

vtctldclient Reshard --workflow cust2cust --target-keyspace customer create --source-shards '0' --target-shards '-80,80-'

./401_teardown.sh

popd

Binary Version

❯ vtgate --version
vtgate version Version: 21.0.0-SNAPSHOT (Git revision 96f9c3d35a1b18e38afdddd067461b0ae8022fb3 branch 'main') built on Thu Jun 27 14:31:27 EDT 2024 by matt@pslord.local using go1.22.4 darwin/arm64

Operating System and Environment details

N/A

Log Fragments

No response

Metadata

Metadata

Assignees

Type

No type

Projects

Status

Done

Milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions