roachpb: v21.2.6: IsCompleteTransaction panicked from supposedly unreachable line

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/3146653297/?referrer=webhooks_plugin

Panic message:

>conn_executor.go:666: panic: ×
--
*errutil.leafError: panic: × (1)
conn_executor.go:666: *withstack.withStack (top exception)
*safedetails.withSafeDetails: while executing: COMMIT TRANSACTION (2)
conn_executor.go:666: *withstack.withStack (3)
(check the extra data payloads)


<details>
<summary>Stacktrace (expand for inline code snippets):</summary>

https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/sql/conn_executor.go#L665-L667 in pkg/sql.(*Server).ServeConn.func1
/usr/local/go/src/runtime/panic.go#L964-L966 in runtime.gopanic
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/util/stop/stopper.go#L230-L232 in pkg/util/stop.(*Stopper).Recover
/usr/local/go/src/runtime/panic.go#L964-L966 in runtime.gopanic
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvserver/store_send.go#L94-L96 in pkg/kv/kvserver.(*Store).Send.func1
/usr/local/go/src/runtime/panic.go#L970-L972 in runtime.gopanic
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/roachpb/batch.go#L318-L320 in pkg/roachpb.(*BatchRequest).IsCompleteTransaction
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvserver/replica_write.go#L791-L793 in pkg/kv/kvserver.isOnePhaseCommit
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvserver/replica_write.go#L389-L391 in pkg/kv/kvserver.(*Replica).canAttempt1PCEvaluation
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvserver/replica_write.go#L444-L446 in pkg/kv/kvserver.(*Replica).evaluateWriteBatch
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvserver/replica_proposal.go#L782-L784 in pkg/kv/kvserver.(*Replica).evaluateProposal
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvserver/replica_proposal.go#L912-L914 in pkg/kv/kvserver.(*Replica).requestToProposal
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvserver/replica_raft.go#L109-L111 in pkg/kv/kvserver.(*Replica).evalAndPropose
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvserver/replica_write.go#L155-L157 in pkg/kv/kvserver.(*Replica).executeWriteBatch
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvserver/replica_send.go#L376-L378 in pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvserver/replica_send.go#L106-L108 in pkg/kv/kvserver.(*Replica).sendWithRangeID
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvserver/replica_send.go#L42-L44 in pkg/kv/kvserver.(*Replica).Send
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvserver/store_send.go#L189-L191 in pkg/kv/kvserver.(*Store).Send
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvserver/stores.go#L190-L192 in pkg/kv/kvserver.(*Stores).Send
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/server/node.go#L911-L913 in pkg/server.(*Node).batchInternal.func1
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/util/stop/stopper.go#L323-L325 in pkg/util/stop.(*Stopper).RunTaskWithErr
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/server/node.go#L899-L901 in pkg/server.(*Node).batchInternal
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/server/node.go#L1014-L1016 in pkg/server.(*Node).Batch
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/rpc/context.go#L503-L505 in pkg/rpc.internalClientAdapter.Batch
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvclient/kvcoord/transport.go#L202-L204 in pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvclient/kvcoord/transport.go#L184-L186 in pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvclient/kvcoord/dist_sender.go#L1989-L1991 in pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvclient/kvcoord/dist_sender.go#L1537-L1539 in pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvclient/kvcoord/dist_sender.go#L1155-L1157 in pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvclient/kvcoord/dist_sender.go#L966-L968 in pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvclient/kvcoord/dist_sender.go#L787-L789 in pkg/kv/kvclient/kvcoord.(*DistSender).Send
https://github.com/cockroachdb/cockroach/blob/c7cfac57476f06e3154b2fc38e77982a8bbc64c2/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go#L81-L83 in pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked
</details>

```
pkg/sql/conn_executor.go in pkg/sql.(*Server).ServeConn.func1 at line 666
/usr/local/go/src/runtime/panic.go in runtime.gopanic at line 965
pkg/util/stop/stopper.go in pkg/util/stop.(*Stopper).Recover at line 231
/usr/local/go/src/runtime/panic.go in runtime.gopanic at line 965
pkg/kv/kvserver/store_send.go in pkg/kv/kvserver.(*Store).Send.func1 at line 95
/usr/local/go/src/runtime/panic.go in runtime.gopanic at line 971
pkg/roachpb/batch.go in pkg/roachpb.(*BatchRequest).IsCompleteTransaction at line 319
pkg/kv/kvserver/replica_write.go in pkg/kv/kvserver.isOnePhaseCommit at line 792
pkg/kv/kvserver/replica_write.go in pkg/kv/kvserver.(*Replica).canAttempt1PCEvaluation at line 390
pkg/kv/kvserver/replica_write.go in pkg/kv/kvserver.(*Replica).evaluateWriteBatch at line 445
pkg/kv/kvserver/replica_proposal.go in pkg/kv/kvserver.(*Replica).evaluateProposal at line 783
pkg/kv/kvserver/replica_proposal.go in pkg/kv/kvserver.(*Replica).requestToProposal at line 913
pkg/kv/kvserver/replica_raft.go in pkg/kv/kvserver.(*Replica).evalAndPropose at line 110
pkg/kv/kvserver/replica_write.go in pkg/kv/kvserver.(*Replica).executeWriteBatch at line 156
pkg/kv/kvserver/replica_send.go in pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries at line 377
pkg/kv/kvserver/replica_send.go in pkg/kv/kvserver.(*Replica).sendWithRangeID at line 107
pkg/kv/kvserver/replica_send.go in pkg/kv/kvserver.(*Replica).Send at line 43
pkg/kv/kvserver/store_send.go in pkg/kv/kvserver.(*Store).Send at line 190
pkg/kv/kvserver/stores.go in pkg/kv/kvserver.(*Stores).Send at line 191
pkg/server/node.go in pkg/server.(*Node).batchInternal.func1 at line 912
pkg/util/stop/stopper.go in pkg/util/stop.(*Stopper).RunTaskWithErr at line 324
pkg/server/node.go in pkg/server.(*Node).batchInternal at line 900
pkg/server/node.go in pkg/server.(*Node).Batch at line 1015
pkg/rpc/context.go in pkg/rpc.internalClientAdapter.Batch at line 504
pkg/kv/kvclient/kvcoord/transport.go in pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch at line 203
pkg/kv/kvclient/kvcoord/transport.go in pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext at line 185
pkg/kv/kvclient/kvcoord/dist_sender.go in pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas at line 1990
pkg/kv/kvclient/kvcoord/dist_sender.go in pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch at line 1538
pkg/kv/kvclient/kvcoord/dist_sender.go in pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges at line 1156
pkg/kv/kvclient/kvcoord/dist_sender.go in pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit at line 967
pkg/kv/kvclient/kvcoord/dist_sender.go in pkg/kv/kvclient/kvcoord.(*DistSender).Send at line 788
pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go in pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked at line 82
```

| Tag | Value |
|---|---|
| Cockroach Release | `v21.2.6` |
| Cockroach SHA: | c7cfac57476f06e3154b2fc38e77982a8bbc64c2 |
| Platform | linux amd64 |
| Distribution | CCL |
| Environment | v21.2.6 |
| Command | server |
| Go Version | ``|
| # of CPUs ||
| # of Goroutines ||


Jira issue: CRDB-14627

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

roachpb: v21.2.6: IsCompleteTransaction panicked from supposedly unreachable line #79188

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

	// indications that the batch's txn will require retry, execute as normal.
	if r.canAttempt1PCEvaluation(ctx, ba, latchSpans) {
	res := r.evaluate1PC(ctx, idKey, ba, latchSpans, lockSpans)

	// in the call stack as well.
	batch, ms, br, res, pErr := r.evaluateWriteBatch(ctx, idKey, ba, lul, latchSpans, lockSpans)

	) (ProposalData, roachpb.Error) {
	res, needConsensus, pErr := r.evaluateProposal(ctx, idKey, ba, lul, latchSpans, lockSpans)

	idKey := makeIDKey()
	proposal, pErr := r.requestToProposal(ctx, idKey, ba, st, lul, g.LatchSpans(), g.LockSpans())
	log.Event(proposal.ctx, "evaluated request")

	// evalAndPropose.
	ch, abandon, _, pErr := r.evalAndPropose(ctx, ba, g, st, localUncertaintyLimit, tok.Move(ctx))
	if pErr != nil {

	fn := (*Replica).executeWriteBatch
	br, pErr = r.executeBatchWithConcurrencyRetries(ctx, ba, fn)
	} else if ba.IsAdmin() {

	) (roachpb.BatchResponse, roachpb.Error) {
	return r.sendWithRangeID(ctx, r.RangeID, &ba)
	}


	br, pErr := store.Send(ctx, ba)
	if br != nil && br.Error != nil {

	var pErr *roachpb.Error
	br, pErr = n.stores.Send(ctx, *args)
	if pErr != nil {

	var br *roachpb.BatchResponse
	if err := n.stopper.RunTaskWithErr(ctx, "node.Node: batch", func(ctx context.Context) error {
	var finishSpan func(*roachpb.BatchResponse)

	r := recover()
	h.ex.closeWrapper(ctx, r)
	}()

	}
	panic(r)
	}

	// again due to undefined state.
	panic(r)
	}

	}
	panic("unreachable")
	}

	}
	br, err := n.batchInternal(ctx, tenantID, args)
	if callAdmittedWorkDoneOnKVAdmissionQ {

	ba.AdmissionHeader.SourceLocation = roachpb.AdmissionHeader_LOCAL
	return a.server.Batch(ctx, ba)
	}

	}
	reply, err := iface.Batch(ctx, &ba)
	// If we queried a remote node, perform extra validation and

	ba.Replica = r
	return gt.sendBatch(ctx, r.NodeID, iface, ba)
	}

	}
	br, err = transport.SendNext(ctx, ba)
	ds.maybeIncrementErrCounters(br, err)

	prevTok = routingTok
	reply, err = ds.sendToReplicas(ctx, ba, routingTok, withCommit)

	if !ri.NeedAnother(rs) {
	resp := ds.sendPartialBatch(
	ctx, ba, rs, ri.Token(), withCommit, batchIdx, false, /* needsTruncate */

	}
	br, pErr = ds.divideAndSendBatchToRanges(ctx, ba, rs, true /* withCommit */, batchIdx)

	if withParallelCommit {
	rpl, pErr = ds.divideAndSendParallelCommit(ctx, ba, rs, 0 /* batchIdx */)
	} else {

Tag	Value
Cockroach Release	`v21.2.6`
Cockroach SHA:	`c7cfac5`
Platform	linux amd64
Distribution	CCL
Environment	v21.2.6
Command	server
Go Version	``
# of CPUs
# of Goroutines

roachpb: v21.2.6: IsCompleteTransaction panicked from supposedly unreachable line #79188

Description

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions