Skip to content

kvserver: v20.1.0-beta.3: received ... results, limit was ... #46652

Closed
#48160
Closed
kvserver: v20.1.0-beta.3: received ... results, limit was ...#46652
#48160
Assignees
itsbilal
Labels
C-bugCode not up to spec/doc, specs & docs deemed correct. Solution expected to change code/behavior.O-sentryOriginated from an in-the-wild panic report.
@cockroach-teamcity

Description

@cockroach-teamcity
cockroach-teamcity
opened on Mar 26, 2020

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/1583480205/?referrer=webhooks_plugin

Panic message:

replica_read.go:95: received %d results, limit was %d | int64; int64

Stacktrace (expand for inline code snippets):

cockroach/pkg/kv/kvserver/replica_read.go

Lines 94 to 96 in fcd74cd

}
br, res, pErr = evaluateBatch(ctx, storagebase.CmdIDKey(""), rw, rec, nil, ba, true /* readOnly */)
in pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes

cockroach/pkg/kv/kvserver/replica_read.go

Lines 66 to 68 in fcd74cd

var result result.Result
br, result, pErr = r.executeReadOnlyBatchWithServersideRefreshes(ctx, rw, rec, ba, spans)
if err := r.handleReadOnlyLocalEvalResult(ctx, ba, result.Local); err != nil {
in pkg/kv/kvserver.(*Replica).executeReadOnlyBatch

cockroach/pkg/kv/kvserver/replica_send.go

Lines 235 to 237 in fcd74cd

br, g, pErr = fn(r, ctx, ba, status, g)
switch t := pErr.GetDetail().(type) {
in pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries

cockroach/pkg/kv/kvserver/replica_send.go

Lines 97 to 99 in fcd74cd

fn := (*Replica).executeReadOnlyBatch
br, pErr = r.executeBatchWithConcurrencyRetries(ctx, ba, fn)
} else if ba.IsAdmin() {
in pkg/kv/kvserver.(*Replica).sendWithRangeID

cockroach/pkg/kv/kvserver/replica_send.go

Lines 35 to 37 in fcd74cd

) (*roachpb.BatchResponse, *roachpb.Error) {
return r.sendWithRangeID(ctx, r.RangeID, &ba)
}
in pkg/kv/kvserver.(*Replica).Send

cockroach/pkg/kv/kvserver/store_send.go

Lines 203 to 205 in fcd74cd

br, pErr = repl.Send(ctx, ba)
if pErr == nil {
in pkg/kv/kvserver.(*Store).Send

cockroach/pkg/kv/kvserver/stores.go

Lines 187 to 189 in fcd74cd

br, pErr := store.Send(ctx, ba)
if br != nil && br.Error != nil {
in pkg/kv/kvserver.(*Stores).Send

cockroach/pkg/server/node.go

Lines 924 to 926 in fcd74cd

var pErr *roachpb.Error
br, pErr = n.stores.Send(ctx, *args)
if pErr != nil {
in pkg/server.(*Node).batchInternal.func1

cockroach/pkg/util/stop/stopper.go

Lines 301 to 303 in fcd74cd

return f(ctx)
}
in pkg/util/stop.(*Stopper).RunTaskWithErr

cockroach/pkg/server/node.go

Lines 912 to 914 in fcd74cd

var br *roachpb.BatchResponse
if err := n.stopper.RunTaskWithErr(ctx, "node.Node: batch", func(ctx context.Context) error {
var finishSpan func(*roachpb.BatchResponse)
in pkg/server.(*Node).batchInternal

cockroach/pkg/server/node.go

Lines 950 to 952 in fcd74cd

br, err := n.batchInternal(ctx, args)
in pkg/server.(*Node).Batch

cockroach/pkg/rpc/context.go

Lines 536 to 538 in fcd74cd

) (*roachpb.BatchResponse, error) {
return a.InternalServer.Batch(ctx, ba)
}
in pkg/rpc.internalClientAdapter.Batch

cockroach/pkg/kv/kvclient/kvcoord/transport.go

Lines 198 to 200 in fcd74cd

}
reply, err := iface.Batch(ctx, &ba)
// If we queried a remote node, perform extra validation and
in pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch

cockroach/pkg/kv/kvclient/kvcoord/transport.go

Lines 167 to 169 in fcd74cd

ba.Replica = client.replica
reply, err := gt.sendBatch(ctx, client.replica.NodeID, iface, ba)
in pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext

cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go

Lines 1629 to 1631 in fcd74cd

}
br, err := transport.SendNext(ctx, ba)
// maxSeenLeaseSequence tracks the maximum LeaseSequence seen in a
in pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas

cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go

Lines 459 to 461 in fcd74cd

return ds.sendToReplicas(
ctx,
in pkg/kv/kvclient/kvcoord.(*DistSender).sendRPC

cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go

Lines 541 to 543 in fcd74cd

class := rpc.ConnectionClassForKey(desc.RSpan().Key)
br, err := ds.sendRPC(ctx, ba, class, desc.RangeID, replicas, cachedLeaseHolder, withCommit)
if err != nil {
in pkg/kv/kvclient/kvcoord.(*DistSender).sendSingleRange

cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go

Lines 1400 to 1402 in fcd74cd

reply, pErr = ds.sendSingleRange(ctx, ba, desc, withCommit)
in pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch

cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go

Lines 1085 to 1087 in fcd74cd

if !ri.NeedAnother(rs) {
resp := ds.sendPartialBatch(
ctx, ba, rs, ri.Desc(), ri.Token(), withCommit, batchIdx, false, /* needsTruncate */
in pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges

cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go

Lines 736 to 738 in fcd74cd

} else {
rpl, pErr = ds.divideAndSendBatchToRanges(ctx, ba, rs, withCommit, 0 /* batchIdx */)
}
in pkg/kv/kvclient/kvcoord.(*DistSender).Send

cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go

Lines 85 to 87 in fcd74cd

defer gs.mu.Lock()
return gs.wrapped.Send(ctx, ba)
}
in pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked

cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go

Lines 45 to 47 in fcd74cd

br, pErr := m.wrapped.SendLocked(ctx, ba)
if pErr != nil {
in pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked

cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go

Lines 125 to 127 in fcd74cd

if !hasET {
return tc.wrapped.SendLocked(ctx, ba)
}
in pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked

cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go

Lines 224 to 226 in fcd74cd

}
br, pErr := sr.wrapped.SendLocked(ctx, ba)
in pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts

cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go

Lines 182 to 184 in fcd74cd

// Send through wrapped lockedSender. Unlocks while sending then re-locks.
br, pErr := sr.sendLockedWithRefreshAttempts(ctx, ba, maxAttempts)
if pErr != nil {
in pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked

cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go

Lines 223 to 225 in fcd74cd

// Send through wrapped lockedSender. Unlocks while sending then re-locks.
br, pErr := tp.wrapped.SendLocked(ctx, ba)
in pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked

cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go

Lines 104 to 106 in fcd74cd

return s.wrapped.SendLocked(ctx, ba)
}
in pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked

cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go

Lines 171 to 173 in fcd74cd

// Forward the batch through the wrapped lockedSender.
return h.wrapped.SendLocked(ctx, ba)
}
in pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked

cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go

Lines 503 to 505 in fcd74cd

// Send the command through the txnInterceptor stack.
br, pErr := tc.interceptorStack[0].SendLocked(ctx, ba)
in pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send

cockroach/pkg/kv/db.go

Lines 738 to 740 in fcd74cd

tracing.AnnotateTrace()
br, pErr := sender.Send(ctx, ba)
if pErr != nil {
in pkg/kv.(*DB).sendUsingSender

cockroach/pkg/kv/txn.go

Lines 893 to 895 in fcd74cd

txn.mu.Unlock()
br, pErr := txn.db.sendUsingSender(ctx, ba, sender)
if pErr == nil {
in pkg/kv.(*Txn).Send

cockroach/pkg/sql/row/kv_batch_fetcher.go

Lines 183 to 185 in fcd74cd

sendFn := func(ctx context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, error) {
res, err := txn.Send(ctx, ba)
if err != nil {
in pkg/sql/row.makeKVBatchFetcher.func1

cockroach/pkg/sql/row/kv_batch_fetcher.go

Lines 314 to 316 in fcd74cd

br, err := f.sendFn(ctx, ba)
if err != nil {
in pkg/sql/row.(*txnKVFetcher).fetch

cockroach/pkg/sql/row/kv_batch_fetcher.go

Lines 398 to 400 in fcd74cd

}
if err := f.fetch(ctx); err != nil {
return false, nil, nil, roachpb.Span{}, err
in pkg/sql/row.(*txnKVFetcher).nextBatch

cockroach/pkg/sql/row/kv_fetcher.go

Lines 86 to 88 in fcd74cd

ok, f.kvs, f.batchResponse, f.Span, err = f.nextBatch(ctx)
if err != nil {
in pkg/sql/row.(*KVFetcher).NextKV

cockroach/pkg/sql/row/fetcher.go

Lines 597 to 599 in fcd74cd

for {
ok, rf.kv, _, err = rf.kvFetcher.NextKV(ctx)
if err != nil {
in pkg/sql/row.(*Fetcher).NextKey

cockroach/pkg/sql/row/fetcher.go

Lines 587 to 589 in fcd74cd

// Retrieve the first key.
_, err := rf.NextKey(ctx)
return err
in pkg/sql/row.(*Fetcher).StartScanFrom

cockroach/pkg/sql/row/fetcher.go

Lines 483 to 485 in fcd74cd

}
return rf.StartScanFrom(ctx, &f)
}
in pkg/sql/row.(*Fetcher).StartScan

cockroach/pkg/sql/rowexec/tablereader.go

Lines 162 to 164 in fcd74cd

if tr.maxTimestampAge == 0 {
err = tr.fetcher.StartScan(
ctx, tr.FlowCtx.Txn, tr.spans,
in pkg/sql/rowexec.(*tableReader).Start

cockroach/pkg/sql/execinfra/processorsbase.go

Lines 747 to 749 in fcd74cd

}
ctx = pb.self.Start(ctx)
Run(ctx, pb.self, pb.Out.output)
in pkg/sql/execinfra.(*ProcessorBase).Run

cockroach/pkg/sql/flowinfra/flow.go

Lines 369 to 371 in fcd74cd

}
headProc.Run(ctx)
return nil
in pkg/sql/flowinfra.(*FlowBase).Run

cockroach/pkg/sql/distsql_running.go

Lines 377 to 379 in fcd74cd

// TODO(radu): this should go through the flow scheduler.
if err := flow.Run(ctx, func() {}); err != nil {
log.Fatalf(ctx, "unexpected error from syncFlow.Start(): %s "+
in pkg/sql.(*DistSQLPlanner).Run

cockroach/pkg/sql/distsql_running.go

Lines 987 to 989 in fcd74cd

recv.expectedRowsRead = int64(physPlan.TotalEstimatedScannedRows)
return dsp.Run(planCtx, txn, &physPlan, recv, evalCtx, nil /* finishedSetupFn */)
}
in pkg/sql.(*DistSQLPlanner).PlanAndRun

cockroach/pkg/sql/conn_executor_exec.go

Lines 867 to 869 in fcd74cd

// the planner whether or not to plan remote table readers.
cleanup := ex.server.cfg.DistSQLPlanner.PlanAndRun(
ctx, evalCtx, planCtx, planner.txn, planner.curPlan.plan, recv,
in pkg/sql.(*connExecutor).execWithDistSQLEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 765 to 767 in fcd74cd

ex.sessionTracing.TraceExecStart(ctx, "distributed")
bytesRead, rowsRead, err := ex.execWithDistSQLEngine(ctx, planner, stmt.AST.StatementType(), res, distributePlan, progAtomic)
ex.sessionTracing.TraceExecEnd(ctx, res.Err(), res.RowsAffected())
in pkg/sql.(*connExecutor).dispatchToExecutionEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 470 to 472 in fcd74cd

p.autoCommit = os.ImplicitTxn.Get() && !ex.server.cfg.TestingKnobs.DisableAutoCommit
if err := ex.dispatchToExecutionEngine(ctx, p, res); err != nil {
return nil, nil, err
in pkg/sql.(*connExecutor).execStmtInOpenState

cockroach/pkg/sql/conn_executor_exec.go

Lines 93 to 95 in fcd74cd

} else {
ev, payload, err = ex.execStmtInOpenState(ctx, stmt, res, pinfo)
}
in pkg/sql.(*connExecutor).execStmt

cockroach/pkg/sql/conn_executor.go

Lines 1393 to 1395 in fcd74cd

stmtCtx := withStatement(ctx, ex.curStmt)
ev, payload, err = ex.execStmt(stmtCtx, curStmt, stmtRes, nil /* pinfo */)
if err != nil {
in pkg/sql.(*connExecutor).execCmd

cockroach/pkg/sql/conn_executor.go

Lines 1322 to 1324 in fcd74cd

var err error
if err = ex.execCmd(ex.Ctx()); err != nil {
if err == io.EOF || err == errDrainingComplete {
in pkg/sql.(*connExecutor).run

cockroach/pkg/sql/conn_executor.go

Lines 477 to 479 in fcd74cd

}()
return h.ex.run(ctx, s.pool, reserved, cancel)
}
in pkg/sql.(*Server).ServeConn

cockroach/pkg/sql/pgwire/conn.go

Lines 593 to 595 in fcd74cd

reservedOwned = false // We're about to pass ownership away.
retErr = sqlServer.ServeConn(ctx, connHandler, reserved, cancelConn)
}()
in pkg/sql/pgwire.(*conn).processCommandsAsync.func1

pkg/kv/kvserver/replica_read.go in pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes at line 95
pkg/kv/kvserver/replica_read.go in pkg/kv/kvserver.(*Replica).executeReadOnlyBatch at line 67
pkg/kv/kvserver/replica_send.go in pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries at line 236
pkg/kv/kvserver/replica_send.go in pkg/kv/kvserver.(*Replica).sendWithRangeID at line 98
pkg/kv/kvserver/replica_send.go in pkg/kv/kvserver.(*Replica).Send at line 36
pkg/kv/kvserver/store_send.go in pkg/kv/kvserver.(*Store).Send at line 204
pkg/kv/kvserver/stores.go in pkg/kv/kvserver.(*Stores).Send at line 188
pkg/server/node.go in pkg/server.(*Node).batchInternal.func1 at line 925
pkg/util/stop/stopper.go in pkg/util/stop.(*Stopper).RunTaskWithErr at line 302
pkg/server/node.go in pkg/server.(*Node).batchInternal at line 913
pkg/server/node.go in pkg/server.(*Node).Batch at line 951
pkg/rpc/context.go in pkg/rpc.internalClientAdapter.Batch at line 537
pkg/kv/kvclient/kvcoord/transport.go in pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch at line 199
pkg/kv/kvclient/kvcoord/transport.go in pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext at line 168
pkg/kv/kvclient/kvcoord/dist_sender.go in pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas at line 1630
pkg/kv/kvclient/kvcoord/dist_sender.go in pkg/kv/kvclient/kvcoord.(*DistSender).sendRPC at line 460
pkg/kv/kvclient/kvcoord/dist_sender.go in pkg/kv/kvclient/kvcoord.(*DistSender).sendSingleRange at line 542
pkg/kv/kvclient/kvcoord/dist_sender.go in pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch at line 1401
pkg/kv/kvclient/kvcoord/dist_sender.go in pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges at line 1086
pkg/kv/kvclient/kvcoord/dist_sender.go in pkg/kv/kvclient/kvcoord.(*DistSender).Send at line 737
pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go in pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked at line 86
pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go in pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked at line 46
pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go in pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked at line 126
pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go in pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts at line 225
pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go in pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked at line 183
pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go in pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked at line 224
pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go in pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked at line 105
pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go in pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked at line 172
pkg/kv/kvclient/kvcoord/txn_coord_sender.go in pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send at line 504
pkg/kv/db.go in pkg/kv.(*DB).sendUsingSender at line 739
pkg/kv/txn.go in pkg/kv.(*Txn).Send at line 894
pkg/sql/row/kv_batch_fetcher.go in pkg/sql/row.makeKVBatchFetcher.func1 at line 184
pkg/sql/row/kv_batch_fetcher.go in pkg/sql/row.(*txnKVFetcher).fetch at line 315
pkg/sql/row/kv_batch_fetcher.go in pkg/sql/row.(*txnKVFetcher).nextBatch at line 399
pkg/sql/row/kv_fetcher.go in pkg/sql/row.(*KVFetcher).NextKV at line 87
pkg/sql/row/fetcher.go in pkg/sql/row.(*Fetcher).NextKey at line 598
pkg/sql/row/fetcher.go in pkg/sql/row.(*Fetcher).StartScanFrom at line 588
pkg/sql/row/fetcher.go in pkg/sql/row.(*Fetcher).StartScan at line 484
pkg/sql/rowexec/tablereader.go in pkg/sql/rowexec.(*tableReader).Start at line 163
pkg/sql/execinfra/processorsbase.go in pkg/sql/execinfra.(*ProcessorBase).Run at line 748
pkg/sql/flowinfra/flow.go in pkg/sql/flowinfra.(*FlowBase).Run at line 370
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).Run at line 378
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).PlanAndRun at line 988
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execWithDistSQLEngine at line 868
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).dispatchToExecutionEngine at line 766
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmtInOpenState at line 471
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmt at line 94
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd at line 1394
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).run at line 1323
pkg/sql/conn_executor.go in pkg/sql.(*Server).ServeConn at line 478
pkg/sql/pgwire/conn.go in pkg/sql/pgwire.(*conn).processCommandsAsync.func1 at line 594
Tag Value
Cockroach Release v20.1.0-beta.3
Cockroach SHA: fcd74cd
Platform linux amd64
Distribution CCL
Environment v20.1.0-beta.3
Command server
Go Version go1.13.5
# of CPUs 8
# of Goroutines 219

Metadata

Metadata

Assignees

Labels

C-bugCode not up to spec/doc, specs & docs deemed correct. Solution expected to change code/behavior.O-sentryOriginated from an in-the-wild panic report.

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions