Skip to content

sentry: crash_reporting.go:338: index-backfill-mon: unexpected 71680 leftover bytes -- *errutil.leafError: index-backfill-mon: unexpected 71680 leftover bytes (1) crash_reporting.go:338: *withstack.withStack (top exception) (check the extra data payloads) #57366

Closed
Closed
sentry: crash_reporting.go:338: index-backfill-mon: unexpected 71680 leftover bytes -- *errutil.leafError: index-backfill-mon: unexpected 71680 leftover bytes (1) crash_reporting.go:338: *withstack.withStack (top exception) (check the extra data payloads)#57366
Labels
C-bugCode not up to spec/doc, specs & docs deemed correct. Solution expected to change code/behavior.O-sentryOriginated from an in-the-wild panic report.
@cockroach-teamcity

Description

@cockroach-teamcity
cockroach-teamcity
opened on Dec 2, 2020

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/2067664750/?referrer=webhooks_plugin

Panic message:

crash_reporting.go:338: index-backfill-mon: unexpected 71680 leftover bytes
--
*errutil.leafError: index-backfill-mon: unexpected 71680 leftover bytes (1)
crash_reporting.go:338: *withstack.withStack (top exception)
(check the extra data payloads)

Stacktrace (expand for inline code snippets):

cockroach/pkg/util/log/crash_reporting.go

Lines 337 to 339 in 92d9495

) {
err := errors.Newf(format, reportables...)
if !build.IsRelease() || (sv != nil && PanicOnAssertions.Get(sv)) {
in pkg/util/log.ReportOrPanic

cockroach/pkg/util/mon/bytes_usage.go

Lines 403 to 405 in 92d9495

if check && mm.mu.curAllocated != 0 {
log.ReportOrPanic(
ctx, &mm.settings.SV,
in pkg/util/mon.(*BytesMonitor).doStop

cockroach/pkg/util/mon/bytes_usage.go

Lines 390 to 392 in 92d9495

func (mm *BytesMonitor) Stop(ctx context.Context) {
mm.doStop(ctx, true)
}
in pkg/util/mon.(*BytesMonitor).Stop

cockroach/pkg/sql/backfill/backfill.go

Lines 516 to 518 in 92d9495

ib.boundAccount.Close(ctx)
ib.mon.Stop(ctx)
}
in pkg/sql/backfill.(*IndexBackfiller).Close

cockroach/pkg/sql/rowexec/indexbackfiller.go

Lines 115 to 117 in 92d9495

func (ib *indexBackfiller) close(ctx context.Context) {
ib.IndexBackfiller.Close(ctx)
ib.adder.Close(ctx)
in pkg/sql/rowexec.(*indexBackfiller).close

cockroach/pkg/sql/rowexec/backfiller.go

Lines 237 to 239 in 92d9495

return finishedSpans, nil
}
in pkg/sql/rowexec.(*backfiller).mainLoop

cockroach/pkg/sql/rowexec/backfiller.go

Lines 138 to 140 in 92d9495

}
finishedSpans, err := b.mainLoop(ctx, mutations)
if err != nil {
in pkg/sql/rowexec.(*backfiller).doRun

cockroach/pkg/sql/rowexec/backfiller.go

Lines 122 to 124 in 92d9495

defer tracing.FinishSpan(span)
meta := b.doRun(ctx)
execinfra.SendTraceData(ctx, b.output)
in pkg/sql/rowexec.(*backfiller).Run

cockroach/pkg/sql/flowinfra/flow.go

Lines 391 to 393 in 92d9495

}
headProc.Run(ctx)
return nil
in pkg/sql/flowinfra.(*FlowBase).Run

cockroach/pkg/sql/distsql_running.go

Lines 421 to 423 in 92d9495

// TODO(radu): this should go through the flow scheduler.
if err := flow.Run(ctx, func() {}); err != nil {
log.Fatalf(ctx, "unexpected error from syncFlow.Start(): %v\n"+
in pkg/sql.(*DistSQLPlanner).Run

cockroach/pkg/sql/backfill.go

Lines 1018 to 1020 in 92d9495

}
sc.distSQLPlanner.Run(
planCtx,
in pkg/sql.(*SchemaChanger).distBackfill.func4

cockroach/pkg/kv/db.go

Lines 706 to 708 in 92d9495

err := txn.exec(ctx, func(ctx context.Context, txn *Txn) error {
return retryable(ctx, txn)
})
in pkg/kv.(*DB).Txn.func1

cockroach/pkg/kv/txn.go

Lines 810 to 812 in 92d9495

}
err = fn(ctx, txn)
in pkg/kv.(*Txn).exec

cockroach/pkg/kv/db.go

Lines 705 to 707 in 92d9495

txn.SetDebugName("unnamed")
err := txn.exec(ctx, func(ctx context.Context, txn *Txn) error {
return retryable(ctx, txn)
in pkg/kv.(*DB).Txn

cockroach/pkg/sql/backfill.go

Lines 957 to 959 in 92d9495

var updatedTodoSpans []roachpb.Span
if err := sc.db.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
updatedTodoSpans = todoSpans
in pkg/sql.(*SchemaChanger).distBackfill

cockroach/pkg/sql/backfill.go

Lines 1499 to 1501 in 92d9495

chunkSize := indexBulkBackfillChunkSize.Get(&sc.settings.SV)
if err := sc.distBackfill(
ctx, version, indexBackfill, chunkSize,
in pkg/sql.(*SchemaChanger).backfillIndexes

cockroach/pkg/sql/backfill.go

Lines 299 to 301 in 92d9495

// Check if bulk-adding is enabled and supported by indexes (ie non-unique).
if err := sc.backfillIndexes(ctx, version, addedIndexSpans); err != nil {
return err
in pkg/sql.(*SchemaChanger).runBackfill

cockroach/pkg/sql/schema_changer.go

Lines 1420 to 1422 in 92d9495

// Run backfill(s).
if err := sc.runBackfill(ctx); err != nil {
return err
in pkg/sql.(*SchemaChanger).runStateMachineAndBackfill

cockroach/pkg/sql/schema_changer.go

Lines 669 to 671 in 92d9495

// Run through mutation state machine and backfill.
if err := sc.runStateMachineAndBackfill(ctx); err != nil {
return err
in pkg/sql.(*SchemaChanger).exec

cockroach/pkg/sql/schema_changer.go

Lines 2072 to 2074 in 92d9495

// called at least once before there is a chance for this loop to exit.
scErr = sc.exec(ctx)
switch {
in pkg/sql.schemaChangeResumer.Resume.func1

cockroach/pkg/sql/schema_changer.go

Lines 2184 to 2186 in 92d9495

if details.DescID != descpb.InvalidID {
return execSchemaChange(details.DescID, details.TableMutationID, details.DroppedDatabaseID)
}
in pkg/sql.schemaChangeResumer.Resume

cockroach/pkg/jobs/registry.go

Lines 1080 to 1082 in 92d9495

defer jm.CurrentlyRunning.Dec(1)
err = resumer.Resume(resumeCtx, phs, resultsCh)
}()
in pkg/jobs.(*Registry).stepThroughStateMachine.func1

cockroach/pkg/jobs/registry.go

Lines 1081 to 1083 in 92d9495

err = resumer.Resume(resumeCtx, phs, resultsCh)
}()
if err == nil {
in pkg/jobs.(*Registry).stepThroughStateMachine

cockroach/pkg/jobs/adopt.go

Lines 243 to 245 in 92d9495

// Run the actual job.
err := r.stepThroughStateMachine(ctx, phs, resumer, resultsCh, job, status, finalResumeError)
if err != nil {
in pkg/jobs.(*Registry).runJob

cockroach/pkg/jobs/adopt.go

Lines 180 to 182 in 92d9495

if err := r.stopper.RunAsyncTask(ctx, job.taskName(), func(ctx context.Context) {
r.runJob(resumeCtx, resumer, resultsCh, errCh, job, status, job.taskName(), nil)
}); err != nil {
in pkg/jobs.(*Registry).resumeJob.func1

cockroach/pkg/util/stop/stopper.go

Lines 346 to 348 in 92d9495

f(ctx)
}()
in pkg/util/stop.(*Stopper).RunAsyncTask.func1
/usr/local/go/src/runtime/asm_amd64.s#L1356-L1358 in runtime.goexit

pkg/util/log/crash_reporting.go in pkg/util/log.ReportOrPanic at line 338
pkg/util/mon/bytes_usage.go in pkg/util/mon.(*BytesMonitor).doStop at line 404
pkg/util/mon/bytes_usage.go in pkg/util/mon.(*BytesMonitor).Stop at line 391
pkg/sql/backfill/backfill.go in pkg/sql/backfill.(*IndexBackfiller).Close at line 517
pkg/sql/rowexec/indexbackfiller.go in pkg/sql/rowexec.(*indexBackfiller).close at line 116
pkg/sql/rowexec/backfiller.go in pkg/sql/rowexec.(*backfiller).mainLoop at line 238
pkg/sql/rowexec/backfiller.go in pkg/sql/rowexec.(*backfiller).doRun at line 139
pkg/sql/rowexec/backfiller.go in pkg/sql/rowexec.(*backfiller).Run at line 123
pkg/sql/flowinfra/flow.go in pkg/sql/flowinfra.(*FlowBase).Run at line 392
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).Run at line 422
pkg/sql/backfill.go in pkg/sql.(*SchemaChanger).distBackfill.func4 at line 1019
pkg/kv/db.go in pkg/kv.(*DB).Txn.func1 at line 707
pkg/kv/txn.go in pkg/kv.(*Txn).exec at line 811
pkg/kv/db.go in pkg/kv.(*DB).Txn at line 706
pkg/sql/backfill.go in pkg/sql.(*SchemaChanger).distBackfill at line 958
pkg/sql/backfill.go in pkg/sql.(*SchemaChanger).backfillIndexes at line 1500
pkg/sql/backfill.go in pkg/sql.(*SchemaChanger).runBackfill at line 300
pkg/sql/schema_changer.go in pkg/sql.(*SchemaChanger).runStateMachineAndBackfill at line 1421
pkg/sql/schema_changer.go in pkg/sql.(*SchemaChanger).exec at line 670
pkg/sql/schema_changer.go in pkg/sql.schemaChangeResumer.Resume.func1 at line 2073
pkg/sql/schema_changer.go in pkg/sql.schemaChangeResumer.Resume at line 2185
pkg/jobs/registry.go in pkg/jobs.(*Registry).stepThroughStateMachine.func1 at line 1081
pkg/jobs/registry.go in pkg/jobs.(*Registry).stepThroughStateMachine at line 1082
pkg/jobs/adopt.go in pkg/jobs.(*Registry).runJob at line 244
pkg/jobs/adopt.go in pkg/jobs.(*Registry).resumeJob.func1 at line 181
pkg/util/stop/stopper.go in pkg/util/stop.(*Stopper).RunAsyncTask.func1 at line 347
/usr/local/go/src/runtime/asm_amd64.s in runtime.goexit at line 1357
Tag Value
Cockroach Release v20.2.2
Cockroach SHA: 92d9495
Platform darwin amd64
Distribution CCL
Environment v20.2.2
Command start-single-node
Go Version ``
# of CPUs
# of Goroutines

Metadata

Metadata

Assignees

No one assigned

    Labels

    C-bugCode not up to spec/doc, specs & docs deemed correct. Solution expected to change code/behavior.O-sentryOriginated from an in-the-wild panic report.

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions