Skip to content

panic crash while on the database pages #83935

@maryliag

Description

@maryliag

Seeing a panic crash, (other people mentioned they saw on other occasions, but these are the steps I was able to reproduce):

Created a build with make build
Started CRDB with ./cockroach demo --insecure --multitenant=false
Start the console with make ui-watch TARGET=http://localhost:8080/
Open the db console on one of the database list of tables, e.g. http://localhost:3000/#/database/system

Wait for awhile (sometime it took me a few minutes, sometimes 30min, sometimes doesn't happen at all) and it will crash and the trace shows on the terminal

Trace:

# Server version: CockroachDB CCL v22.2.0-alpha.00000000-1090-g4dc922688e (x86_64-apple-darwin21.5.0, built 2022/07/05 21:13:24, go1.17.2) (same version as client)
# Cluster ID: 02f64d33-30bc-468c-967a-8884de0ff2ba
# Organization: Cockroach Demo
#
# Enter \? for a brief introduction.
#
root@127.0.0.1:26257/movr> panic: kvfetcher-0-unlimited-1: no bytes in account to release, current 0, free 82 [recovered]
	panic: kvfetcher-0-unlimited-1: no bytes in account to release, current 0, free 82 [recovered]
	panic: kvfetcher-0-unlimited-1: no bytes in account to release, current 0, free 82

goroutine 101639 [running]:
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError.func1()
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:58 +0x3bb
panic({0x86dc4e0, 0xc00cb04f00})
	/usr/local/opt/go/libexec/src/runtime/panic.go:1038 +0x215
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError.func1()
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:58 +0x3bb
panic({0x86dc4e0, 0xc00cb04f00})
	/usr/local/opt/go/libexec/src/runtime/panic.go:1038 +0x215
github.com/cockroachdb/cockroach/pkg/util/log/logcrash.ReportOrPanic({0xb5560e8, 0xc00e45c3c0}, 0xc001108a80, {0x8bbc4a0, 0xc00cb07f00}, {0xc00cd1d, 0x5, 0x5})
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/util/log/logcrash/crash_reporting.go:378 +0x1c5
github.com/cockroachdb/cockroach/pkg/util/mon.(*BoundAccount).Shrink(0xc00e45c270, {0xb5560e8, 0xc00e45c3c0}, 0x52)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/util/mon/bytes_usage.go:715 +0x1d5
github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).reset(0xc00cb7c300, {0xb5560e8, 0xc00e45c3c0})
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:648 +0x125
github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).close(0x400e6bd, {0xb5560e8, 0xc00e45c3c0})
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:654 +0x25
github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).Close(...)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:300
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).Close(0xc00ba2c000, {0xb5560e8, 0xc00e45c3c0})
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:1330 +0x6d
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Close(0xc00b330d20, {0x8a82b1a, 0x8a8183c})
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:298 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexecop.Closers.CloseAndLogOnErr.func1()
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecop/operator.go:177 +0xae
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xe62b7f8)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexecop.Closers.CloseAndLogOnErr({0xc00cb07e90, 0xb4b50e0, 0xc00f5e42e0}, {0xb556040, 0xc00cb49800}, {0x8a9fa10, 0xc})
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecop/operator.go:175 +0xcd
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).close(0xc00cba4960)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:324 +0x98
github.com/cockroachdb/cockroach/pkg/sql/colexec.newMaterializerInternal.func1()
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:210 +0x1d
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).moveToTrailingMeta(0xc00cba4960)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:682 +0x364
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).DrainHelper(0xc00cba4960)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:561 +0x189
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc00cba4960)
	/Users/maryliag/go/src/github.com/cockroachdb/coc:311 +0xa5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).DrainHelper(0xc003de2240)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:557 +0x11c
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).next(0x0)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:143 +0x92
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).nextAdapter(...)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:147
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x0)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Next(0xc003de2240)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:152 +0x3e
github.com/cockroachdb/cockroach/pkg/sql/execinfra.DrainAndForwardMetadata({0xb556040, 0xc00cb49800}, {0xb57f4d8, 0xc003de2240}, {0xb4cfc50, 0xc004348700})
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/base.go:220 +0x75
github.com/cockroachdb/cockroach/pkg/sql/execinfra.Run({0xb556040, 0xc00cb49800}, {0xb57f4d8, 0xc003de2240}, {0xb4cfc50, 0xc004348700})
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/base.go:193 +0xe5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc003de2240, {0xb556040, 0xc00cb49800})
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:722 +0x5b
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc00cba4780, {0xb556040, 0xc00cb49800}, 0xc003de2240)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:475 +0x258
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc00c7182d0, {0xb556040, 0xc00cb49800}, 0xc00cc888c0)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:249 +0x205
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc001ea3a40, {0xb5560e8, 0xc00e45c0c0}, 0xc00c76b420, 0xc00cc888c0, 0xc00cd02300, 0xc004348700, 0xc00c7182d0, 0x0)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:607 +0xb04
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xb5560e8, {0xb5560e8, 0xc00e45c0c0}, 0xc00c718010, 0xc00c76b420, 0xc00e81bf20, {{0xb557c08, 0xc00cd02280}, 0x0}, 0xc004348700)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1461 +0x25c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00c717900, {0xb5560e8, 0xc00e45c0c0}, 0xc00c718010, 0xc00e45c0c0, {0xb5fdf18, 0xc00e81bf20}, 0x50, 0xc0091a3618)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1485 +0x614
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00c717900, {0xb5560e8, 0xc00e81bfb0}, 0xc00c718010, {0xb5fdf18, 0xc00e81bf20})
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1159 +0xb87
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00c717900, {0xb556040, 0xc00cb495c0}, {{0xb586798, 0xc00c72be50}, {0xc002e1f380, 0x51}, 0x0, 0x1}, 0x0, ...)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:690 +0x2091
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00c717900, {0xb556040, 0xc00cb495c0}, {{0xb586798, 0xc00c72be50}, {0xc002e1f380, 0x51}, 0x0, 0x1}, 0x0, ...)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:145 +0x59e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0xb586798, 0xc00c72be50}, {0xc002e1f380, 0x51}, 0x0, 0x1}, {0xc0a94c0b66814cc0, 0x366afd67ace, 0x0}, {0xc0a94c0b66814cc0, ...}, ...}, ...)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1892 +0x2f6
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00c717900)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1896 +0xb48
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00c717900, {0xb5560e8, 0xc00e81bbc0}, 0xc00e81b9e0, {0x0, 0x0, 0x0, 0x0, 0x0}, 0x0)
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1818 +0x26c
github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).initConnEx.func1()
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/internal.go:206 +0xa5
created by github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).initConnEx
	/Users/maryliag/go/src/github.com/cockroachdb/cockroach/pkg/sql/internal.go:205 +0x5f1

Jira issue: CRDB-17362

Metadata

Metadata

Assignees

Labels

C-bugCode not up to spec/doc, specs & docs deemed correct. Solution expected to change code/behavior.T-sql-queriesSQL Queries Team

Type

No type

Projects

Status

Done

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions