panic: concurrent write operations detected on file [recovered]
	panic: concurrent write operations detected on file [recovered]
	panic: concurrent write operations detected on file [recovered]
	panic: concurrent write operations detected on file

goroutine 18636 [running]:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).recover(0x4af857?, {0x9d23388, 0xc004ac1ef0})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:229 +0x79
panic({0x65b59c0, 0x9ce4880})
	GOROOT/src/runtime/panic.go:890 +0x262
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).recover(0x4af857?, {0x9d23388, 0xc008a68660})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:229 +0x79
panic({0x65b59c0, 0x9ce4880})
	GOROOT/src/runtime/panic.go:890 +0x262
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).applySnapshot.func1()
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raftstorage.go:525 +0x405
panic({0x65b59c0, 0x9ce4880})
	GOROOT/src/runtime/panic.go:890 +0x262
github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).timeDiskOp(0xc005418780, 0x2, 0xc00964dde0)
	github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:262 +0x1a5
github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).Sync(0xc005418780)
	github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:219 +0x69
github.com/cockroachdb/pebble/vfs.(*enospcFile).Sync(0xc0070946f0)
	github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_full.go:391 +0x6d
github.com/cockroachdb/pebble.(*DB).ingest(0xc005269400, {0xc006963e80, 0x6, 0x8}, 0x70418a0)
	github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/ingest.go:728 +0x442
github.com/cockroachdb/pebble.(*DB).IngestWithStats(0xc005269400, {0xc006963e80, 0x6, 0x8})
	github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/ingest.go:680 +0x111
github.com/cockroachdb/cockroach/pkg/storage.(*Pebble).IngestExternalFilesWithStats(0xc002ff1b80, {0x0?, 0x0?}, {0xc006963e80, 0x6, 0x8})
	github.com/cockroachdb/cockroach/pkg/storage/pebble.go:1764 +0x66
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).applySnapshot(0xc005833200, {0x9d23388, _}, {{0x45, 0x22, 0xc6, 0xc8, 0x9b, 0x5b, 0x42, ...}, ...}, ...)
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raftstorage.go:657 +0x1274
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x45, 0x22, 0xc6, 0xc8, 0x9b, 0x5b, 0x42, ...}, ...})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:806 +0xa0e
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processRaftSnapshotRequest.func1({0x9d23388, 0xc00644ddd0}, 0xc005833200)
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:463 +0x3b0
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).withReplicaForRequest(0x0?, {0x9d23388, 0xc00644ddd0}, 0xc004de5e68, 0xc0096500b8)
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:341 +0x15c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processRaftSnapshotRequest(0xc001394a80, {0x9d23388, 0xc00644ddd0}, 0xc004de5e00, {{0x45, 0x22, 0xc6, 0xc8, 0x9b, 0x5b, ...}, ...})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:404 +0xe5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).receiveSnapshot(0xc001394a80, {0x9d23388, 0xc000d38840}, 0xc004de5e00, {0x7f1ca7b70b90, 0xc004bde470})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:1089 +0xc6e
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot.func1({0x9d23388, 0xc008a68660})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:210 +0xfd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000e67c20, {0x9d23388, 0xc008a68660}, {0xc006d6e480?, 0x3?}, 0xc00bd46b68)
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0x148
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot(0xc001394a80, {0x9d232e0, 0xc0051f7bc0}, 0xc004de5e00, {0x7f1ca7b70b68, 0xc004bde470})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:207 +0x114
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftSnapshot(0xc000d2bf00, {0x9d5e1f8, 0xc004bde470})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:375 +0x25b
github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftSnapshot_Handler({0x6ca9ce0?, 0xc000d2bf00}, {0x9d53320?, 0xc002f20620})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-fastbuild/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:270 +0xc3
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x6ca9ce0, 0xc000d2bf00}, {0x9d53830?, 0xc0035b0ee0?}, 0xc00b089ab8, 0x703b0f0)
	github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:163 +0x67a
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x6ca9ce0, 0xc000d2bf00}, {0x9d53830, 0xc0035b0ee0})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x106
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x6ca9ce0, 0xc000d2bf00}, {0x9d53830, 0xc0035b0ee0}, 0xc00b089ab8, 0xc0051f7a80)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:272 +0xe9
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x6ca9ce0, 0xc000d2bf00}, {0x9d53830, 0xc0035b0ee0})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x1ea
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({{{0x6284920?}}}, {0x6ca9ce0, 0xc000d2bf00}, {0x9d53830, 0xc0035b0ee0}, 0xc00b089ab8, 0xc0051f7a80)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:136 +0x468
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x6ca9ce0, 0xc000d2bf00}, {0x9d53830, 0xc0035b0ee0})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x1ea
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0xc000e67c20?, 0x174002b35f477d3a?})
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:241 +0x70
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000e67c20, {0x9d23388, 0xc004ac1ef0}, {0x203001?, 0x203001?}, 0xc0043426c0)
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0x148
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x6ca9ce0, 0xc000d2bf00}, {0x9d53830?, 0xc0035b0ee0?}, 0xc00b089ab8, 0xc0051f7a80)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:240 +0x14d
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x6ca9ce0, 0xc000d2bf00}, {0x9d53830, 0xc0035b0ee0})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x1ea
google.golang.org/grpc.chainStreamInterceptors.func1({0x6ca9ce0, 0xc000d2bf00}, {0x9d53830, 0xc0035b0ee0}, 0xc00b089ab8, 0x703b0f0)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x276
google.golang.org/grpc.(*Server).processStreamingRPC(0xc002adb2c0, {0x9d68d00, 0xc00573aea0}, 0xc001624900, 0xc007923020, 0xc532d40, 0x0)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x1ef6
google.golang.org/grpc.(*Server).handleStream(0xc002adb2c0, {0x9d68d00, 0xc00573aea0}, 0xc001624900, 0x0)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0xfaf
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0xed
created by google.golang.org/grpc.(*Server).serveStreams.func1
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x4de