Panic after HDD removal #2107
Description
Panic on hdd failover with writecache.
- generate io with writecache object size (1Mb for example)
- remove hdd
ноя 28 11:47:47 az neofs-node[1360]: 2022-11-28T11:47:47.777Z info log/log.go:13 local object storage operation {"shard_id": "WgNyS4Z8ozVPirhYcuU5Qp", "address": "4Ks2F4fQx8pnD>
ноя 28 11:47:47 az neofs-node[1360]: 2022-11-28T11:47:47.791Z info log/log.go:13 local object storage operation {"shard_id": "9wSDGtjaBMEHmZShXYPzmM", "address": "FJkKa8cE1yTWB>
ноя 28 11:47:47 az neofs-node[1360]: 2022-11-28T11:47:47.791Z warn engine/engine.go:160 could not put object to shard {"shard_id": "E5QCeMJ7ew6LN9QHsg62vr", "error count": 3, ">
ноя 28 11:47:47 az neofs-node[1360]: 2022-11-28T11:47:47.798Z info log/log.go:13 local object storage operation {"shard_id": "G6EapQ29PXY7yt3tCn8Wqp", "address": "FF4Rf4nuXpUQh>
ноя 28 11:47:47 az neofs-node[1360]: 2022-11-28T11:47:47.801Z info log/log.go:13 local object storage operation {"shard_id": "LsPJbLkMKveTAgPfvEaBzj", "address": "5Cv3UoLaBn3VT>
ноя 28 11:47:47 az neofs-node[1360]: 2022-11-28T11:47:47.802Z info log/log.go:13 local object storage operation {"shard_id": "UcEi9Xf58Q4uo4wWuduASu", "address": "7hnAU1LQFgBcs>
ноя 28 11:47:47 az neofs-node[1360]: 2022-11-28T11:47:47.802Z warn engine/engine.go:160 could not get object from shard {"shard_id": "E5QCeMJ7ew6LN9QHsg62vr", "error count": 4,>
ноя 28 11:47:47 az neofs-node[1360]: unexpected fault address 0x7f6d3b9ba040
ноя 28 11:47:47 az neofs-node[1360]: fatal error: fault
ноя 28 11:47:47 az neofs-node[1360]: [signal SIGBUS: bus error code=0x2 addr=0x7f6d3b9ba040 pc=0xbeaaf0]
ноя 28 11:47:47 az neofs-node[1360]: goroutine 115428819 [running]:
ноя 28 11:47:47 az neofs-node[1360]: runtime.throw({0x10d4cfa?, 0x2030b9?})
ноя 28 11:47:47 az neofs-node[1360]: runtime/panic.go:992 +0x71 fp=0xc0123593a0 sp=0xc012359370 pc=0x438bf1
ноя 28 11:47:47 az neofs-node[1360]: runtime.sigpanic()
ноя 28 11:47:47 az neofs-node[1360]: runtime/signal_unix.go:815 +0x125 fp=0xc0123593f0 sp=0xc0123593a0 pc=0x44ed85
ноя 28 11:47:47 az neofs-node[1360]: go.etcd.io/bbolt.(*DB).meta(0x7f6f1aee2c78?)
ноя 28 11:47:47 az neofs-node[1360]: go.etcd.io/bbolt@v1.3.6/db.go:948 +0x30 fp=0xc012359438 sp=0xc0123593f0 pc=0xbeaaf0
ноя 28 11:47:47 az neofs-node[1360]: go.etcd.io/bbolt.(*Tx).init(0xc2e6930460, 0xc00095cd80)
ноя 28 11:47:47 az neofs-node[1360]: go.etcd.io/bbolt@v1.3.6/tx.go:50 +0x91 fp=0xc0123594a0 sp=0xc012359438 pc=0xbf2c71
ноя 28 11:47:47 az neofs-node[1360]: go.etcd.io/bbolt.(*DB).beginTx(0xc00095cd80)
ноя 28 11:47:47 az neofs-node[1360]: go.etcd.io/bbolt@v1.3.6/db.go:614 +0x105 fp=0xc012359510 sp=0xc0123594a0 pc=0xbe9205
ноя 28 11:47:47 az neofs-node[1360]: go.etcd.io/bbolt.(*DB).Begin(0xc29adf75c0?, 0x59?)
ноя 28 11:47:47 az neofs-node[1360]: go.etcd.io/bbolt@v1.3.6/db.go:591 +0x2c fp=0xc012359528 sp=0xc012359510 pc=0xbe90cc
ноя 28 11:47:47 az neofs-node[1360]: go.etcd.io/bbolt.(*DB).View(0x2509d0f37fcb68ba?, 0xc012359618)
ноя 28 11:47:47 az neofs-node[1360]: go.etcd.io/bbolt@v1.3.6/db.go:756 +0x3b fp=0xc012359590 sp=0xc012359528 pc=0xbe9d1b
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza.(*Blobovnicza).Get(0xc000118500, {{{0xba, 0x68, 0xcb, 0x7f, 0xf3, 0xd0, 0x9, 0x25, 0x38, ...}, ...>
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza/get.go:48 +0xee fp=0xc012359680 sp=0xc012359590 pc=0xbf89ee
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree.(*Blobovniczas).getObject(0xc0003ce460, 0x21?, {{{0xba, 0x68, 0xcb, 0x7f, 0xf3, 0xd0,>
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree/get.go:126 +0x5b fp=0xc012359708 sp=0xc012359680 pc=0xc3c41b
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree.(*Blobovniczas).getObjectFromLevel(0xc0003ce460, {{{0xba, 0x68, 0xcb, 0x7f, 0xf3, 0xd>
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree/get.go:75 +0x175 fp=0xc012359970 sp=0xc012359708 pc=0xc3bbd5
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree.(*Blobovniczas).Get.func1({0xc00713c8ab, 0x3})
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree/get.go:40 +0xe5 fp=0xc012359ab0 sp=0xc012359970 pc=0xc3b785
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree.(*Blobovniczas).iterateSortedLeaves.func1({0xc4a5906540?, 0xc10e2d32c0?, 0xa?})
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go:70 +0x31 fp=0xc012359ae0 sp=0xc012359ab0 pc=0xc3e5b1
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree.(*Blobovniczas).iterateSorted(0xc0003ce460, 0xc51476a0d8?, {0xc10e2d32c0, 0x1, 0x1}, >
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go:109 +0x2e5 fp=0xc012359b80 sp=0xc012359ae0 pc=0xc3ea25
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree.(*Blobovniczas).iterateSorted(0xc0003ce460, 0xc063c65000?, {0xc10e2d32c0, 0x0, 0x1}, >
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go:114 +0x31d fp=0xc012359c20 sp=0xc012359b80 pc=0xc3ea5d
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree.(*Blobovniczas).iterateSortedLeaves(0xc0003ce460, 0xc0022d1d40?, 0xc012359db0)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go:66 +0x7c fp=0xc012359c80 sp=0xc012359c20 pc=0xc3e53c
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree.(*Blobovniczas).Get(0xc0003ce460, {{{0xba, 0x68, 0xcb, 0x7f, 0xf3, 0xd0, 0x9, 0x25, 0>
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree/get.go:35 +0x2bb fp=0xc012359e80 sp=0xc012359c80 pc=0xc3b55b
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor.(*BlobStor).Get(0xc000325ce0, {{{0xba, 0x68, 0xcb, 0x7f, 0xf3, 0xd0, 0x9, 0x25, 0x38, ...}, ...}, ...>
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/get.go:20 +0x2f5 fp=0xc012359fc8 sp=0xc012359e80 pc=0xc46cb5
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard.(*Shard).Get.func1(0x1271880?, {0x0?, 0xf7d120?, 0x0?})
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/get.go:70 +0xb8 fp=0xc01235a0a0 sp=0xc012359fc8 pc=0xc7c898
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard.(*Shard).fetchObjectData(0xc0000efb30, {{0xba, 0x68, 0xcb, 0x7f, 0xf3, 0xd0, 0x9, 0x25, 0x38, ...}, ...}>
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/get.go:124 +0x5b6 fp=0xc01235a2d0 sp=0xc01235a0a0 pc=0xc7ceb6
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard.(*Shard).Get(0x4010100?, {{{0xba, 0x68, 0xcb, 0x7f, 0xf3, 0xd0, 0x9, 0x25, 0x38, ...}, ...}, ...})
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/get.go:83 +0x19b fp=0xc01235a3e8 sp=0xc01235a2d0 pc=0xc7c73b
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).get.func2(0xc000324e70?, {0xc000421c20?, 0xc0000efb30?})
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/get.go:149 +0x87 fp=0xc01235a460 sp=0xc01235a3e8 pc=0xc8c287
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).iterateOverSortedShards(0xc109eba000?, {{0xba, 0x68, 0xcb, 0x7f, 0xf3, 0xd0, 0x9, 0x25>
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/shards.go:217 +0xc8 fp=0xc01235a4e8 sp=0xc01235a460 pc=0xc96648
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).get(0xc000324e70, {{{0xba, 0x68, 0xcb, 0x7f, 0xf3, 0xd0, 0x9, 0x25, 0x38, ...}, ...}})
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/get.go:143 +0x4c5 fp=0xc01235a7b8 sp=0xc01235a4e8 pc=0xc8bea5
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).Get.func1()
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/get.go:48 +0x56 fp=0xc01235a820 sp=0xc01235a7b8 pc=0xc8b976
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).execIfNotBlocked(0x10511c0?, 0xc4f49b21c0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/control.go:147 +0xcf fp=0xc01235a878 sp=0xc01235a820 pc=0xc8588f
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).Get(0xc01235a9c8?, {{{0xba, 0x68, 0xcb, 0x7f, 0xf3, 0xd0, 0x9, 0x25, 0x38, ...}, ...}})
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/get.go:47 +0xb9 fp=0xc01235a910 sp=0xc01235a878 pc=0xc8b8d9
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/get.(*storageEngineWrapper).get(0xc7ed9850757523de?, 0xfd6473d6c5f2ea83?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/get/util.go:225 +0x1e5 fp=0xc01235ad90 sp=0xc01235a910 pc=0xcfc8c5
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/get.(*execCtx).executeLocal(0xc329da56c0)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/get/local.go:14 +0x4b fp=0xc01235aea8 sp=0xc01235ad90 pc=0xcfa5ab
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/get.(*execCtx).execute(0xc329da56c0)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/get/get.go:90 +0x48 fp=0xc01235aee8 sp=0xc01235aea8 pc=0xcfa2e8
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/get.(*Service).get(0xc001119fa0, {0x127c160?, 0xc494f5f7a0}, {{0x1277910, 0xc0fc1506e0}, 0xc015cdcf00, {{0xba, 0x68>
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/get/get.go:81 +0x17e fp=0xc01235af30 sp=0xc01235aee8 pc=0xcfa1fe
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/get.(*Service).Get(...)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/get/get.go:13
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/get/v2.(*Service).Get(0xc001119fa8, 0xf6bbe0?, {0x12776b8, 0xc15fd37f20})
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/get/v2/service.go:48 +0xfa fp=0xc01235b070 sp=0xc01235af30 pc=0xdf90da
ноя 28 11:47:47 az neofs-node[1360]: main.(*objectSvc).Get(0x0?, 0x21?, {0x12776b8?, 0xc15fd37f20?})
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/cmd/neofs-node/object.go:85 +0x27 fp=0xc01235b0a0 sp=0xc01235b070 pc=0xe32ec7
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object.(*TransportSplitter).Get(0xc005cf5920, 0x1?, {0x1277848?, 0xc004e630e0})
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/transport_splitter.go:83 +0xdb fp=0xc01235b0e8 sp=0xc01235b0a0 pc=0xde07fb
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/acl/v2.Service.Get({0xc00edfeae0, {0xc0003e8200, {0x1270880, 0xc005cf5960}, {0x127b418, 0xc007016c90}}}, 0xc4167fa6>
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/acl/v2/service.go:160 +0x676 fp=0xc01235b788 sp=0xc01235b0e8 pc=0xde91d6
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/acl/v2.(*Service).Get(0xc0b86c9838?, 0x40b936?, {0x12776e0?, 0xc4167fa7c8?})
ноя 28 11:47:47 az neofs-node[1360]: <autogenerated>:1 +0x8d fp=0xc01235b810 sp=0xc01235b788 pc=0xdef62d
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object.(*Common).Get(0xc005cf5980, 0xc4167fa798?, {0x12776e0, 0xc4167fa7c8})
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/common.go:42 +0x8d fp=0xc01235b848 sp=0xc01235b810 pc=0xddc30d
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object.(*ResponseService).Get(0xc0060ff698, 0xc4917c3160?, {0x1277708?, 0xc4167fa780})
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/response.go:54 +0x115 fp=0xc01235b898 sp=0xc01235b848 pc=0xdde115
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object.(*SignService).Get.func3(0xc4917c3160)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/sign.go:65 +0xce fp=0xc01235b8f0 sp=0xc01235b898 pc=0xddf22e
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/util.(*SignService).HandleServerStreamRequest(0xc001119fc8, {0x10455c0?, 0xc4167fa6a8}, 0xc4167fa768, 0x1137788, 0xc0b86c9>
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/util/sign.go:149 +0x132 fp=0xc01235b968 sp=0xc01235b8f0 pc=0xa41c32
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object.(*SignService).Get(0xc005cf59a0, 0xc4167fa6a8, {0x1277690?, 0xc15fd37800})
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/sign.go:57 +0xe6 fp=0xc01235b9d8 sp=0xc01235b968 pc=0xddf106
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object.MetricCollector.Get({{0x12810c0?, 0xc005cf59a0?}, {0x12851f0?, 0xc0000001e0?}}, 0x10?, {0x1277190?, 0xc0fc1504c0})
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object/metrics.go:65 +0x1f8 fp=0xc01235baa0 sp=0xc01235b9d8 pc=0xddca78
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/object.(*MetricCollector).Get(0xc4167fa6a8?, 0x10590c0?, {0x1277190?, 0xc0fc1504c0?})
ноя 28 11:47:47 az neofs-node[1360]: <autogenerated>:1 +0x55 fp=0xc01235bae8 sp=0xc01235baa0 pc=0xde2435
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/network/transport/object/grpc.(*Server).Get(0xc00980ddb0, 0x10590c0?, {0x1280da0?, 0xc0fc150470})
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/network/transport/object/grpc/get.go:26 +0xb4 fp=0xc01235bb30 sp=0xc01235bae8 pc=0xde3074
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-api-go/v2/object/grpc._ObjectService_Get_Handler({0x100a180?, 0xc00980ddb0}, {0x127fe08, 0xc01711a4e0})
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-api-go/v2@v2.14.0/object/grpc/service_grpc.pb.go:640 +0xd0 fp=0xc01235bb70 sp=0xc01235bb30 pc=0xa117d0
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc.(*Server).processStreamingRPC(0xc0000014a0, {0x12823e0, 0xc049725860}, 0xc243761d40, 0xc005617b60, 0x1a82fa0, 0x0)
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/server.go:1558 +0xf46 fp=0xc01235be48 sp=0xc01235bb70 pc=0x8af106
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc.(*Server).handleStream(0xc0000014a0, {0x12823e0, 0xc049725860}, 0xc243761d40, 0x0)
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/server.go:1640 +0x9d6 fp=0xc01235bf68 sp=0xc01235be48 pc=0x8b0936
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc.(*Server).serveStreams.func1.2()
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/server.go:932 +0x98 fp=0xc01235bfe0 sp=0xc01235bf68 pc=0x8a9bb8
ноя 28 11:47:47 az neofs-node[1360]: runtime.goexit()
ноя 28 11:47:47 az neofs-node[1360]: runtime/asm_amd64.s:1571 +0x1 fp=0xc01235bfe8 sp=0xc01235bfe0 pc=0x46b921
ноя 28 11:47:47 az neofs-node[1360]: created by google.golang.org/grpc.(*Server).serveStreams.func1
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/server.go:930 +0x28a
ноя 28 11:47:47 az neofs-node[1360]: goroutine 1 [select, 195 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: main.wait(0xc000101400)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/cmd/neofs-node/main.go:131 +0x195
ноя 28 11:47:47 az neofs-node[1360]: main.main()
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/cmd/neofs-node/main.go:68 +0x331
ноя 28 11:47:47 az neofs-node[1360]: goroutine 6 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001ac500)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 82 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001ac580)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 83 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001ac600)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 84 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001ac680)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 85 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001acd00)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 86 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001ace00)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 117 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001ad080)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 118 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001ad200)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 119 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001ad300)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 120 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001ad480)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 121 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001ad580)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 122 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001ad680)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 123 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001ad780)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 124 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001ad880)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 125 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001ad980)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 126 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001ada80)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 127 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001adb80)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 128 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc0001adc80)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 130 [syscall, 198 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: os/signal.signal_recv()
ноя 28 11:47:47 az neofs-node[1360]: runtime/sigqueue.go:151 +0x2f
ноя 28 11:47:47 az neofs-node[1360]: os/signal.loop()
ноя 28 11:47:47 az neofs-node[1360]: os/signal/signal_unix.go:23 +0x19
ноя 28 11:47:47 az neofs-node[1360]: created by os/signal.Notify.func1.1
ноя 28 11:47:47 az neofs-node[1360]: os/signal/signal.go:151 +0x2a
ноя 28 11:47:47 az neofs-node[1360]: goroutine 131 [select, 198 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: os/signal.NotifyContext.func1()
ноя 28 11:47:47 az neofs-node[1360]: os/signal/signal.go:288 +0x70
ноя 28 11:47:47 az neofs-node[1360]: created by os/signal.NotifyContext
ноя 28 11:47:47 az neofs-node[1360]: os/signal/signal.go:287 +0x165
ноя 28 11:47:47 az neofs-node[1360]: goroutine 194 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc000301b80, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 260 [select, 3 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0067580a0, 0x1)
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/internal/transport/controlbuf.go:408 +0x115
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc006c88d80)
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/internal/transport/controlbuf.go:535 +0x85
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/internal/transport/http2_client.go:415 +0x65
ноя 28 11:47:47 az neofs-node[1360]: created by google.golang.org/grpc/internal/transport.newHTTP2Client
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/internal/transport/http2_client.go:413 +0x1eb1
ноя 28 11:47:47 az neofs-node[1360]: goroutine 6229 [select]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/tree.(*Service).replicationWorker(0xc0001fe690)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/tree/replicator.go:36 +0xc6
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/services/tree.(*Service).replicateLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/tree/replicator.go:76 +0x5e
ноя 28 11:47:47 az neofs-node[1360]: goroutine 76 [select]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard.(*gc).tickRemover(0xc005b9ab40)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/gc.go:160 +0x11a
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard.(*gc).init
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/gc.go:108 +0x13c
ноя 28 11:47:47 az neofs-node[1360]: goroutine 45856964 [IO wait, 51 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: internal/poll.runtime_pollWait(0x7f6dd3695510, 0x72)
ноя 28 11:47:47 az neofs-node[1360]: runtime/netpoll.go:302 +0x89
ноя 28 11:47:47 az neofs-node[1360]: internal/poll.(*pollDesc).wait(0xc17b0d2c80?, 0xc00dcfa000?, 0x0)
ноя 28 11:47:47 az neofs-node[1360]: internal/poll/fd_poll_runtime.go:83 +0x32
ноя 28 11:47:47 az neofs-node[1360]: internal/poll.(*pollDesc).waitRead(...)
ноя 28 11:47:47 az neofs-node[1360]: internal/poll/fd_poll_runtime.go:88
ноя 28 11:47:47 az neofs-node[1360]: internal/poll.(*FD).Read(0xc17b0d2c80, {0xc00dcfa000, 0x8000, 0x8000})
ноя 28 11:47:47 az neofs-node[1360]: internal/poll/fd_unix.go:167 +0x25a
ноя 28 11:47:47 az neofs-node[1360]: net.(*netFD).Read(0xc17b0d2c80, {0xc00dcfa000?, 0x46988e?, 0x43b787?})
ноя 28 11:47:47 az neofs-node[1360]: net/fd_posix.go:55 +0x29
ноя 28 11:47:47 az neofs-node[1360]: net.(*conn).Read(0xc1eeb805c0, {0xc00dcfa000?, 0x0?, 0x7f6ef3f4ec28?})
ноя 28 11:47:47 az neofs-node[1360]: net/net.go:183 +0x45
ноя 28 11:47:47 az neofs-node[1360]: bufio.(*Reader).Read(0xc3caa7ade0, {0xc0be991000, 0x9, 0x18?})
ноя 28 11:47:47 az neofs-node[1360]: bufio/bufio.go:236 +0x1b4
ноя 28 11:47:47 az neofs-node[1360]: io.ReadAtLeast({0x126b400, 0xc3caa7ade0}, {0xc0be991000, 0x9, 0x9}, 0x9)
ноя 28 11:47:47 az neofs-node[1360]: io/io.go:331 +0x9a
ноя 28 11:47:47 az neofs-node[1360]: io.ReadFull(...)
ноя 28 11:47:47 az neofs-node[1360]: io/io.go:350
ноя 28 11:47:47 az neofs-node[1360]: golang.org/x/net/http2.readFrameHeader({0xc0be991000?, 0x9?, 0xc0ae5f06c0?}, {0x126b400?, 0xc3caa7ade0?})
ноя 28 11:47:47 az neofs-node[1360]: golang.org/x/net@v0.0.0-20220722155237-a158d28d115b/http2/frame.go:237 +0x6e
ноя 28 11:47:47 az neofs-node[1360]: golang.org/x/net/http2.(*Framer).ReadFrame(0xc0be990fc0)
ноя 28 11:47:47 az neofs-node[1360]: golang.org/x/net@v0.0.0-20220722155237-a158d28d115b/http2/frame.go:498 +0x95
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc248c211e0, 0x12700a0?, 0xc1582149ac?)
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/internal/transport/http2_server.go:614 +0xb2
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc.(*Server).serveStreams(0xc000001a40, {0x12823e0?, 0xc248c211e0})
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/server.go:916 +0x142
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc.(*Server).handleRawConn.func1()
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/server.go:858 +0x46
ноя 28 11:47:47 az neofs-node[1360]: created by google.golang.org/grpc.(*Server).handleRawConn
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/server.go:857 +0x185
ноя 28 11:47:47 az neofs-node[1360]: goroutine 30209149 [select, 61 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc039ce0000)
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/internal/transport/http2_server.go:1132 +0x233
ноя 28 11:47:47 az neofs-node[1360]: created by google.golang.org/grpc/internal/transport.NewServerTransport
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/internal/transport/http2_server.go:340 +0x1918
ноя 28 11:47:47 az neofs-node[1360]: goroutine 75 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc004bda580)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 7121 [IO wait]:
ноя 28 11:47:47 az neofs-node[1360]: internal/poll.runtime_pollWait(0x7f6ef403b2e8, 0x72)
ноя 28 11:47:47 az neofs-node[1360]: runtime/netpoll.go:302 +0x89
ноя 28 11:47:47 az neofs-node[1360]: internal/poll.(*pollDesc).wait(0xc0001e6d80?, 0xc000e70000?, 0x0)
ноя 28 11:47:47 az neofs-node[1360]: internal/poll/fd_poll_runtime.go:83 +0x32
ноя 28 11:47:47 az neofs-node[1360]: internal/poll.(*pollDesc).waitRead(...)
ноя 28 11:47:47 az neofs-node[1360]: internal/poll/fd_poll_runtime.go:88
ноя 28 11:47:47 az neofs-node[1360]: internal/poll.(*FD).Read(0xc0001e6d80, {0xc000e70000, 0x1000, 0x1000})
ноя 28 11:47:47 az neofs-node[1360]: internal/poll/fd_unix.go:167 +0x25a
ноя 28 11:47:47 az neofs-node[1360]: net.(*netFD).Read(0xc0001e6d80, {0xc000e70000?, 0xd62f85?, 0x0?})
ноя 28 11:47:47 az neofs-node[1360]: net/fd_posix.go:55 +0x29
ноя 28 11:47:47 az neofs-node[1360]: net.(*conn).Read(0xc012b17b78, {0xc000e70000?, 0x41?, 0x0?})
ноя 28 11:47:47 az neofs-node[1360]: net/net.go:183 +0x45
ноя 28 11:47:47 az neofs-node[1360]: net/http.(*connReader).Read(0xc01073bda0, {0xc000e70000, 0x1000, 0x1000})
ноя 28 11:47:47 az neofs-node[1360]: net/http/server.go:780 +0x16d
ноя 28 11:47:47 az neofs-node[1360]: bufio.(*Reader).fill(0xc01074e6c0)
ноя 28 11:47:47 az neofs-node[1360]: bufio/bufio.go:106 +0x103
ноя 28 11:47:47 az neofs-node[1360]: bufio.(*Reader).ReadSlice(0xc01074e6c0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: bufio/bufio.go:371 +0x2f
ноя 28 11:47:47 az neofs-node[1360]: bufio.(*Reader).ReadLine(0xc01074e6c0)
ноя 28 11:47:47 az neofs-node[1360]: bufio/bufio.go:400 +0x27
ноя 28 11:47:47 az neofs-node[1360]: net/textproto.(*Reader).readLineSlice(0xc330365620)
ноя 28 11:47:47 az neofs-node[1360]: net/textproto/reader.go:57 +0x99
ноя 28 11:47:47 az neofs-node[1360]: net/textproto.(*Reader).ReadLine(...)
ноя 28 11:47:47 az neofs-node[1360]: net/textproto/reader.go:38
ноя 28 11:47:47 az neofs-node[1360]: net/http.readRequest(0xc012b17b78?)
ноя 28 11:47:47 az neofs-node[1360]: net/http/request.go:1029 +0x79
ноя 28 11:47:47 az neofs-node[1360]: net/http.(*conn).readRequest(0xc01072f0e0, {0x127c0b8, 0xc01074b4c0})
ноя 28 11:47:47 az neofs-node[1360]: net/http/server.go:988 +0x24a
ноя 28 11:47:47 az neofs-node[1360]: net/http.(*conn).serve(0xc01072f0e0, {0x127c160, 0xc00729ab40})
ноя 28 11:47:47 az neofs-node[1360]: net/http/server.go:1891 +0x32b
ноя 28 11:47:47 az neofs-node[1360]: created by net/http.(*Server).Serve
ноя 28 11:47:47 az neofs-node[1360]: net/http/server.go:3071 +0x4db
ноя 28 11:47:47 az neofs-node[1360]: goroutine 6196 [select]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/policer.(*Policer).poolCapacityWorker(0xc0060ff650, {0x127ceb8, 0xc000368140})
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/policer/process.go:81 +0xa9
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/services/policer.(*Policer).Run
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/services/policer/process.go:18 +0xcb
ноя 28 11:47:47 az neofs-node[1360]: goroutine 2753 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc000300f20, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 178 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0xf13edda5a40bfdac?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 180 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 290 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 5809 [select]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/morph/client.(*Client).notificationLoop(0xc0002cac80)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/morph/client/multi.go:86 +0x12d
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/morph/client.New
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/morph/client/constructor.go:148 +0x6c5
ноя 28 11:47:47 az neofs-node[1360]: goroutine 292 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 289 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 291 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 179 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 226 [chan receive]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2.(*Pool).periodicallyPurge(0xc002bf7700)
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:72 +0x8b
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/panjf2000/ants/v2.NewPool
ноя 28 11:47:47 az neofs-node[1360]: github.com/panjf2000/ants/v2@v2.4.0/pool.go:135 +0x305
ноя 28 11:47:47 az neofs-node[1360]: goroutine 2754 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc000300f20, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 181 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 8585 [IO wait, 1 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: internal/poll.runtime_pollWait(0x7f6ef403aa78, 0x72)
ноя 28 11:47:47 az neofs-node[1360]: runtime/netpoll.go:302 +0x89
ноя 28 11:47:47 az neofs-node[1360]: internal/poll.(*pollDesc).wait(0xc006ba9200?, 0xc004ad4000?, 0x0)
ноя 28 11:47:47 az neofs-node[1360]: internal/poll/fd_poll_runtime.go:83 +0x32
ноя 28 11:47:47 az neofs-node[1360]: internal/poll.(*pollDesc).waitRead(...)
ноя 28 11:47:47 az neofs-node[1360]: internal/poll/fd_poll_runtime.go:88
ноя 28 11:47:47 az neofs-node[1360]: internal/poll.(*FD).Read(0xc006ba9200, {0xc004ad4000, 0x8000, 0x8000})
ноя 28 11:47:47 az neofs-node[1360]: internal/poll/fd_unix.go:167 +0x25a
ноя 28 11:47:47 az neofs-node[1360]: net.(*netFD).Read(0xc006ba9200, {0xc004ad4000?, 0x46988e?, 0x43b787?})
ноя 28 11:47:47 az neofs-node[1360]: net/fd_posix.go:55 +0x29
ноя 28 11:47:47 az neofs-node[1360]: net.(*conn).Read(0xc013adebf0, {0xc004ad4000?, 0x0?, 0x7f6dc6b43758?})
ноя 28 11:47:47 az neofs-node[1360]: net/net.go:183 +0x45
ноя 28 11:47:47 az neofs-node[1360]: bufio.(*Reader).Read(0xc018503e00, {0xc011ea4580, 0x9, 0x18?})
ноя 28 11:47:47 az neofs-node[1360]: bufio/bufio.go:236 +0x1b4
ноя 28 11:47:47 az neofs-node[1360]: io.ReadAtLeast({0x126b400, 0xc018503e00}, {0xc011ea4580, 0x9, 0x9}, 0x9)
ноя 28 11:47:47 az neofs-node[1360]: io/io.go:331 +0x9a
ноя 28 11:47:47 az neofs-node[1360]: io.ReadFull(...)
ноя 28 11:47:47 az neofs-node[1360]: io/io.go:350
ноя 28 11:47:47 az neofs-node[1360]: golang.org/x/net/http2.readFrameHeader({0xc011ea4580?, 0x9?, 0xc37b5137b8?}, {0x126b400?, 0xc018503e00?})
ноя 28 11:47:47 az neofs-node[1360]: golang.org/x/net@v0.0.0-20220722155237-a158d28d115b/http2/frame.go:237 +0x6e
ноя 28 11:47:47 az neofs-node[1360]: golang.org/x/net/http2.(*Framer).ReadFrame(0xc011ea4540)
ноя 28 11:47:47 az neofs-node[1360]: golang.org/x/net@v0.0.0-20220722155237-a158d28d115b/http2/frame.go:498 +0x95
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc013cf1040, 0x127c0b8?, 0xc000d50900?)
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/internal/transport/http2_server.go:614 +0xb2
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc.(*Server).serveStreams(0xc000001a40, {0x12823e0?, 0xc013cf1040})
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/server.go:916 +0x142
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc.(*Server).handleRawConn.func1()
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/server.go:858 +0x46
ноя 28 11:47:47 az neofs-node[1360]: created by google.golang.org/grpc.(*Server).handleRawConn
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/server.go:857 +0x185
ноя 28 11:47:47 az neofs-node[1360]: goroutine 5803 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).setModeLoop(0xc000324e70)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/engine.go:58 +0x10c
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).Init
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/control.go:96 +0x67c
ноя 28 11:47:47 az neofs-node[1360]: goroutine 2537 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc000301080, 0x83b96bb41fcb8d52?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 4891 [IO wait]:
ноя 28 11:47:47 az neofs-node[1360]: internal/poll.runtime_pollWait(0x7f6ef40e1958, 0x72)
ноя 28 11:47:47 az neofs-node[1360]: runtime/netpoll.go:302 +0x89
ноя 28 11:47:47 az neofs-node[1360]: internal/poll.(*pollDesc).wait(0xc004ac6100?, 0xc0084ca000?, 0x0)
ноя 28 11:47:47 az neofs-node[1360]: internal/poll/fd_poll_runtime.go:83 +0x32
ноя 28 11:47:47 az neofs-node[1360]: internal/poll.(*pollDesc).waitRead(...)
ноя 28 11:47:47 az neofs-node[1360]: internal/poll/fd_poll_runtime.go:88
ноя 28 11:47:47 az neofs-node[1360]: internal/poll.(*FD).Read(0xc004ac6100, {0xc0084ca000, 0x8000, 0x8000})
ноя 28 11:47:47 az neofs-node[1360]: internal/poll/fd_unix.go:167 +0x25a
ноя 28 11:47:47 az neofs-node[1360]: net.(*netFD).Read(0xc004ac6100, {0xc0084ca000?, 0x46988e?, 0x43b787?})
ноя 28 11:47:47 az neofs-node[1360]: net/fd_posix.go:55 +0x29
ноя 28 11:47:47 az neofs-node[1360]: net.(*conn).Read(0xc00002a078, {0xc0084ca000?, 0x0?, 0x7f6dc7420f38?})
ноя 28 11:47:47 az neofs-node[1360]: net/net.go:183 +0x45
ноя 28 11:47:47 az neofs-node[1360]: bufio.(*Reader).Read(0xc002ac72c0, {0xc0008aa580, 0x9, 0x18?})
ноя 28 11:47:47 az neofs-node[1360]: bufio/bufio.go:236 +0x1b4
ноя 28 11:47:47 az neofs-node[1360]: io.ReadAtLeast({0x126b400, 0xc002ac72c0}, {0xc0008aa580, 0x9, 0x9}, 0x9)
ноя 28 11:47:47 az neofs-node[1360]: io/io.go:331 +0x9a
ноя 28 11:47:47 az neofs-node[1360]: io.ReadFull(...)
ноя 28 11:47:47 az neofs-node[1360]: io/io.go:350
ноя 28 11:47:47 az neofs-node[1360]: golang.org/x/net/http2.readFrameHeader({0xc0008aa580?, 0x9?, 0xc36bec2d08?}, {0x126b400?, 0xc002ac72c0?})
ноя 28 11:47:47 az neofs-node[1360]: golang.org/x/net@v0.0.0-20220722155237-a158d28d115b/http2/frame.go:237 +0x6e
ноя 28 11:47:47 az neofs-node[1360]: golang.org/x/net/http2.(*Framer).ReadFrame(0xc0008aa540)
ноя 28 11:47:47 az neofs-node[1360]: golang.org/x/net@v0.0.0-20220722155237-a158d28d115b/http2/frame.go:498 +0x95
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc007bf6000, 0xc0035267b8?, 0xc67845?)
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/internal/transport/http2_server.go:614 +0xb2
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc.(*Server).serveStreams(0xc000001680, {0x12823e0?, 0xc007bf6000})
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/server.go:916 +0x142
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc.(*Server).handleRawConn.func1()
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/server.go:858 +0x46
ноя 28 11:47:47 az neofs-node[1360]: created by google.golang.org/grpc.(*Server).handleRawConn
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/server.go:857 +0x185
ноя 28 11:47:47 az neofs-node[1360]: goroutine 5348 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc000301340, 0xdfcfcb1727ad88d4?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 5839 [select]:
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc005c99db0, 0x1)
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/internal/transport/controlbuf.go:408 +0x115
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc004cf4a80)
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/internal/transport/controlbuf.go:535 +0x85
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc/internal/transport.NewServerTransport.func2()
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/internal/transport/http2_server.go:331 +0xc6
ноя 28 11:47:47 az neofs-node[1360]: created by google.golang.org/grpc/internal/transport.NewServerTransport
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/internal/transport/http2_server.go:328 +0x18d3
ноя 28 11:47:47 az neofs-node[1360]: goroutine 78 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc000300dc0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 3968 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc000301760, 0x190168dc31f1105e?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 261 [select, 195 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc.newClientStreamWithParams.func4()
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/stream.go:355 +0x92
ноя 28 11:47:47 az neofs-node[1360]: created by google.golang.org/grpc.newClientStreamWithParams
ноя 28 11:47:47 az neofs-node[1360]: google.golang.org/grpc@v1.48.0/stream.go:354 +0xbca
ноя 28 11:47:47 az neofs-node[1360]: goroutine 293 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 294 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 295 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 296 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 297 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 298 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 299 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 300 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0xc00009e000?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 301 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 302 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0x0?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:35 +0x30
ноя 28 11:47:47 az neofs-node[1360]: goroutine 303 [select, 197 minutes]:
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).flushWorker(0xc0003011e0, 0xaba22c2a599a9628?)
ноя 28 11:47:47 az neofs-node[1360]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache/flush.go:227 +0xf7
ноя 28 11:47:47 az neofs-node[1360]: created by github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache.(*cache).runFlushLoop
...