-
Notifications
You must be signed in to change notification settings - Fork 316
Closed
Description
Detected an issue with concurrent map operations to the unsavedFastNodeAdditions & unsavedFastNodeRemovals maps on the UnsavedFastIterator and MutableTree structs.
I tried adding a mutex and use the lock when these operations happen and seems to solve the issue.
More info here cosmos/cosmos-sdk#10568 (comment)
Error with the stack trace:
�[90m12:54PM�[0m �[32mINF�[0m executed block �[36mheight=�[0m9522889 �[36mmodule=�[0mstate �[36mnum_invalid_txs=�[0m0 �[36mnum_valid_txs=�[0m3 �[36mserver=�[0mnode
fatal error: concurrent map iteration and map write
goroutine 963386 [running]:
runtime.throw({0x27ef174?, 0x8?})
runtime/panic.go:992 +0x71 fp=0xc0a2e49fb0 sp=0xc0a2e49f80 pc=0x43f0b1
runtime.mapiternext(0xc0a2e4a0a0?)
runtime/map.go:871 +0x4eb fp=0xc0a2e4a020 sp=0xc0a2e49fb0 pc=0x41698b
github.com/cosmos/iavl.NewUnsavedFastIterator({0xc09c5e3520, 0x1e, 0x20}, {0xc09c5e3540, 0x1e, 0x20}, 0x1, 0xc0012f2880, 0xc07132da40, 0xc07132da70)
github.com/cosmos/iavl@v0.19.4/unsaved_fast_iterator.go:55 +0x236 fp=0xc0a2e4a110 sp=0xc0a2e4a020 pc=0xe6f2d6
github.com/cosmos/iavl.(*MutableTree).Iterator(0xc0018040f0, {0xc09c5e3520, 0x1e, 0x20}, {0xc09c5e3540, 0x1e, 0x20}, 0xe0?)
github.com/cosmos/iavl@v0.19.4/mutable_tree.go:216 +0x10c fp=0xc0a2e4a178 sp=0xc0a2e4a110 pc=0xe55f2c
github.com/cosmos/cosmos-sdk/store/iavl.(*Store).Iterator(0xc0705fe500?, {0xc09c5e3520?, 0x20?, 0xc0705f6720?}, {0xc09c5e3540?, 0x18?, 0xc09d8b26e0?})
github.com/cosmos/cosmos-sdk@v0.46.7/store/iavl/store.go:244 +0x31 fp=0xc0a2e4a1c8 sp=0xc0a2e4a178 pc=0xe902d1
github.com/cosmos/cosmos-sdk/store/cache.(*CommitKVStoreCache).Iterator(0xc05efd0e94?, {0xc09c5e3520?, 0xc0452f3d40?, 0x195?}, {0xc09c5e3540?, 0x343ff98?, 0xc0019ca7e0?})
<autogenerated>:1 +0x34 fp=0xc0a2e4a210 sp=0xc0a2e4a1c8 pc=0x1104b34
github.com/cosmos/cosmos-sdk/store/cachekv.(*Store).iterator(0xc07134b0c0, {0xc09c5e3520, 0x1e, 0x20}, {0xc09c5e3540, 0x1e, 0x20}, 0x1)
github.com/cosmos/cosmos-sdk@v0.46.7/store/cachekv/store.go:180 +0x126 fp=0xc0a2e4a2b0 sp=0xc0a2e4a210 pc=0xe44246
github.com/cosmos/cosmos-sdk/store/cachekv.(*Store).Iterator(0x64?, {0xc09c5e3520?, 0xc8?, 0x3?}, {0xc09c5e3540?, 0x343e980?, 0xc0000600c0?})
github.com/cosmos/cosmos-sdk@v0.46.7/store/cachekv/store.go:165 +0x29 fp=0xc0a2e4a300 sp=0xc0a2e4a2b0 pc=0xe44029
github.com/cosmos/cosmos-sdk/store/cachekv.(*Store).iterator(0xc0a2b20880, {0xc09c5e3520, 0x1e, 0x20}, {0xc09c5e3540, 0x1e, 0x20}, 0x1)
github.com/cosmos/cosmos-sdk@v0.46.7/store/cachekv/store.go:180 +0x126 fp=0xc0a2e4a3a0 sp=0xc0a2e4a300 pc=0xe44246
github.com/cosmos/cosmos-sdk/store/cachekv.(*Store).Iterator(0xc0705fe460?, {0xc09c5e3520?, 0x20?, 0xc0705fe480?}, {0xc09c5e3540?, 0x20?, 0xc0705fe4a0?})
github.com/cosmos/cosmos-sdk@v0.46.7/store/cachekv/store.go:165 +0x29 fp=0xc0a2e4a3f0 sp=0xc0a2e4a3a0 pc=0xe44029
github.com/cosmos/cosmos-sdk/store/cachekv.(*Store).iterator(0xc0a2b214c0, {0xc09c5e3520, 0x1e, 0x20}, {0xc09c5e3540, 0x1e, 0x20}, 0x1)
github.com/cosmos/cosmos-sdk@v0.46.7/store/cachekv/store.go:180 +0x126 fp=0xc0a2e4a490 sp=0xc0a2e4a3f0 pc=0xe44246
github.com/cosmos/cosmos-sdk/store/cachekv.(*Store).Iterator(0xc1507b9800?, {0xc09c5e3520?, 0x0?, 0xc09cd6ae61?}, {0xc09c5e3540?, 0x456ef4?, 0x20?})
github.com/cosmos/cosmos-sdk@v0.46.7/store/cachekv/store.go:165 +0x29 fp=0xc0a2e4a4e0 sp=0xc0a2e4a490 pc=0xe44029
github.com/cosmos/cosmos-sdk/store/gaskv.(*Store).iterator(0xc09d7b5560, {0xc09c5e3520?, 0x64?, 0x15?}, {0xc09c5e3540?, 0x20?, 0xc94e910000000000?}, 0x40?)
github.com/cosmos/cosmos-sdk@v0.46.7/store/gaskv/store.go:98 +0x56 fp=0xc0a2e4a588 sp=0xc0a2e4a4e0 pc=0xca7d56
github.com/cosmos/cosmos-sdk/store/gaskv.(*Store).Iterator(0xc09d7bd530?, {0xc09c5e3520?, 0x344e8f0?, 0xc0a2b21880?}, {0xc09c5e3540?, 0x0?, 0xc05efd0e94?})
github.com/cosmos/cosmos-sdk@v0.46.7/store/gaskv/store.go:74 +0x29 fp=0xc0a2e4a5d8 sp=0xc0a2e4a588 pc=0xca7b89
github.com/cosmos/cosmos-sdk/x/distribution/keeper.Keeper.IterateValidatorSlashEventsBetween({{0x34236a8, 0xc00298c550}, {0x344e0a0, 0xc0012237b0}, {{0x344e0a0, 0xc0012237b0}, 0xc00011d1e0, {0x34236a8, 0xc00298c580}, {0x34236f8, ...}, ...}, ...}, ...)
github.com/cosmos/cosmos-sdk@v0.46.7/x/distribution/keeper/store.go:337 +0x156 fp=0xc0a2e4a908 sp=0xc0a2e4a5d8 pc=0x16ada36
github.com/cosmos/cosmos-sdk/x/distribution/keeper.Keeper.CalculateDelegationRewards({{0x34236a8, 0xc00298c550}, {0x344e0a0, 0xc0012237b0}, {{0x344e0a0, 0xc0012237b0}, 0xc00011d1e0, {0x34236a8, 0xc00298c580}, {0x34236f8, ...}, ...}, ...}, ...)
github.com/cosmos/cosmos-sdk@v0.46.7/x/distribution/keeper/delegation.go:80 +0x2d9 fp=0xc0a2e4b308 sp=0xc0a2e4a908 pc=0x16969f9
github.com/cosmos/cosmos-sdk/x/distribution/keeper.Keeper.withdrawDelegationRewards({{0x34236a8, 0xc00298c550}, {0x344e0a0, 0xc0012237b0}, {{0x344e0a0, 0xc0012237b0}, 0xc00011d1e0, {0x34236a8, 0xc00298c580}, {0x34236f8, ...}, ...}, ...}, ...)
github.com/cosmos/cosmos-sdk@v0.46.7/x/distribution/keeper/delegation.go:147 +0x1e5 fp=0xc0a2e4c170 sp=0xc0a2e4b308 pc=0x16973c5
github.com/cosmos/cosmos-sdk/x/distribution/keeper.Keeper.WithdrawDelegationRewards({{0x34236a8, 0xc00298c550}, {0x344e0a0, 0xc0012237b0}, {{0x344e0a0, 0xc0012237b0}, 0xc00011d1e0, {0x34236a8, 0xc00298c580}, {0x34236f8, ...}, ...}, ...}, ...)
github.com/cosmos/cosmos-sdk@v0.46.7/x/distribution/keeper/keeper.go:94 +0x198 fp=0xc0a2e4c548 sp=0xc0a2e4c170 pc=0x16a2638
github.com/cosmos/cosmos-sdk/x/distribution/keeper.msgServer.WithdrawDelegatorReward({{{0x34236a8, 0xc00298c550}, {0x344e0a0, 0xc0012237b0}, {{0x344e0a0, 0xc0012237b0}, 0xc00011d1e0, {0x34236a8, 0xc00298c580}, {0x34236f8, ...}, ...}, ...}}, ...)
github.com/cosmos/cosmos-sdk@v0.46.7/x/distribution/keeper/msg_server.go:63 +0x335 fp=0xc0a2e4da40 sp=0xc0a2e4c548 pc=0x16a4295
github.com/cosmos/cosmos-sdk/x/distribution/keeper.(*msgServer).WithdrawDelegatorReward(0x41196b?, {0x343e9f0?, 0xc0a2b39fb0?}, 0x2776600?)
<autogenerated>:1 +0xc5 fp=0xc0a2e4dbd8 sp=0xc0a2e4da40 pc=0x16b5bc5
github.com/cosmos/cosmos-sdk/x/distribution/types._Msg_WithdrawDelegatorReward_Handler.func1({0x343e9f0, 0xc0a2b39fb0}, {0x26db920?, 0xc0a2b1d920})
github.com/cosmos/cosmos-sdk@v0.46.7/x/distribution/types/tx.pb.go:657 +0x78 fp=0xc0a2e4dc18 sp=0xc0a2e4dbd8 pc=0x166b498
github.com/cosmos/cosmos-sdk/baseapp.(*MsgServiceRouter).RegisterService.func2.1({0x343ffd0, 0xc0a2b50580}, {0xc0a2e4df50?, 0x41196b?}, 0x2776600?, 0xc0a2b134b8)
github.com/cosmos/cosmos-sdk@v0.46.7/baseapp/msg_service_router.go:113 +0xd2 fp=0xc0a2e4df08 sp=0xc0a2e4dc18 pc=0x11183b2
github.com/cosmos/cosmos-sdk/x/distribution/types._Msg_WithdrawDelegatorReward_Handler({0x2777080?, 0xc000017ec0}, {0x343ffd0, 0xc0a2b50580}, 0x3013700, 0xc0a2b54720)
github.com/cosmos/cosmos-sdk@v0.46.7/x/distribution/types/tx.pb.go:659 +0x138 fp=0xc0a2e4df60 sp=0xc0a2e4df08 pc=0x166b358
github.com/cosmos/cosmos-sdk/baseapp.(*MsgServiceRouter).RegisterService.func2({{0x343e980, 0xc0000600c0}, {0x344e8f0, 0xc0a2b21880}, {{0xb, 0x0}, {0xc05efd0e94, 0xc}, 0x914ec8, {0x3c658c6, ...}, ...}, ...}, ...)
github.com/cosmos/cosmos-sdk@v0.46.7/baseapp/msg_service_router.go:126 +0x344 fp=0xc0a2e4ed18 sp=0xc0a2e4df60 pc=0x1118124
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).runMsgs(_, {{0x343e980, 0xc0000600c0}, {0x344e8f0, 0xc0a2b21880}, {{0xb, 0x0}, {0xc05efd0e94, 0xc}, 0x914ec8, ...}, ...}, ...)
github.com/cosmos/cosmos-sdk@v0.46.7/baseapp/baseapp.go:779 +0x625 fp=0xc0a2e4f3c8 sp=0xc0a2e4ed18 pc=0x11155e5
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).runTx(0xc001854380, 0x2, {0xc0452f3d40, 0x195, 0x1a0})
github.com/cosmos/cosmos-sdk@v0.46.7/baseapp/baseapp.go:719 +0xc45 fp=0xc0a2e527f8 sp=0xc0a2e4f3c8 pc=0x1114385
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).Simulate(0x4142e7?, {0xc0452f3d40?, 0x2543ec0?, 0x1?})
github.com/cosmos/cosmos-sdk@v0.46.7/baseapp/test_helpers.go:25 +0x2c fp=0xc0a2e52830 sp=0xc0a2e527f8 pc=0x111abcc
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).Simulate-fm({0xc0452f3d40?, 0x20?, 0xc09c7b9600?})
<autogenerated>:1 +0x37 fp=0xc0a2e52860 sp=0xc0a2e52830 pc=0x1c15957
github.com/cosmos/cosmos-sdk/x/auth/tx.txServer.Simulate({{{0x0, 0x0, 0x0}, {0x345a070, 0xc18b37c280}, 0x0, {0xc000152360, 0xc}, {0x3453450, 0xc0012237b0}, ...}, ...}, ...)
github.com/cosmos/cosmos-sdk@v0.46.7/x/auth/tx/service.go:131 +0x12e fp=0xc0a2e52910 sp=0xc0a2e52860 pc=0x1186b2e
github.com/cosmos/cosmos-sdk/x/auth/tx.(*txServer).Simulate(0x7?, {0x343e9f0?, 0xc0a2b1e780?}, 0x343e9f0?)
<autogenerated>:1 +0xa5 fp=0xc0a2e52cb8 sp=0xc0a2e52910 pc=0x118a205
github.com/cosmos/cosmos-sdk/types/tx._Service_Simulate_Handler.func1({0x343e9f0, 0xc0a2b1e780}, {0x2698920?, 0xc09ced3a00})
github.com/cosmos/cosmos-sdk@v0.46.7/types/tx/service.pb.go:933 +0x78 fp=0xc0a2e52cf8 sp=0xc0a2e52cb8 pc=0xe1f838
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).RegisterGRPCServer.func1({0x343e9f0, 0xc09d36ac60}, {0x2698920, 0xc09ced3a00}, 0xc09ced3a20?, 0xc09d34c9d8)
github.com/cosmos/cosmos-sdk@v0.46.7/baseapp/grpcserver.go:68 +0x3e2 fp=0xc0a2e53808 sp=0xc0a2e52cf8 pc=0x1117642
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x343e9f0?, 0xc09d36ac60?}, {0x2698920?, 0xc09ced3a00?})
github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/chain.go:25 +0x3a fp=0xc0a2e53848 sp=0xc0a2e53808 pc=0x1109e5a
github.com/grpc-ecosystem/go-grpc-middleware/recovery.UnaryServerInterceptor.func1({0x343e9f0?, 0xc09d36ac60?}, {0x2698920?, 0xc09ced3a00?}, 0x7f3676742d98?, 0xc09ced3a20?)
github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/recovery/interceptors.go:33 +0xc4 fp=0xc0a2e538d8 sp=0xc0a2e53848 pc=0x110a064
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x343e9f0?, 0xc09d36ac60?}, {0x2698920?, 0xc09ced3a00?})
github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/chain.go:25 +0x3a fp=0xc0a2e53918 sp=0xc0a2e538d8 pc=0x1109e5a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x343e9f0, 0xc09d36ac60}, {0x2698920, 0xc09ced3a00}, 0xc0e8075901?, 0xc0e80759b8?)
github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/chain.go:34 +0xbf fp=0xc0a2e53970 sp=0xc0a2e53918 pc=0x1109cff
github.com/cosmos/cosmos-sdk/types/tx._Service_Simulate_Handler({0x2630240?, 0xc18df0a540}, {0x343e9f0, 0xc09d36ac60}, 0xc0804b9570, 0xc09d36ac90)
github.com/cosmos/cosmos-sdk@v0.46.7/types/tx/service.pb.go:935 +0x138 fp=0xc0a2e539c8 sp=0xc0a2e53970 pc=0xe1f6f8
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).RegisterGRPCServer.func2({0x2630240, 0xc18df0a540}, {0x343e9f0, 0xc09d36ac60}, 0x270fd80?, 0xc09d3630e0?)
github.com/cosmos/cosmos-sdk@v0.46.7/baseapp/grpcserver.go:82 +0xdc fp=0xc0a2e53a28 sp=0xc0a2e539c8 pc=0x11171fc
google.golang.org/grpc.(*Server).processUnaryRPC(0xc002282000, {0x344c468, 0xc0ed00da00}, 0xc09d3630e0, 0xc0e9ef0330, 0xc066cdab00, 0x0)
google.golang.org/grpc@v1.51.0/server.go:1340 +0xd13 fp=0xc0a2e53e48 sp=0xc0a2e53a28 pc=0xa676b3
google.golang.org/grpc.(*Server).handleStream(0xc002282000, {0x344c468, 0xc0ed00da00}, 0xc09d3630e0, 0x0)
google.golang.org/grpc@v1.51.0/server.go:1713 +0xa1b fp=0xc0a2e53f68 sp=0xc0a2e53e48 pc=0xa6c6bb
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.51.0/server.go:965 +0x98 fp=0xc0a2e53fe0 sp=0xc0a2e53f68 pc=0xa65158
runtime.goexit()
runtime/asm_amd64.s:1571 +0x1 fp=0xc0a2e53fe8 sp=0xc0a2e53fe0 pc=0x472b41
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.51.0/server.go:963 +0x28a
Reactions are currently unavailable
Metadata
Metadata
Assignees
Labels
No labels