Skip to content

Commit 76b99ac

Browse files
mjfhmjfh
authored
Snap sync update and tools layout reshuffle (#3978)
* Rearrange/re-implement state DB layout, file names etc. * Rearrange/re-implement accounts download logic, file layout * Update MPT tools, add support for storage slots * Implement download framework for storage slots * Cosmetics, remove cruft, etc.
1 parent 2c31d45 commit 76b99ac

24 files changed

Lines changed: 1850 additions & 595 deletions

‎execution_chain/sync/snap/worker.nim‎

Lines changed: 53 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@
1313
import
1414
std/os,
1515
pkg/[chronicles, chronos, minilru, results],
16-
./worker/[account, helpers, start_stop, state_db, worker_desc]
16+
./worker/[
17+
account, download, header, helpers, start_stop, state_db, worker_desc]
1718

1819
logScope:
1920
topics = "snap sync"
@@ -22,6 +23,39 @@ logScope:
2223
# Private helpers
2324
# ------------------------------------------------------------------------------
2425

26+
template updateTarget(
27+
buddy: SnapPeerRef;
28+
info: static[string];
29+
) =
30+
## Async/template
31+
##
32+
block body:
33+
# Check whether explicit target setup is configured
34+
if buddy.ctx.pool.target.isSome():
35+
let
36+
peer {.inject,used.} = $buddy.peer # logging only
37+
ctx = buddy.ctx
38+
39+
# Single target block hash
40+
if ctx.pool.target.value.blockHash != BlockHash(zeroHash32):
41+
let rc = buddy.headerStateRegister(ctx.pool.target.value.blockHash)
42+
if rc.isErr and rc.error: # real error
43+
trace info & ": failed fetching pivot hash", peer,
44+
hash=ctx.pool.target.value.blockHash.toStr
45+
elif 0 < ctx.pool.target.value.updateFile.len:
46+
var target = ctx.pool.target.value
47+
target.blockHash = BlockHash(zeroHash32)
48+
ctx.pool.target = Opt.some(target)
49+
else:
50+
ctx.pool.target = Opt.none(SnapTarget) # No more target entries
51+
break body # noting more to do here
52+
53+
# Check whether a file target setup is configured
54+
if 0 < ctx.pool.target.value.updateFile.len:
55+
discard buddy.headerStateLoad(ctx.pool.target.value.updateFile, info)
56+
57+
discard # visual alignment
58+
2559
# ------------------------------------------------------------------------------
2660
# Public start/stop and admin functions
2761
# ------------------------------------------------------------------------------
@@ -81,8 +115,19 @@ template runDaemon*(ctx: SnapCtxRef; info: static[string]): Duration =
81115
##
82116
## The template returns a suggested idle time for after this task.
83117
##
84-
debug info & ": snap sync not implemented yet", nSyncPeers=ctx.nSyncPeers()
85-
chronos.seconds(30)
118+
var bodyRc = chronos.nanoseconds(0) # to be re-invoked, soon?
119+
block body:
120+
# Run the DB verification and update jobs only while there are no active
121+
# peers. So that downloading will get all the available processing time.
122+
if ctx.nSyncPeers() == 0:
123+
if ctx.accountRequeue(info):
124+
bodyRc = daemonOkInterval
125+
break body
126+
127+
bodyRc = daemonWaitInterval # take a short nap
128+
# End block: `body`
129+
130+
bodyRc
86131

87132
proc runPool*(
88133
buddy: SnapPeerRef;
@@ -104,9 +149,7 @@ proc runPool*(
104149
##
105150
## Note that this function does not run in `async` mode.
106151
##
107-
debug info & ": snap sync not implemented yet", peer=buddy.peer,
108-
nSyncPeers=buddy.ctx.nSyncPeers()
109-
true # stop
152+
true # stop
110153

111154
template runPeer*(
112155
buddy: SnapPeerRef;
@@ -121,14 +164,11 @@ template runPeer*(
121164
##
122165
var bodyRc = chronos.nanoseconds(0)
123166
block body:
124-
let
125-
ctx = buddy.ctx
126-
peer {.inject,used.} = $buddy.peer # logging only
127-
128-
buddy.accountRangeImport info
167+
# Check for manual target settings
168+
buddy.updateTarget info
129169

130-
debug info & ": snap sync not implemented yet", peer,
131-
nSyncPeers=ctx.nSyncPeers()
170+
# Download and chace accounts, storage slots, contracts
171+
buddy.download info
132172

133173
bodyRc = chronos.seconds(10)
134174
# End block: `body`

‎execution_chain/sync/snap/worker/account.nim‎

Lines changed: 4 additions & 127 deletions
Original file line numberDiff line numberDiff line change
@@ -11,133 +11,10 @@
1111
{.push raises: [].}
1212

1313
import
14-
pkg/[chronicles, chronos],
15-
account/account_fetch,
16-
./[helpers, header, state_db, worker_desc]
14+
account/[account_download, account_requeue]
1715

18-
# ------------------------------------------------------------------------------
19-
# Private helpers
20-
# ------------------------------------------------------------------------------
16+
export
17+
account_download,
18+
account_requeue
2119

22-
template updateTarget(
23-
buddy: SnapPeerRef;
24-
info: static[string];
25-
) =
26-
## Async/template
27-
##
28-
block body:
29-
# Check whether explicit target setup is configured
30-
if buddy.ctx.pool.target.isSome():
31-
let
32-
peer {.inject,used.} = $buddy.peer # logging only
33-
ctx = buddy.ctx
34-
35-
# Single target block hash
36-
if ctx.pool.target.value.blockHash != BlockHash(zeroHash32):
37-
let rc = buddy.headerStateRegister(ctx.pool.target.value.blockHash)
38-
if rc.isErr and rc.error: # real error
39-
trace info & ": failed fetching pivot hash", peer,
40-
hash=ctx.pool.target.value.blockHash.toStr
41-
elif 0 < ctx.pool.target.value.updateFile.len:
42-
var target = ctx.pool.target.value
43-
target.blockHash = BlockHash(zeroHash32)
44-
ctx.pool.target = Opt.some(target)
45-
else:
46-
ctx.pool.target = Opt.none(SnapTarget) # No more target entries
47-
break body # noting more to do here
48-
49-
# Check whether a file target setup is configured
50-
if 0 < ctx.pool.target.value.updateFile.len:
51-
trace info & ": target update from file", peer,
52-
file=ctx.pool.target.value.updateFile
53-
discard buddy.headerStateLoad(ctx.pool.target.value.updateFile, info)
54-
trace info & ": target update from file.. done", peer
55-
56-
discard # visual alignment
57-
58-
59-
template download(
60-
buddy: SnapPeerRef;
61-
state: StateDataRef;
62-
info: static[string];
63-
) =
64-
## Async/template
65-
##
66-
block body:
67-
let
68-
ctx = buddy.ctx
69-
sdb = ctx.pool.stateDB
70-
71-
peer {.inject,used.} = $buddy.peer # logging only
72-
root {.inject,used.} = state.rootStr # logging only
73-
74-
iv = state.unproc.fetchLeast(unprocAccountsRangeMax).valueOr:
75-
trace info & ": no more unpocessed", peer, root, stateDB=sdb.toStr
76-
break body
77-
78-
accData = buddy.fetchAccounts(state.root, iv).valueOr:
79-
state.unproc.commit(iv, iv) # registry roll back
80-
state.downScore()
81-
trace info & ": account download failed", peer, root,
82-
iv=iv.to(float).toStr, stateDB=sdb.toStr
83-
break body
84-
85-
# Accept, update registry
86-
if 0 < accData.accounts.len:
87-
let accTop = accData.accounts[^1].accHash.to(ItemKey)
88-
state.unproc.commit(iv, accTop + 1, iv.maxPt)
89-
state.unproc.overCommit(iv.maxPt + 1, accTop)
90-
state.upScore()
91-
else:
92-
state.downScore()
93-
94-
debug info & ": accounts downloaded", peer, root, iv=iv.to(float).toStr,
95-
nAccounts=accData.accounts.len, nProof=accData.proof.len,
96-
stateDB=sdb.toStr
97-
98-
discard # visual alignment
99-
100-
# ------------------------------------------------------------------------------
101-
# Public function
102-
# ------------------------------------------------------------------------------
103-
104-
template accountRangeImport*(buddy: SnapPeerRef; info: static[string]) =
105-
## Async/template
106-
##
107-
## Fetch and stash account ranges -- TBD
108-
##
109-
block body:
110-
let
111-
ctx = buddy.ctx
112-
sdb = ctx.pool.stateDB
113-
peer {.inject,used.} = $buddy.peer # logging only
114-
115-
# Update state db, add new state records if pivots are available
116-
if buddy.only.pivotRoot.isNone():
117-
let ethPeer = buddy.getEthPeer() # get `ethXX` peer if avail
118-
if not ethPeer.isNil:
119-
trace info & ": processing best/latest pivotHash", peer,
120-
hash=ethPeer.only.pivotHash.short
121-
buddy.headerStateRegister(BlockHash(ethPeer.only.pivotHash)).isErrOr:
122-
buddy.only.pivotRoot = Opt.some(value)
123-
124-
# Check for maual target settings
125-
buddy.updateTarget info
126-
127-
if ctx.pool.stateDB.len == 0:
128-
trace info & ": no state records", peer, stateDB=sdb.toStr
129-
break body
130-
131-
# Fetch for state DB items, start with pivot root
132-
for state in sdb.items(startWith=buddy.only.pivotRoot, truncate=true):
133-
if buddy.ctrl.stopped:
134-
break
135-
trace info & ": download state", peer, root=state.rootStr,
136-
stateDB=sdb.toStr
137-
buddy.download(state, info)
138-
139-
discard # visual alignment
140-
141-
# ------------------------------------------------------------------------------
14220
# End
143-
# ------------------------------------------------------------------------------

‎execution_chain/sync/snap/worker/account/account_download.nim‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Nimbus
2+
# Copyright (c) 2025-2026 Status Research & Development GmbH
3+
# Licensed under either of
4+
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
5+
# http://www.apache.org/licenses/LICENSE-2.0)
6+
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
7+
# http://opensource.org/licenses/MIT)
8+
# at your option. This file may not be copied, modified, or distributed
9+
# except according to those terms.
10+
11+
{.push raises: [].}
12+
13+
import
14+
pkg/[chronicles, chronos],
15+
../[helpers, mpt, state_db, worker_desc],
16+
./account_fetch
17+
18+
# ------------------------------------------------------------------------------
19+
# Public function
20+
# ------------------------------------------------------------------------------
21+
22+
template accountDownload*(
23+
buddy: SnapPeerRef; # Snap peer
24+
state: StateDataRef; # Current state
25+
info: static[string]; # Log message prefix
26+
): Opt[seq[SnapAccount]] =
27+
## Async/template
28+
##
29+
## On success, the template returns a list of accounts for storage and
30+
## code processing.
31+
##
32+
var bodyRc = Opt[seq[SnapAccount]].err()
33+
block body:
34+
let
35+
ctx = buddy.ctx
36+
sdb = ctx.pool.stateDB
37+
adb = ctx.pool.mptAsm
38+
39+
peer {.inject,used.} = $buddy.peer # logging only
40+
root {.inject,used.} = state.rootStr # logging only
41+
42+
ivReq = sdb.fetchAccountRange(state).valueOr:
43+
trace info & ": no more unpocessed", peer, root
44+
break body # return err()
45+
46+
iv {.inject,used.} = ivReq.flStr # logging only
47+
48+
trace info & ": requesting account range", peer, root, iv
49+
50+
let
51+
data = buddy.fetchAccounts(state.root, ivReq).valueOr:
52+
sdb.rollbackAccountRange(state, ivReq) # registry roll back
53+
if error == ENoDataAvailable:
54+
state.downScore()
55+
trace info & ": account download failed", peer, root, iv,
56+
`error`=error
57+
break body # return err()
58+
59+
limit = if data.accounts.len == 0: high(ItemKey)
60+
else: data.accounts[^1].accHash.to(ItemKey)
61+
62+
nAccounts {.inject,used.} = data.accounts.len # logging only
63+
nProof {.inject,used.} = data.proof.len # logging only
64+
65+
# Stash accounts data packet to be processed later
66+
adb.putRawAccounts(state.root,ivReq.minPt,limit,data,buddy.peerID).isOkOr:
67+
sdb.rollbackAccountRange(state, ivReq) # registry roll back
68+
debug info & ": caching accounts failed", peer, root, iv,
69+
nAccounts, nProof
70+
break body # return err()
71+
72+
state.upScore() # got some data
73+
sdb.commitAccountRange(state, ivReq, limit) # update registry
74+
bodyRc = typeof(bodyRc).ok(data.accounts) # return code
75+
76+
debug info & ": accounts downloaded and cached", peer, root, iv,
77+
nAccounts, nProof
78+
79+
bodyRc
80+
81+
# ------------------------------------------------------------------------------
82+
# End
83+
# ------------------------------------------------------------------------------

0 commit comments

Comments
 (0)