Skip to content

Commit 582f1fb

Browse files
wmitsudawmitsuda
authored
snapcfg: load preverified hashes for single chain only (#19785)
## Summary Part 3 of optimizing remote preverified hash loading (after #19641, #19722). - `LoadPreverified` now takes a `chainName` parameter and calls `LoadRemotePreverified` instead of the old bulk variant that fetched all 10 chains - Refactored `webseeds.Verify` to load preverified per-chain inside the iteration loop instead of bulk-loading all chains upfront - Removed unused functions: old bulk `LoadRemotePreverified`, `registry.All`, `registry.ResetRaw`, `GetAllCurrentPreverified` - Renamed `LoadRemotePreverifiedForChain` → `LoadRemotePreverified` since it's now the only variant ## Test plan - [x] Built `erigon` and `downloader` binaries - [x] Tested `erigon seg reset --dry-run` with mainnet and hoodi ephemeral datadirs - [x] Tested `downloader verify_webseeds --chain=chiado --preverified=embedded` to completion - [x] Verified only the requested chain is fetched (confirmed via log output) ## Tasks - [ ] Cherry-pick merge commit to `release/3.4`
1 parent 0f3624a commit 582f1fb

5 files changed

Lines changed: 13 additions & 82 deletions

File tree

‎cmd/utils/app/reset-datadir.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ func resetCliAction(cliCtx *cli.Context) (err error) {
9090
}
9191
defer unlock()
9292

93-
err = snapcfg.LoadPreverified(cliCtx.Context, PreverifiedFlag.Get(cliCtx), &dirs)
93+
err = snapcfg.LoadPreverified(cliCtx.Context, PreverifiedFlag.Get(cliCtx), &dirs, chainName)
9494
if err != nil {
9595
return
9696
}

‎db/downloader/downloadercfg/downloadercfg.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -328,7 +328,7 @@ func LoadSnapshotsHashes(ctx context.Context, dirs datadir.Dirs, chainName strin
328328
snapcfg.SetToml(chainName, haveToml, true)
329329
} else {
330330
// Fetch the snapshot hashes from the web
331-
err := snapcfg.LoadRemotePreverifiedForChain(ctx, chainName)
331+
err := snapcfg.LoadRemotePreverified(ctx, chainName)
332332
if err != nil {
333333
log.Root().Crit("Snapshot hashes for supported networks was not loaded. Please check your network connection and/or GitHub status here https://www.githubstatus.com/", "chain", chainName, "err", err)
334334
return fmt.Errorf("failed to fetch remote snapshot hashes for chain %s", chainName)

‎db/downloader/webseeds/verify.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,6 @@ func Verify(
4444
if dataDir != "" {
4545
dirs = datadir.Open(dataDir)
4646
}
47-
err = snapcfg.LoadPreverified(ctx, preverifiedFlagValue, &dirs)
48-
if err != nil {
49-
return
50-
}
51-
allPreverified := snapcfg.GetAllCurrentPreverified()
5247
chains := selectChains(targetChain, snapcfg.EmbeddedWebseedsRaw)
5348
if len(chains) == 0 {
5449
err = errors.New("no matching chains")
@@ -78,6 +73,11 @@ func Verify(
7873
for _, chain := range chains {
7974
// Shift left?
8075
//
76+
err = snapcfg.LoadPreverified(ctx, preverifiedFlagValue, &dirs, chain)
77+
if err != nil {
78+
return
79+
}
80+
cfg, _ := snapcfg.KnownCfg(chain)
8181
webseeds, _ := snapcfg.GetEmbeddedWebseeds(chain)
8282
var baseUrl string
8383
err := errors.New("no valid webseeds")
@@ -88,7 +88,7 @@ func Verify(
8888
}
8989
}
9090
panicif.Err(err)
91-
preverified := g.MapMustGet(allPreverified, chain)
91+
preverified := cfg.Preverified
9292
panicif.True(preverified.Local)
9393
//
9494
// end shift left?

‎db/snapcfg/preverified.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,17 @@ import (
1212

1313
// Loads preverified from locations other than just remote. Usually for utility commands that want
1414
// to test different preverified sources.
15-
func LoadPreverified(ctx context.Context, flagValue string, dirs *datadir.Dirs) (err error) {
15+
func LoadPreverified(ctx context.Context, flagValue string, dirs *datadir.Dirs, chainName string) (err error) {
1616
switch flagValue {
1717
case "local":
1818
panicif.Err(os.Setenv(RemotePreverifiedEnvKey, dirs.PreverifiedPath()))
1919
fallthrough
2020
case "remote":
21-
err = LoadRemotePreverified(ctx)
21+
err = LoadRemotePreverified(ctx, chainName)
2222
if err != nil {
2323
// TODO: Check if we should continue? What if we ask for a git revision and
2424
// can't get it? What about a branch? Can we reset to the embedded snapshot hashes?
25-
return fmt.Errorf("loading remote preverified snapshots: %w", err)
25+
return fmt.Errorf("loading remote preverified snapshots for chain %q: %w", chainName, err)
2626
}
2727
case "embedded":
2828
// Should already be loaded.

‎db/snapcfg/util.go‎

Lines changed: 2 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323
"errors"
2424
"fmt"
2525
"io"
26-
"maps"
2726
"net/http"
2827
"os"
2928
"path/filepath"
@@ -76,19 +75,6 @@ var registry = &preverifiedRegistry{
7675
cached: make(map[string]*Cfg),
7776
}
7877

79-
// All forces parsing of all remaining raw entries and returns a clone of the data map.
80-
func (r *preverifiedRegistry) All() map[string]Preverified {
81-
r.mu.Lock()
82-
defer r.mu.Unlock()
83-
for name, rawBytes := range r.raw {
84-
if _, ok := r.data[name]; !ok {
85-
r.data[name] = fromEmbeddedToml(rawBytes)
86-
}
87-
}
88-
r.raw = nil
89-
return maps.Clone(r.data)
90-
}
91-
9278
func (r *preverifiedRegistry) Get(networkName string) (*Cfg, bool) {
9379
r.mu.RLock()
9480
if cfg, ok := r.cached[networkName]; ok {
@@ -147,16 +133,6 @@ func (r *preverifiedRegistry) Reset(data map[string]Preverified) {
147133
r.mu.Unlock()
148134
}
149135

150-
// ResetRaw replaces the raw bytes, clearing data and cached. Used after remote loading
151-
// updates the module-level snapshot hash vars.
152-
func (r *preverifiedRegistry) ResetRaw(raw map[string][]byte) {
153-
r.mu.Lock()
154-
r.raw = raw
155-
r.data = make(map[string]Preverified)
156-
r.cached = make(map[string]*Cfg)
157-
r.mu.Unlock()
158-
}
159-
160136
// Has checks whether a chain name exists in either raw or data without triggering parsing.
161137
func (r *preverifiedRegistry) Has(networkName string) bool {
162138
r.mu.RLock()
@@ -593,8 +569,8 @@ func fetchChainToml(ctx context.Context, source snapshothashes.SnapshotSource, b
593569
return res, nil
594570
}
595571

596-
// LoadRemotePreverifiedForChain fetches and loads snapshot hashes for a single chain only.
597-
func LoadRemotePreverifiedForChain(ctx context.Context, chainName string) error {
572+
// LoadRemotePreverified fetches and loads snapshot hashes for a single chain.
573+
func LoadRemotePreverified(ctx context.Context, chainName string) error {
598574
if s, ok := os.LookupEnv(RemotePreverifiedEnvKey); ok {
599575
log.Info("Loading local preverified override file", "file", s)
600576

@@ -630,46 +606,6 @@ func LoadRemotePreverifiedForChain(ctx context.Context, chainName string) error
630606
return nil
631607
}
632608

633-
func LoadRemotePreverified(ctx context.Context) (err error) {
634-
if s, ok := os.LookupEnv(RemotePreverifiedEnvKey); ok {
635-
log.Info("Loading local preverified override file", "file", s)
636-
637-
b, err := os.ReadFile(s)
638-
if err != nil {
639-
return fmt.Errorf("reading remote preverified override file: %w", err)
640-
}
641-
for _, sh := range snapshotHashPtrs {
642-
*sh = bytes.Clone(b)
643-
}
644-
} else {
645-
log.Info("Loading remote snapshot hashes")
646-
647-
err = snapshothashes.LoadSnapshots(ctx, snapshothashes.R2, snapshotGitBranch)
648-
if err != nil {
649-
log.Root().Warn("Failed to load snapshot hashes from R2; falling back to GitHub", "err", err)
650-
651-
// Fallback to GitHub if R2 fails
652-
err = snapshothashes.LoadSnapshots(ctx, snapshothashes.Github, snapshotGitBranch)
653-
if err != nil {
654-
return err
655-
}
656-
}
657-
}
658-
659-
// Re-load the preverified hashes
660-
registry.ResetRaw(map[string][]byte{
661-
networkname.Mainnet: snapshothashes.Mainnet,
662-
networkname.Sepolia: snapshothashes.Sepolia,
663-
networkname.Amoy: snapshothashes.Amoy,
664-
networkname.BorMainnet: snapshothashes.BorMainnet,
665-
networkname.Gnosis: snapshothashes.Gnosis,
666-
networkname.Chiado: snapshothashes.Chiado,
667-
networkname.Hoodi: snapshothashes.Hoodi,
668-
networkname.Bloatnet: snapshothashes.Bloatnet,
669-
})
670-
return
671-
}
672-
673609
func SetToml(networkName string, toml []byte, local bool) {
674610
if registry.Has(networkName) {
675611
registry.Set(networkName, Preverified{Local: local, Items: fromToml(toml)})
@@ -683,11 +619,6 @@ func GetToml(networkName string) []byte {
683619
return nil
684620
}
685621

686-
// Gets the current preverified for all chains.
687-
func GetAllCurrentPreverified() map[string]Preverified {
688-
return registry.All()
689-
}
690-
691622
// Converts webseed value to URL. Mostly this is just stripping v1: for now, as nothing else is in
692623
// active use.
693624
func WebseedToUrl(s string) (_ string, err error) {

0 commit comments

Comments
 (0)