diff --git a/lld/ELF/MarkLive.cpp b/lld/ELF/MarkLive.cpp index 5ac93c1e0ed23..3ed1a51ef13bd 100644 --- a/lld/ELF/MarkLive.cpp +++ b/lld/ELF/MarkLive.cpp @@ -30,6 +30,7 @@ #include "lld/Common/Strings.h" #include "llvm/ADT/DenseMapInfoVariant.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Parallel.h" #include "llvm/Support/TimeProfiler.h" #include #include @@ -66,6 +67,7 @@ template class MarkLive { LiveReason reason); void markSymbol(Symbol *sym, StringRef reason); void mark(); + void markParallel(); template void resolveReloc(InputSectionBase &sec, const RelTy &rel, bool fromFDE); @@ -460,7 +462,12 @@ void MarkLive::run() { template void MarkLive::mark() { - // Mark all reachable sections. + if constexpr (!TrackWhyLive) { + if (ctx.partitions.size() == 1) { + markParallel(); + return; + } + } while (!queue.empty()) { InputSectionBase &sec = *queue.pop_back_val(); @@ -483,6 +490,97 @@ void MarkLive::mark() { } } +// Helper function for markParallel. Walk all GC edges from sec, marking +// everything that needs to be live. Call fn(target section, offset) for each +// edge, which will mark the section live and handle further processing of edges +// from that section. +template +static void processSectionEdges( + Ctx &ctx, InputSectionBase &sec, + const DenseMap> + &cNamedSections, + Fn fn) { + auto resolveEdge = [&](const auto &rel) { + Symbol &sym = sec.file->getRelocTargetSym(rel); + if (!sym.hasFlag(USED)) + sym.setFlags(USED); + if (auto *d = dyn_cast(&sym)) { + if (auto *relSec = dyn_cast_or_null(d->section)) { + uint64_t offset = d->value; + if (d->isSection()) { + offset += getAddend(ctx, sec, rel); + if (auto *ms = dyn_cast(relSec); + ms && offset >= ms->content().size()) + return; + } + if (auto *ms = dyn_cast(relSec)) { + auto &piece = ms->getSectionPiece(offset); + auto *word = + reinterpret_cast *>(&piece.inputOff + 1); + constexpr uint32_t liveBit = sys::IsBigEndianHost ? (1U << 31) : 1U; + word->fetch_or(liveBit, std::memory_order_relaxed); + } + fn(relSec, offset); + } + return; + } + for (InputSectionBase *csec : cNamedSections.lookup(sym.getName())) + fn(csec, 0); + }; + const RelsOrRelas rels = sec.template relsOrRelas(); + for (const typename ELFT::Rel &rel : rels.rels) + resolveEdge(rel); + for (const typename ELFT::Rela &rel : rels.relas) + resolveEdge(rel); + for (const typename ELFT::Crel &rel : rels.crels) + resolveEdge(rel); + for (InputSectionBase *isec : sec.dependentSections) + fn(isec, 0); + if (sec.nextInSectionGroup) + fn(sec.nextInSectionGroup, 0); +} + +// Parallel mark using level-synchronized BFS with depth-limited inline +// recursion. Each parallelFor iteration processes a subtree up to depth 3 +// (DFS for cache locality), then queues deeper discoveries for the next level. +template +void MarkLive::markParallel() { + const size_t numThreads = parallel::getThreadCount(); + auto visit = [&](InputSection *sec, int depth, + SmallVector &localQueue, + auto &self) -> void { + processSectionEdges( + ctx, *sec, cNamedSections, + [&](InputSectionBase *target, uint64_t offset) { + auto &part = + reinterpret_cast &>(target->partition); + // Optimistic load-then-exchange avoids expensive atomic + // RMW on already-visited sections. + if (part.load(std::memory_order_relaxed) != 0 || + part.exchange(1, std::memory_order_relaxed) != 0) + return; + if (auto *s = dyn_cast(target)) { + if (depth < 3) + self(s, depth + 1, localQueue, self); + else + localQueue.push_back(s); + } + }); + }; + + while (!queue.empty()) { + auto queues = + std::make_unique[]>(numThreads); + parallelFor(0, queue.size(), [&](size_t i) { + const unsigned tid = parallel::getThreadIndex(); + visit(queue[i], 0, queues[tid], visit); + }); + queue.clear(); + for (size_t t = 0; t < numThreads; ++t) + queue.append(std::move(queues[t])); + } +} + // Move the sections for some symbols to the main partition, specifically ifuncs // (because they can result in an IRELATIVE being added to the main partition's // GOT, which means that the ifunc must be available when the main partition is @@ -527,8 +625,8 @@ template void elf::markLive(Ctx &ctx) { return; } - for (InputSectionBase *sec : ctx.inputSections) - sec->markDead(); + parallelForEach(ctx.inputSections, + [](InputSectionBase *sec) { sec->markDead(); }); // Follow the graph to mark all live sections. for (unsigned i = 1, e = ctx.partitions.size(); i <= e; ++i)