diff --git a/src/backend/ir/module.h b/src/backend/ir/module.h index eee4e60..e5286f2 100644 --- a/src/backend/ir/module.h +++ b/src/backend/ir/module.h @@ -72,6 +72,7 @@ namespace rat { FunctionIterator begin() const { return {funcs.begin()}; } FunctionIterator end() const { return {funcs.end()}; } + U32 size() const { return (U32)funcs.size(); } private: String name; List funcs; diff --git a/src/backend/pass/opt/inline.cpp b/src/backend/pass/opt/inline.cpp index 2cd329a..3fb3ee3 100644 --- a/src/backend/pass/opt/inline.cpp +++ b/src/backend/pass/opt/inline.cpp @@ -140,78 +140,106 @@ namespace rat { } Function* InlinePass::lookup(const String& name) const { - auto it = graphByName.find(name); - return it == graphByName.end() ? nullptr : graphFuncs[it->second]; + auto it = byName.find(name); + return it == byName.end() ? nullptr : it->second; } - void InlinePass::refreshCallees(Function& fn) { - auto self = graphIndex.find(&fn); - if(self == graphIndex.end()) - return; - List& row = graphCallees[self->second]; - row.clear(); - ++edgeStampCur; + // rebuild fn's callee row + B32 InlinePass::refreshCallees(Function& fn, Info& info) { + List row; for(Node* n : fn) { CallNode* c = dyn_cast(n); if(!c) continue; - Function* next = lookup(c->getCallee()); - if(!next) - continue; - auto ni = graphIndex.find(next); - if(ni == graphIndex.end()) - continue; - U32 j = ni->second; - if(edgeStamp[j] == edgeStampCur) - continue; - edgeStamp[j] = edgeStampCur; - row.push_back(j); + if(Function* next = lookup(c->getCallee())) + row.push_back(&infos[next]); } + std::sort(row.begin(), row.end()); + row.erase(std::unique(row.begin(), row.end()), row.end()); + info.version = fn.getVersion(); + if(row == info.callees) + return false; + info.callees.swap(row); + return true; // edges changed } - void InlinePass::buildCallGraph(Module& m) { - graphModule = &m; - graphFuncs.clear(); - graphByName.clear(); - graphIndex.clear(); + // forget the functions dfe removed, and the edges into them + void InlinePass::dropDeadRows(Module& m) { + Set alive; + for(Function* fn : m) + alive.insert(fn); + for(auto& [fn, info] : infos) { + List& row = info.callees; + U32 w = 0; + for(Info* callee : row) + if(alive.count(callee->fn)) + row[w++] = callee; + row.resize(w); + } + for(auto it = infos.begin(); it != infos.end();) { + if(alive.count(it->first)) + ++it; + else + it = infos.erase(it); + } + } + + void InlinePass::forgetCycles() { + for(auto& [fn, info] : infos) + info.cyclic = -1; + } + + // refresh the rows of functions mutated since their row was built + void InlinePass::syncCallGraph(Module& m) { + if(module != &m) { + infos.clear(); + byName.clear(); + module = &m; + } + if(byName.size() != m.size()) { + byName.clear(); + for(Function* fn : m) { + byName.emplace(fn->getName(), fn); + infos[fn].fn = fn; + } + if(infos.size() > m.size()) + dropDeadRows(m); + } + B32 changed = false; for(Function* fn : m) { - U32 i = (U32)graphFuncs.size(); - graphByName.emplace(fn->getName(), i); - graphIndex[fn] = i; - graphFuncs.push_back(fn); + Info& info = infos[fn]; + if(info.version != fn->getVersion() && refreshCallees(*fn, info)) + changed = true; } - graphCallees.assign(graphFuncs.size(), List()); - visitStamp.assign(graphFuncs.size(), 0); - edgeStamp.assign(graphFuncs.size(), 0); - visitStampCur = 0; - edgeStampCur = 0; - cyclicCache.clear(); - for(Function* fn : graphFuncs) - refreshCallees(*fn); + if(changed) + forgetCycles(); } - B32 InlinePass::reachesIndex(U32 from, U32 target) { - if(visitStamp[from] == visitStampCur) + B32 InlinePass::reaches(Info* from, Info* target) { + if(from->visit == visitCur) return false; - visitStamp[from] = visitStampCur; - for(U32 next : graphCallees[from]) - if(next == target || reachesIndex(next, target)) + from->visit = visitCur; + for(Info* next : from->callees) + if(next == target || reaches(next, target)) return true; return false; } B32 InlinePass::isCyclic(Function* fn) { - auto it = cyclicCache.find(fn); - if(it != cyclicCache.end()) - return it->second; - B32 cyc = false; - auto gi = graphIndex.find(fn); - if(gi != graphIndex.end()) { - ++visitStampCur; - cyc = reachesIndex(gi->second, gi->second); + Info& info = infos[fn]; + if(info.cyclic < 0) { + ++visitCur; + info.cyclic = reaches(&info, &info); } - cyclicCache[fn] = cyc; - return cyc; + return info.cyclic; + } + + // caller version folded with its direct callees' versions + U64 InlinePass::quietStamp(const Function& caller, const Info& info) const { + U64 stamp = 14695981039346656037ull ^ caller.getVersion(); + for(Info* callee : info.callees) + stamp = stamp * 1099511628211ull + callee->fn->getVersion(); + return stamp; } B32 InlinePass::shouldInline(const Function& caller, CallNode* call, Function* callee) { @@ -237,33 +265,29 @@ namespace rat { } B32 InlinePass::run(Module& module, const TargetInfo& target) { - graphModule = nullptr; + syncCallGraph(module); return FunctionPass::run(module, target); } U32 InlinePass::runOnFunction(Function& caller, const TargetInfo&) { - Module& m = caller.getModule(); - if(graphModule != &m) - buildCallGraph(m); - - U64 stamp = caller.getVersion(); - if(auto gi = graphIndex.find(&caller); gi != graphIndex.end()) - for(U32 j : graphCallees[gi->second]) - stamp = stamp * 1099511628211ull + graphFuncs[j]->getVersion(); - if(auto qa = quietAt.find(&caller); qa != quietAt.end() && qa->second == stamp) + if(module != &caller.getModule()) + syncCallGraph(caller.getModule()); + Info& info = infos[&caller]; + U64 stamp = quietStamp(caller, info); + if(info.quietAt == stamp) return 0; - - U32 count = 0; - U32 baseline = caller.size(); + if(!info.firstSize) + info.firstSize = caller.size(); + U32 limit = info.firstSize + kCallerGrowthBudget; worklist.clear(); for(Node* n : caller) if(CallNode* c = dyn_cast(n)) worklist.push_back(c); + U32 count = 0; B32 changed = true; - while(changed && count < kMaxInlinesPerFunction && - caller.size() <= baseline + kCallerGrowthBudget) { + while(changed && count < kMaxInlinesPerFunction && caller.size() <= limit) { changed = false; for(U32 i = 0; i < worklist.size(); ++i) { CallNode* c = worklist[i]; @@ -282,13 +306,13 @@ namespace rat { } } } - if(count) { - caller.eliminateDeadNodes(); - refreshCallees(caller); - cyclicCache.clear(); - } else { - quietAt[&caller] = stamp; + if(!count) { + info.quietAt = stamp; + return 0; } + caller.eliminateDeadNodes(); + if(refreshCallees(caller, info)) + forgetCycles(); return count; } diff --git a/src/backend/pass/opt/inline.h b/src/backend/pass/opt/inline.h index ea0d69a..6bdfda8 100644 --- a/src/backend/pass/opt/inline.h +++ b/src/backend/pass/opt/inline.h @@ -29,33 +29,40 @@ namespace rat { U32 runOnFunction(Function& caller, const TargetInfo& target) override; B32 onlyReadsFunction() const override { return false; } // reads callees private: - B32 isStartProj(const Function& callee, Node* n); - void buildCallGraph(Module& m); - void refreshCallees(Function& fn); + // call graph row + struct Info { + Function* fn = nullptr; + List callees; // direct callees, sorted, live functions only + U64 version = kNoRow; // caller version the row was built from + U64 quietAt = 0; // stamp of the last run that inlined nothing, 0 = none + U32 firstSize = 0; // caller size when first seen, bounds growth per module run + U32 visit = 0; // dfs stamp + I8 cyclic = -1; // cached isCyclic, -1 = unknown + }; + static constexpr U64 kNoRow = ~(U64)0; + + void syncCallGraph(Module& m); + void dropDeadRows(Module& m); + B32 refreshCallees(Function& fn, Info& info); + void forgetCycles(); Function* lookup(const String& name) const; - B32 reachesIndex(U32 from, U32 target); + B32 reaches(Info* from, Info* target); B32 isCyclic(Function* fn); - private: - Module* graphModule = nullptr; - Map quietAt; - List graphFuncs; - List> graphCallees; - Map graphByName; - Map graphIndex; - List visitStamp; - List edgeStamp; - U32 visitStampCur = 0; - U32 edgeStampCur = 0; - Map cyclicCache; + U64 quietStamp(const Function& caller, const Info& info) const; + B32 isStartProj(const Function& callee, Node* n); Node* incomingForStartProj(CallNode* call, U32 startProjIdx); B32 shouldInline(const Function& caller, CallNode* call, Function* callee); - B32 inlineCallSite(Function& caller, CallNode* call, Function& callee, List& newCalls); - // callee node id -> caller node, reused across call sites - List cloneMap; + Module* module = nullptr; + Map infos; + Map byName; + U32 visitCur = 0; + + // scratch reused across call sites + List cloneMap; // callee node id -> caller node List worklist; List ctrls, mems, vals; };