From 947adb89e3792c9dd6931d949803e819e6b9b110 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Sat, 5 Mar 2022 21:23:16 +0100 Subject: [PATCH] [rust] Mark inlined calls to functions in same SCC as callee as noinline This fixes the catastrophic inlining issues observerd with the new LLVM pass manager. Patch by Arthur Eubanks from https://reviews.llvm.org/D120584. --- llvm/lib/Transforms/IPO/Inliner.cpp | 23 +++++-- llvm/test/Transforms/Inline/mut-rec-scc-2.ll | 19 ++++++ llvm/test/Transforms/Inline/mut-rec-scc.ll | 68 ++++++++++++++++++++ 3 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 llvm/test/Transforms/Inline/mut-rec-scc-2.ll create mode 100644 llvm/test/Transforms/Inline/mut-rec-scc.ll diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp index 49babc24cb82c..1aca41c227aac 100644 --- a/llvm/lib/Transforms/IPO/Inliner.cpp +++ b/llvm/lib/Transforms/IPO/Inliner.cpp @@ -876,8 +876,8 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, // trigger infinite inlining, much like is prevented within the inliner // itself by the InlineHistory above, but spread across CGSCC iterations // and thus hidden from the full inline history. - if (CG.lookupSCC(*CG.lookup(Callee)) == C && - UR.InlinedInternalEdges.count({&N, C})) { + LazyCallGraph::SCC *CalleeSCC = CG.lookupSCC(*CG.lookup(Callee)); + if (CalleeSCC == C && UR.InlinedInternalEdges.count({&N, C})) { LLVM_DEBUG(dbgs() << "Skipping inlining internal SCC edge from a node " "previously split out of this SCC by inlining: " << F.getName() << " -> " << Callee.getName() << "\n"); @@ -935,9 +935,24 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, if (tryPromoteCall(*ICB)) NewCallee = ICB->getCalledFunction(); } - if (NewCallee) - if (!NewCallee->isDeclaration()) + if (NewCallee) { + if (!NewCallee->isDeclaration()) { Calls->push({ICB, NewHistoryID}); + // Continually inlining through an SCC can result in huge compile + // times and bloated code since we arbitrarily stop at some point + // when the inliner decides it's not profitable to inline + // anymore. We put a stop at the first potential attempt at + // inlining through an SCC by marking the call site as noinline. + // This doesn't apply to calls in the same SCC since if we do + // inline through the SCC the function will end up being + // self-recursive which the inliner bails out on, and inlining + // within an SCC is necessary for performance. + if (CalleeSCC != C && + CalleeSCC == CG.lookupSCC(CG.get(*NewCallee))) { + ICB->addFnAttr(Attribute::NoInline); + } + } + } } } diff --git a/llvm/test/Transforms/Inline/mut-rec-scc-2.ll b/llvm/test/Transforms/Inline/mut-rec-scc-2.ll new file mode 100644 index 0000000000000..b51d56c359ba3 --- /dev/null +++ b/llvm/test/Transforms/Inline/mut-rec-scc-2.ll @@ -0,0 +1,19 @@ +; RUN: opt -S -passes='inline' < %s | FileCheck %s + +; Make sure we don't mark calls within the same SCC as original function with noinline. +; CHECK-NOT: noinline + +define void @samescc1() { + call void @samescc2() + ret void +} + +define void @samescc2() { + call void @samescc3() + ret void +} + +define void @samescc3() { + call void @samescc1() + ret void +} diff --git a/llvm/test/Transforms/Inline/mut-rec-scc.ll b/llvm/test/Transforms/Inline/mut-rec-scc.ll new file mode 100644 index 0000000000000..41a43709a1360 --- /dev/null +++ b/llvm/test/Transforms/Inline/mut-rec-scc.ll @@ -0,0 +1,68 @@ +; RUN: opt -S -passes='cgscc(inline,instcombine)' < %s | FileCheck %s + +; We use call to a dummy function to avoid inlining test1 into test2 or vice +; versa, such that we aren't left with a trivial cycle, as trivial cycles are +; special-cased to never be inlined. +; However, InstCombine will eliminate these calls after inlining, and thus +; make the functions eligible for inlining in their callers. +declare void @dummy() readnone nounwind willreturn + +define void @test1() { +; CHECK-LABEL: define void @test1( +; CHECK-NEXT: call void @test2() +; CHECK-NEXT: call void @test2() +; CHECK-NEXT: ret void +; + call void @test2() + call void @test2() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + ret void +} + +define void @test2() { +; CHECK-LABEL: define void @test2( +; CHECK-NEXT: call void @test1() +; CHECK-NEXT: call void @test1() +; CHECK-NEXT: ret void +; + call void @test1() + call void @test1() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + ret void +} + +; We should inline the @test2 calls and mark the inlined @test1 calls as noinline +define void @test3() { +; CHECK-LABEL: define void @test3( +; CHECK-NEXT: call void @test1() #[[NOINLINE:[0-9]+]] +; CHECK-NEXT: call void @test1() #[[NOINLINE]] +; CHECK-NEXT: call void @test1() #[[NOINLINE]] +; CHECK-NEXT: call void @test1() #[[NOINLINE]] +; CHECK-NEXT: ret void +; + call void @test2() + call void @test2() + ret void +} + +; CHECK: [[NOINLINE]] = { noinline }