1//===- InlineFunction.cpp - Code to perform function inlining -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements inlining of a function into a call site, resolving
10// parameters and the return value as appropriate.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ADT/DenseMap.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SetVector.h"
17#include "llvm/ADT/SmallPtrSet.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/StringExtras.h"
20#include "llvm/ADT/iterator_range.h"
21#include "llvm/Analysis/AliasAnalysis.h"
22#include "llvm/Analysis/AssumptionCache.h"
23#include "llvm/Analysis/BlockFrequencyInfo.h"
24#include "llvm/Analysis/CallGraph.h"
25#include "llvm/Analysis/CaptureTracking.h"
26#include "llvm/Analysis/CtxProfAnalysis.h"
27#include "llvm/Analysis/IndirectCallVisitor.h"
28#include "llvm/Analysis/InstructionSimplify.h"
29#include "llvm/Analysis/MemoryProfileInfo.h"
30#include "llvm/Analysis/ObjCARCAnalysisUtils.h"
31#include "llvm/Analysis/ObjCARCUtil.h"
32#include "llvm/Analysis/ProfileSummaryInfo.h"
33#include "llvm/Analysis/ValueTracking.h"
34#include "llvm/Analysis/VectorUtils.h"
35#include "llvm/IR/Argument.h"
36#include "llvm/IR/AttributeMask.h"
37#include "llvm/IR/Attributes.h"
38#include "llvm/IR/BasicBlock.h"
39#include "llvm/IR/CFG.h"
40#include "llvm/IR/Constant.h"
41#include "llvm/IR/ConstantRange.h"
42#include "llvm/IR/Constants.h"
43#include "llvm/IR/DataLayout.h"
44#include "llvm/IR/DebugInfo.h"
45#include "llvm/IR/DebugInfoMetadata.h"
46#include "llvm/IR/DebugLoc.h"
47#include "llvm/IR/DerivedTypes.h"
48#include "llvm/IR/Dominators.h"
49#include "llvm/IR/EHPersonalities.h"
50#include "llvm/IR/Function.h"
51#include "llvm/IR/GlobalVariable.h"
52#include "llvm/IR/IRBuilder.h"
53#include "llvm/IR/InlineAsm.h"
54#include "llvm/IR/InstrTypes.h"
55#include "llvm/IR/Instruction.h"
56#include "llvm/IR/Instructions.h"
57#include "llvm/IR/IntrinsicInst.h"
58#include "llvm/IR/Intrinsics.h"
59#include "llvm/IR/LLVMContext.h"
60#include "llvm/IR/MDBuilder.h"
61#include "llvm/IR/Metadata.h"
62#include "llvm/IR/Module.h"
63#include "llvm/IR/PatternMatch.h"
64#include "llvm/IR/ProfDataUtils.h"
65#include "llvm/IR/Type.h"
66#include "llvm/IR/User.h"
67#include "llvm/IR/Value.h"
68#include "llvm/Support/Casting.h"
69#include "llvm/Support/CommandLine.h"
70#include "llvm/Support/ErrorHandling.h"
71#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
72#include "llvm/Transforms/Utils/Cloning.h"
73#include "llvm/Transforms/Utils/Local.h"
74#include "llvm/Transforms/Utils/ValueMapper.h"
75#include <algorithm>
76#include <cassert>
77#include <cstdint>
78#include <deque>
79#include <iterator>
80#include <optional>
81#include <string>
82#include <utility>
83#include <vector>
84
85#define DEBUG_TYPE "inline-function"
86
87using namespace llvm;
88using namespace llvm::memprof;
89
90static cl::opt<bool>
91EnableNoAliasConversion("enable-noalias-to-md-conversion", cl::init(Val: true),
92 cl::Hidden,
93 cl::desc("Convert noalias attributes to metadata during inlining."));
94
95static cl::opt<bool>
96 UseNoAliasIntrinsic("use-noalias-intrinsic-during-inlining", cl::Hidden,
97 cl::init(Val: true),
98 cl::desc("Use the llvm.experimental.noalias.scope.decl "
99 "intrinsic during inlining."));
100
101// Disabled by default, because the added alignment assumptions may increase
102// compile-time and block optimizations. This option is not suitable for use
103// with frontends that emit comprehensive parameter alignment annotations.
104static cl::opt<bool>
105PreserveAlignmentAssumptions("preserve-alignment-assumptions-during-inlining",
106 cl::init(Val: false), cl::Hidden,
107 cl::desc("Convert align attributes to assumptions during inlining."));
108
109static cl::opt<unsigned> InlinerAttributeWindow(
110 "max-inst-checked-for-throw-during-inlining", cl::Hidden,
111 cl::desc("the maximum number of instructions analyzed for may throw during "
112 "attribute inference in inlined body"),
113 cl::init(Val: 4));
114
115namespace {
116
117 /// A class for recording information about inlining a landing pad.
118 class LandingPadInliningInfo {
119 /// Destination of the invoke's unwind.
120 BasicBlock *OuterResumeDest;
121
122 /// Destination for the callee's resume.
123 BasicBlock *InnerResumeDest = nullptr;
124
125 /// LandingPadInst associated with the invoke.
126 LandingPadInst *CallerLPad = nullptr;
127
128 /// PHI for EH values from landingpad insts.
129 PHINode *InnerEHValuesPHI = nullptr;
130
131 SmallVector<Value*, 8> UnwindDestPHIValues;
132
133 public:
134 LandingPadInliningInfo(InvokeInst *II)
135 : OuterResumeDest(II->getUnwindDest()) {
136 // If there are PHI nodes in the unwind destination block, we need to keep
137 // track of which values came into them from the invoke before removing
138 // the edge from this block.
139 BasicBlock *InvokeBB = II->getParent();
140 BasicBlock::iterator I = OuterResumeDest->begin();
141 for (; isa<PHINode>(Val: I); ++I) {
142 // Save the value to use for this edge.
143 PHINode *PHI = cast<PHINode>(Val&: I);
144 UnwindDestPHIValues.push_back(Elt: PHI->getIncomingValueForBlock(BB: InvokeBB));
145 }
146
147 CallerLPad = cast<LandingPadInst>(Val&: I);
148 }
149
150 /// The outer unwind destination is the target of
151 /// unwind edges introduced for calls within the inlined function.
152 BasicBlock *getOuterResumeDest() const {
153 return OuterResumeDest;
154 }
155
156 BasicBlock *getInnerResumeDest();
157
158 LandingPadInst *getLandingPadInst() const { return CallerLPad; }
159
160 /// Forward the 'resume' instruction to the caller's landing pad block.
161 /// When the landing pad block has only one predecessor, this is
162 /// a simple branch. When there is more than one predecessor, we need to
163 /// split the landing pad block after the landingpad instruction and jump
164 /// to there.
165 void forwardResume(ResumeInst *RI,
166 SmallPtrSetImpl<LandingPadInst*> &InlinedLPads);
167
168 /// Add incoming-PHI values to the unwind destination block for the given
169 /// basic block, using the values for the original invoke's source block.
170 void addIncomingPHIValuesFor(BasicBlock *BB) const {
171 addIncomingPHIValuesForInto(src: BB, dest: OuterResumeDest);
172 }
173
174 void addIncomingPHIValuesForInto(BasicBlock *src, BasicBlock *dest) const {
175 BasicBlock::iterator I = dest->begin();
176 for (unsigned i = 0, e = UnwindDestPHIValues.size(); i != e; ++i, ++I) {
177 PHINode *phi = cast<PHINode>(Val&: I);
178 phi->addIncoming(V: UnwindDestPHIValues[i], BB: src);
179 }
180 }
181 };
182} // end anonymous namespace
183
184static IntrinsicInst *getConvergenceEntry(BasicBlock &BB) {
185 BasicBlock::iterator It = BB.getFirstNonPHIIt();
186 while (It != BB.end()) {
187 if (auto *IntrinsicCall = dyn_cast<ConvergenceControlInst>(Val&: It)) {
188 if (IntrinsicCall->isEntry()) {
189 return IntrinsicCall;
190 }
191 }
192 It = std::next(x: It);
193 }
194 return nullptr;
195}
196
197/// Get or create a target for the branch from ResumeInsts.
198BasicBlock *LandingPadInliningInfo::getInnerResumeDest() {
199 if (InnerResumeDest) return InnerResumeDest;
200
201 // Split the landing pad.
202 BasicBlock::iterator SplitPoint = ++CallerLPad->getIterator();
203 InnerResumeDest =
204 OuterResumeDest->splitBasicBlock(I: SplitPoint,
205 BBName: OuterResumeDest->getName() + ".body");
206
207 // The number of incoming edges we expect to the inner landing pad.
208 const unsigned PHICapacity = 2;
209
210 // Create corresponding new PHIs for all the PHIs in the outer landing pad.
211 BasicBlock::iterator InsertPoint = InnerResumeDest->begin();
212 BasicBlock::iterator I = OuterResumeDest->begin();
213 for (unsigned i = 0, e = UnwindDestPHIValues.size(); i != e; ++i, ++I) {
214 PHINode *OuterPHI = cast<PHINode>(Val&: I);
215 PHINode *InnerPHI = PHINode::Create(Ty: OuterPHI->getType(), NumReservedValues: PHICapacity,
216 NameStr: OuterPHI->getName() + ".lpad-body");
217 InnerPHI->insertBefore(InsertPos: InsertPoint);
218 OuterPHI->replaceAllUsesWith(V: InnerPHI);
219 InnerPHI->addIncoming(V: OuterPHI, BB: OuterResumeDest);
220 }
221
222 // Create a PHI for the exception values.
223 InnerEHValuesPHI =
224 PHINode::Create(Ty: CallerLPad->getType(), NumReservedValues: PHICapacity, NameStr: "eh.lpad-body");
225 InnerEHValuesPHI->insertBefore(InsertPos: InsertPoint);
226 CallerLPad->replaceAllUsesWith(V: InnerEHValuesPHI);
227 InnerEHValuesPHI->addIncoming(V: CallerLPad, BB: OuterResumeDest);
228
229 // All done.
230 return InnerResumeDest;
231}
232
233/// Forward the 'resume' instruction to the caller's landing pad block.
234/// When the landing pad block has only one predecessor, this is a simple
235/// branch. When there is more than one predecessor, we need to split the
236/// landing pad block after the landingpad instruction and jump to there.
237void LandingPadInliningInfo::forwardResume(
238 ResumeInst *RI, SmallPtrSetImpl<LandingPadInst *> &InlinedLPads) {
239 BasicBlock *Dest = getInnerResumeDest();
240 BasicBlock *Src = RI->getParent();
241
242 auto *BI = UncondBrInst::Create(Target: Dest, InsertBefore: Src);
243 BI->setDebugLoc(RI->getDebugLoc());
244
245 // Update the PHIs in the destination. They were inserted in an order which
246 // makes this work.
247 addIncomingPHIValuesForInto(src: Src, dest: Dest);
248
249 InnerEHValuesPHI->addIncoming(V: RI->getOperand(i_nocapture: 0), BB: Src);
250 RI->eraseFromParent();
251}
252
253/// Helper for getUnwindDestToken/getUnwindDestTokenHelper.
254static Value *getParentPad(Value *EHPad) {
255 if (auto *FPI = dyn_cast<FuncletPadInst>(Val: EHPad))
256 return FPI->getParentPad();
257 return cast<CatchSwitchInst>(Val: EHPad)->getParentPad();
258}
259
260using UnwindDestMemoTy = DenseMap<Instruction *, Value *>;
261
262/// Helper for getUnwindDestToken that does the descendant-ward part of
263/// the search.
264static Value *getUnwindDestTokenHelper(Instruction *EHPad,
265 UnwindDestMemoTy &MemoMap) {
266 SmallVector<Instruction *, 8> Worklist(1, EHPad);
267
268 while (!Worklist.empty()) {
269 Instruction *CurrentPad = Worklist.pop_back_val();
270 // We only put pads on the worklist that aren't in the MemoMap. When
271 // we find an unwind dest for a pad we may update its ancestors, but
272 // the queue only ever contains uncles/great-uncles/etc. of CurrentPad,
273 // so they should never get updated while queued on the worklist.
274 assert(!MemoMap.count(CurrentPad));
275 Value *UnwindDestToken = nullptr;
276 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val: CurrentPad)) {
277 if (CatchSwitch->hasUnwindDest()) {
278 UnwindDestToken = &*CatchSwitch->getUnwindDest()->getFirstNonPHIIt();
279 } else {
280 // Catchswitch doesn't have a 'nounwind' variant, and one might be
281 // annotated as "unwinds to caller" when really it's nounwind (see
282 // e.g. SimplifyCFGOpt::SimplifyUnreachable), so we can't infer the
283 // parent's unwind dest from this. We can check its catchpads'
284 // descendants, since they might include a cleanuppad with an
285 // "unwinds to caller" cleanupret, which can be trusted.
286 for (auto HI = CatchSwitch->handler_begin(),
287 HE = CatchSwitch->handler_end();
288 HI != HE && !UnwindDestToken; ++HI) {
289 BasicBlock *HandlerBlock = *HI;
290 auto *CatchPad =
291 cast<CatchPadInst>(Val: &*HandlerBlock->getFirstNonPHIIt());
292 for (User *Child : CatchPad->users()) {
293 // Intentionally ignore invokes here -- since the catchswitch is
294 // marked "unwind to caller", it would be a verifier error if it
295 // contained an invoke which unwinds out of it, so any invoke we'd
296 // encounter must unwind to some child of the catch.
297 if (!isa<CleanupPadInst>(Val: Child) && !isa<CatchSwitchInst>(Val: Child))
298 continue;
299
300 Instruction *ChildPad = cast<Instruction>(Val: Child);
301 auto Memo = MemoMap.find(Val: ChildPad);
302 if (Memo == MemoMap.end()) {
303 // Haven't figured out this child pad yet; queue it.
304 Worklist.push_back(Elt: ChildPad);
305 continue;
306 }
307 // We've already checked this child, but might have found that
308 // it offers no proof either way.
309 Value *ChildUnwindDestToken = Memo->second;
310 if (!ChildUnwindDestToken)
311 continue;
312 // We already know the child's unwind dest, which can either
313 // be ConstantTokenNone to indicate unwind to caller, or can
314 // be another child of the catchpad. Only the former indicates
315 // the unwind dest of the catchswitch.
316 if (isa<ConstantTokenNone>(Val: ChildUnwindDestToken)) {
317 UnwindDestToken = ChildUnwindDestToken;
318 break;
319 }
320 assert(getParentPad(ChildUnwindDestToken) == CatchPad);
321 }
322 }
323 }
324 } else {
325 auto *CleanupPad = cast<CleanupPadInst>(Val: CurrentPad);
326 for (User *U : CleanupPad->users()) {
327 if (auto *CleanupRet = dyn_cast<CleanupReturnInst>(Val: U)) {
328 if (BasicBlock *RetUnwindDest = CleanupRet->getUnwindDest())
329 UnwindDestToken = &*RetUnwindDest->getFirstNonPHIIt();
330 else
331 UnwindDestToken = ConstantTokenNone::get(Context&: CleanupPad->getContext());
332 break;
333 }
334 Value *ChildUnwindDestToken;
335 if (auto *Invoke = dyn_cast<InvokeInst>(Val: U)) {
336 ChildUnwindDestToken = &*Invoke->getUnwindDest()->getFirstNonPHIIt();
337 } else if (isa<CleanupPadInst>(Val: U) || isa<CatchSwitchInst>(Val: U)) {
338 Instruction *ChildPad = cast<Instruction>(Val: U);
339 auto Memo = MemoMap.find(Val: ChildPad);
340 if (Memo == MemoMap.end()) {
341 // Haven't resolved this child yet; queue it and keep searching.
342 Worklist.push_back(Elt: ChildPad);
343 continue;
344 }
345 // We've checked this child, but still need to ignore it if it
346 // had no proof either way.
347 ChildUnwindDestToken = Memo->second;
348 if (!ChildUnwindDestToken)
349 continue;
350 } else {
351 // Not a relevant user of the cleanuppad
352 continue;
353 }
354 // In a well-formed program, the child/invoke must either unwind to
355 // an(other) child of the cleanup, or exit the cleanup. In the
356 // first case, continue searching.
357 if (isa<Instruction>(Val: ChildUnwindDestToken) &&
358 getParentPad(EHPad: ChildUnwindDestToken) == CleanupPad)
359 continue;
360 UnwindDestToken = ChildUnwindDestToken;
361 break;
362 }
363 }
364 // If we haven't found an unwind dest for CurrentPad, we may have queued its
365 // children, so move on to the next in the worklist.
366 if (!UnwindDestToken)
367 continue;
368
369 // Now we know that CurrentPad unwinds to UnwindDestToken. It also exits
370 // any ancestors of CurrentPad up to but not including UnwindDestToken's
371 // parent pad. Record this in the memo map, and check to see if the
372 // original EHPad being queried is one of the ones exited.
373 Value *UnwindParent;
374 if (auto *UnwindPad = dyn_cast<Instruction>(Val: UnwindDestToken))
375 UnwindParent = getParentPad(EHPad: UnwindPad);
376 else
377 UnwindParent = nullptr;
378 bool ExitedOriginalPad = false;
379 for (Instruction *ExitedPad = CurrentPad;
380 ExitedPad && ExitedPad != UnwindParent;
381 ExitedPad = dyn_cast<Instruction>(Val: getParentPad(EHPad: ExitedPad))) {
382 // Skip over catchpads since they just follow their catchswitches.
383 if (isa<CatchPadInst>(Val: ExitedPad))
384 continue;
385 MemoMap[ExitedPad] = UnwindDestToken;
386 ExitedOriginalPad |= (ExitedPad == EHPad);
387 }
388
389 if (ExitedOriginalPad)
390 return UnwindDestToken;
391
392 // Continue the search.
393 }
394
395 // No definitive information is contained within this funclet.
396 return nullptr;
397}
398
399/// Given an EH pad, find where it unwinds. If it unwinds to an EH pad,
400/// return that pad instruction. If it unwinds to caller, return
401/// ConstantTokenNone. If it does not have a definitive unwind destination,
402/// return nullptr.
403///
404/// This routine gets invoked for calls in funclets in inlinees when inlining
405/// an invoke. Since many funclets don't have calls inside them, it's queried
406/// on-demand rather than building a map of pads to unwind dests up front.
407/// Determining a funclet's unwind dest may require recursively searching its
408/// descendants, and also ancestors and cousins if the descendants don't provide
409/// an answer. Since most funclets will have their unwind dest immediately
410/// available as the unwind dest of a catchswitch or cleanupret, this routine
411/// searches top-down from the given pad and then up. To avoid worst-case
412/// quadratic run-time given that approach, it uses a memo map to avoid
413/// re-processing funclet trees. The callers that rewrite the IR as they go
414/// take advantage of this, for correctness, by checking/forcing rewritten
415/// pads' entries to match the original callee view.
416static Value *getUnwindDestToken(Instruction *EHPad,
417 UnwindDestMemoTy &MemoMap) {
418 // Catchpads unwind to the same place as their catchswitch;
419 // redirct any queries on catchpads so the code below can
420 // deal with just catchswitches and cleanuppads.
421 if (auto *CPI = dyn_cast<CatchPadInst>(Val: EHPad))
422 EHPad = CPI->getCatchSwitch();
423
424 // Check if we've already determined the unwind dest for this pad.
425 auto Memo = MemoMap.find(Val: EHPad);
426 if (Memo != MemoMap.end())
427 return Memo->second;
428
429 // Search EHPad and, if necessary, its descendants.
430 Value *UnwindDestToken = getUnwindDestTokenHelper(EHPad, MemoMap);
431 assert((UnwindDestToken == nullptr) != (MemoMap.count(EHPad) != 0));
432 if (UnwindDestToken)
433 return UnwindDestToken;
434
435 // No information is available for this EHPad from itself or any of its
436 // descendants. An unwind all the way out to a pad in the caller would
437 // need also to agree with the unwind dest of the parent funclet, so
438 // search up the chain to try to find a funclet with information. Put
439 // null entries in the memo map to avoid re-processing as we go up.
440 MemoMap[EHPad] = nullptr;
441#ifndef NDEBUG
442 SmallPtrSet<Instruction *, 4> TempMemos;
443 TempMemos.insert(EHPad);
444#endif
445 Instruction *LastUselessPad = EHPad;
446 Value *AncestorToken;
447 for (AncestorToken = getParentPad(EHPad);
448 auto *AncestorPad = dyn_cast<Instruction>(Val: AncestorToken);
449 AncestorToken = getParentPad(EHPad: AncestorToken)) {
450 // Skip over catchpads since they just follow their catchswitches.
451 if (isa<CatchPadInst>(Val: AncestorPad))
452 continue;
453 // If the MemoMap had an entry mapping AncestorPad to nullptr, since we
454 // haven't yet called getUnwindDestTokenHelper for AncestorPad in this
455 // call to getUnwindDestToken, that would mean that AncestorPad had no
456 // information in itself, its descendants, or its ancestors. If that
457 // were the case, then we should also have recorded the lack of information
458 // for the descendant that we're coming from. So assert that we don't
459 // find a null entry in the MemoMap for AncestorPad.
460 assert(!MemoMap.count(AncestorPad) || MemoMap[AncestorPad]);
461 auto AncestorMemo = MemoMap.find(Val: AncestorPad);
462 if (AncestorMemo == MemoMap.end()) {
463 UnwindDestToken = getUnwindDestTokenHelper(EHPad: AncestorPad, MemoMap);
464 } else {
465 UnwindDestToken = AncestorMemo->second;
466 }
467 if (UnwindDestToken)
468 break;
469 LastUselessPad = AncestorPad;
470 MemoMap[LastUselessPad] = nullptr;
471#ifndef NDEBUG
472 TempMemos.insert(LastUselessPad);
473#endif
474 }
475
476 // We know that getUnwindDestTokenHelper was called on LastUselessPad and
477 // returned nullptr (and likewise for EHPad and any of its ancestors up to
478 // LastUselessPad), so LastUselessPad has no information from below. Since
479 // getUnwindDestTokenHelper must investigate all downward paths through
480 // no-information nodes to prove that a node has no information like this,
481 // and since any time it finds information it records it in the MemoMap for
482 // not just the immediately-containing funclet but also any ancestors also
483 // exited, it must be the case that, walking downward from LastUselessPad,
484 // visiting just those nodes which have not been mapped to an unwind dest
485 // by getUnwindDestTokenHelper (the nullptr TempMemos notwithstanding, since
486 // they are just used to keep getUnwindDestTokenHelper from repeating work),
487 // any node visited must have been exhaustively searched with no information
488 // for it found.
489 SmallVector<Instruction *, 8> Worklist(1, LastUselessPad);
490 while (!Worklist.empty()) {
491 Instruction *UselessPad = Worklist.pop_back_val();
492 auto Memo = MemoMap.find(Val: UselessPad);
493 if (Memo != MemoMap.end() && Memo->second) {
494 // Here the name 'UselessPad' is a bit of a misnomer, because we've found
495 // that it is a funclet that does have information about unwinding to
496 // a particular destination; its parent was a useless pad.
497 // Since its parent has no information, the unwind edge must not escape
498 // the parent, and must target a sibling of this pad. This local unwind
499 // gives us no information about EHPad. Leave it and the subtree rooted
500 // at it alone.
501 assert(getParentPad(Memo->second) == getParentPad(UselessPad));
502 continue;
503 }
504 // We know we don't have information for UselesPad. If it has an entry in
505 // the MemoMap (mapping it to nullptr), it must be one of the TempMemos
506 // added on this invocation of getUnwindDestToken; if a previous invocation
507 // recorded nullptr, it would have had to prove that the ancestors of
508 // UselessPad, which include LastUselessPad, had no information, and that
509 // in turn would have required proving that the descendants of
510 // LastUselesPad, which include EHPad, have no information about
511 // LastUselessPad, which would imply that EHPad was mapped to nullptr in
512 // the MemoMap on that invocation, which isn't the case if we got here.
513 assert(!MemoMap.count(UselessPad) || TempMemos.count(UselessPad));
514 // Assert as we enumerate users that 'UselessPad' doesn't have any unwind
515 // information that we'd be contradicting by making a map entry for it
516 // (which is something that getUnwindDestTokenHelper must have proved for
517 // us to get here). Just assert on is direct users here; the checks in
518 // this downward walk at its descendants will verify that they don't have
519 // any unwind edges that exit 'UselessPad' either (i.e. they either have no
520 // unwind edges or unwind to a sibling).
521 MemoMap[UselessPad] = UnwindDestToken;
522 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val: UselessPad)) {
523 assert(CatchSwitch->getUnwindDest() == nullptr && "Expected useless pad");
524 for (BasicBlock *HandlerBlock : CatchSwitch->handlers()) {
525 auto *CatchPad = &*HandlerBlock->getFirstNonPHIIt();
526 for (User *U : CatchPad->users()) {
527 assert((!isa<InvokeInst>(U) ||
528 (getParentPad(&*cast<InvokeInst>(U)
529 ->getUnwindDest()
530 ->getFirstNonPHIIt()) == CatchPad)) &&
531 "Expected useless pad");
532 if (isa<CatchSwitchInst>(Val: U) || isa<CleanupPadInst>(Val: U))
533 Worklist.push_back(Elt: cast<Instruction>(Val: U));
534 }
535 }
536 } else {
537 assert(isa<CleanupPadInst>(UselessPad));
538 for (User *U : UselessPad->users()) {
539 assert(!isa<CleanupReturnInst>(U) && "Expected useless pad");
540 assert(
541 (!isa<InvokeInst>(U) ||
542 (getParentPad(
543 &*cast<InvokeInst>(U)->getUnwindDest()->getFirstNonPHIIt()) ==
544 UselessPad)) &&
545 "Expected useless pad");
546 if (isa<CatchSwitchInst>(Val: U) || isa<CleanupPadInst>(Val: U))
547 Worklist.push_back(Elt: cast<Instruction>(Val: U));
548 }
549 }
550 }
551
552 return UnwindDestToken;
553}
554
555/// When we inline a basic block into an invoke,
556/// we have to turn all of the calls that can throw into invokes.
557/// This function analyze BB to see if there are any calls, and if so,
558/// it rewrites them to be invokes that jump to InvokeDest and fills in the PHI
559/// nodes in that block with the values specified in InvokeDestPHIValues.
560static BasicBlock *HandleCallsInBlockInlinedThroughInvoke(
561 BasicBlock *BB, BasicBlock *UnwindEdge,
562 SmallSetVector<const Value *, 4> &OriginallyIndirectCalls,
563 UnwindDestMemoTy *FuncletUnwindMap = nullptr) {
564 for (Instruction &I : llvm::make_early_inc_range(Range&: *BB)) {
565 // We only need to check for function calls: inlined invoke
566 // instructions require no special handling.
567 CallInst *CI = dyn_cast<CallInst>(Val: &I);
568
569 if (!CI || CI->doesNotThrow())
570 continue;
571
572 // We do not need to (and in fact, cannot) convert possibly throwing calls
573 // to @llvm.experimental_deoptimize (resp. @llvm.experimental.guard) into
574 // invokes. The caller's "segment" of the deoptimization continuation
575 // attached to the newly inlined @llvm.experimental_deoptimize
576 // (resp. @llvm.experimental.guard) call should contain the exception
577 // handling logic, if any.
578 if (auto *F = CI->getCalledFunction())
579 if (F->getIntrinsicID() == Intrinsic::experimental_deoptimize ||
580 F->getIntrinsicID() == Intrinsic::experimental_guard)
581 continue;
582
583 if (auto FuncletBundle = CI->getOperandBundle(ID: LLVMContext::OB_funclet)) {
584 // This call is nested inside a funclet. If that funclet has an unwind
585 // destination within the inlinee, then unwinding out of this call would
586 // be UB. Rewriting this call to an invoke which targets the inlined
587 // invoke's unwind dest would give the call's parent funclet multiple
588 // unwind destinations, which is something that subsequent EH table
589 // generation can't handle and that the veirifer rejects. So when we
590 // see such a call, leave it as a call.
591 auto *FuncletPad = cast<Instruction>(Val: FuncletBundle->Inputs[0]);
592 Value *UnwindDestToken =
593 getUnwindDestToken(EHPad: FuncletPad, MemoMap&: *FuncletUnwindMap);
594 if (UnwindDestToken && !isa<ConstantTokenNone>(Val: UnwindDestToken))
595 continue;
596#ifndef NDEBUG
597 Instruction *MemoKey;
598 if (auto *CatchPad = dyn_cast<CatchPadInst>(FuncletPad))
599 MemoKey = CatchPad->getCatchSwitch();
600 else
601 MemoKey = FuncletPad;
602 assert(FuncletUnwindMap->count(MemoKey) &&
603 (*FuncletUnwindMap)[MemoKey] == UnwindDestToken &&
604 "must get memoized to avoid confusing later searches");
605#endif // NDEBUG
606 }
607
608 bool WasIndirect = OriginallyIndirectCalls.remove(X: CI);
609 changeToInvokeAndSplitBasicBlock(CI, UnwindEdge);
610 if (WasIndirect)
611 OriginallyIndirectCalls.insert(X: BB->getTerminator());
612 return BB;
613 }
614 return nullptr;
615}
616
617/// If we inlined an invoke site, we need to convert calls
618/// in the body of the inlined function into invokes.
619///
620/// II is the invoke instruction being inlined. FirstNewBlock is the first
621/// block of the inlined code (the last block is the end of the function),
622/// and InlineCodeInfo is information about the code that got inlined.
623static void HandleInlinedLandingPad(InvokeInst *II, BasicBlock *FirstNewBlock,
624 ClonedCodeInfo &InlinedCodeInfo) {
625 BasicBlock *InvokeDest = II->getUnwindDest();
626
627 Function *Caller = FirstNewBlock->getParent();
628
629 // The inlined code is currently at the end of the function, scan from the
630 // start of the inlined code to its end, checking for stuff we need to
631 // rewrite.
632 LandingPadInliningInfo Invoke(II);
633
634 // Get all of the inlined landing pad instructions.
635 SmallPtrSet<LandingPadInst*, 16> InlinedLPads;
636 for (Function::iterator I = FirstNewBlock->getIterator(), E = Caller->end();
637 I != E; ++I)
638 if (InvokeInst *II = dyn_cast<InvokeInst>(Val: I->getTerminator()))
639 InlinedLPads.insert(Ptr: II->getLandingPadInst());
640
641 // Append the clauses from the outer landing pad instruction into the inlined
642 // landing pad instructions.
643 LandingPadInst *OuterLPad = Invoke.getLandingPadInst();
644 for (LandingPadInst *InlinedLPad : InlinedLPads) {
645 unsigned OuterNum = OuterLPad->getNumClauses();
646 InlinedLPad->reserveClauses(Size: OuterNum);
647 for (unsigned OuterIdx = 0; OuterIdx != OuterNum; ++OuterIdx)
648 InlinedLPad->addClause(ClauseVal: OuterLPad->getClause(Idx: OuterIdx));
649 if (OuterLPad->isCleanup())
650 InlinedLPad->setCleanup(true);
651 }
652
653 for (Function::iterator BB = FirstNewBlock->getIterator(), E = Caller->end();
654 BB != E; ++BB) {
655 if (InlinedCodeInfo.ContainsCalls)
656 if (BasicBlock *NewBB = HandleCallsInBlockInlinedThroughInvoke(
657 BB: &*BB, UnwindEdge: Invoke.getOuterResumeDest(),
658 OriginallyIndirectCalls&: InlinedCodeInfo.OriginallyIndirectCalls))
659 // Update any PHI nodes in the exceptional block to indicate that there
660 // is now a new entry in them.
661 Invoke.addIncomingPHIValuesFor(BB: NewBB);
662
663 // Forward any resumes that are remaining here.
664 if (ResumeInst *RI = dyn_cast<ResumeInst>(Val: BB->getTerminator()))
665 Invoke.forwardResume(RI, InlinedLPads);
666 }
667
668 // Now that everything is happy, we have one final detail. The PHI nodes in
669 // the exception destination block still have entries due to the original
670 // invoke instruction. Eliminate these entries (which might even delete the
671 // PHI node) now.
672 InvokeDest->removePredecessor(Pred: II->getParent());
673}
674
675/// If we inlined an invoke site, we need to convert calls
676/// in the body of the inlined function into invokes.
677///
678/// II is the invoke instruction being inlined. FirstNewBlock is the first
679/// block of the inlined code (the last block is the end of the function),
680/// and InlineCodeInfo is information about the code that got inlined.
681static void HandleInlinedEHPad(InvokeInst *II, BasicBlock *FirstNewBlock,
682 ClonedCodeInfo &InlinedCodeInfo) {
683 BasicBlock *UnwindDest = II->getUnwindDest();
684 Function *Caller = FirstNewBlock->getParent();
685
686 assert(UnwindDest->getFirstNonPHIIt()->isEHPad() && "unexpected BasicBlock!");
687
688 // If there are PHI nodes in the unwind destination block, we need to keep
689 // track of which values came into them from the invoke before removing the
690 // edge from this block.
691 SmallVector<Value *, 8> UnwindDestPHIValues;
692 BasicBlock *InvokeBB = II->getParent();
693 for (PHINode &PHI : UnwindDest->phis()) {
694 // Save the value to use for this edge.
695 UnwindDestPHIValues.push_back(Elt: PHI.getIncomingValueForBlock(BB: InvokeBB));
696 }
697
698 // Add incoming-PHI values to the unwind destination block for the given basic
699 // block, using the values for the original invoke's source block.
700 auto UpdatePHINodes = [&](BasicBlock *Src) {
701 BasicBlock::iterator I = UnwindDest->begin();
702 for (Value *V : UnwindDestPHIValues) {
703 PHINode *PHI = cast<PHINode>(Val&: I);
704 PHI->addIncoming(V, BB: Src);
705 ++I;
706 }
707 };
708
709 // This connects all the instructions which 'unwind to caller' to the invoke
710 // destination.
711 UnwindDestMemoTy FuncletUnwindMap;
712 for (Function::iterator BB = FirstNewBlock->getIterator(), E = Caller->end();
713 BB != E; ++BB) {
714 if (auto *CRI = dyn_cast<CleanupReturnInst>(Val: BB->getTerminator())) {
715 if (CRI->unwindsToCaller()) {
716 auto *CleanupPad = CRI->getCleanupPad();
717 CleanupReturnInst::Create(CleanupPad, UnwindBB: UnwindDest, InsertBefore: CRI->getIterator());
718 CRI->eraseFromParent();
719 UpdatePHINodes(&*BB);
720 // Finding a cleanupret with an unwind destination would confuse
721 // subsequent calls to getUnwindDestToken, so map the cleanuppad
722 // to short-circuit any such calls and recognize this as an "unwind
723 // to caller" cleanup.
724 assert(!FuncletUnwindMap.count(CleanupPad) ||
725 isa<ConstantTokenNone>(FuncletUnwindMap[CleanupPad]));
726 FuncletUnwindMap[CleanupPad] =
727 ConstantTokenNone::get(Context&: Caller->getContext());
728 }
729 }
730
731 BasicBlock::iterator I = BB->getFirstNonPHIIt();
732 if (!I->isEHPad())
733 continue;
734
735 Instruction *Replacement = nullptr;
736 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val&: I)) {
737 if (CatchSwitch->unwindsToCaller()) {
738 Value *UnwindDestToken;
739 if (auto *ParentPad =
740 dyn_cast<Instruction>(Val: CatchSwitch->getParentPad())) {
741 // This catchswitch is nested inside another funclet. If that
742 // funclet has an unwind destination within the inlinee, then
743 // unwinding out of this catchswitch would be UB. Rewriting this
744 // catchswitch to unwind to the inlined invoke's unwind dest would
745 // give the parent funclet multiple unwind destinations, which is
746 // something that subsequent EH table generation can't handle and
747 // that the veirifer rejects. So when we see such a call, leave it
748 // as "unwind to caller".
749 UnwindDestToken = getUnwindDestToken(EHPad: ParentPad, MemoMap&: FuncletUnwindMap);
750 if (UnwindDestToken && !isa<ConstantTokenNone>(Val: UnwindDestToken))
751 continue;
752 } else {
753 // This catchswitch has no parent to inherit constraints from, and
754 // none of its descendants can have an unwind edge that exits it and
755 // targets another funclet in the inlinee. It may or may not have a
756 // descendant that definitively has an unwind to caller. In either
757 // case, we'll have to assume that any unwinds out of it may need to
758 // be routed to the caller, so treat it as though it has a definitive
759 // unwind to caller.
760 UnwindDestToken = ConstantTokenNone::get(Context&: Caller->getContext());
761 }
762 auto *NewCatchSwitch = CatchSwitchInst::Create(
763 ParentPad: CatchSwitch->getParentPad(), UnwindDest,
764 NumHandlers: CatchSwitch->getNumHandlers(), NameStr: CatchSwitch->getName(),
765 InsertBefore: CatchSwitch->getIterator());
766 for (BasicBlock *PadBB : CatchSwitch->handlers())
767 NewCatchSwitch->addHandler(Dest: PadBB);
768 // Propagate info for the old catchswitch over to the new one in
769 // the unwind map. This also serves to short-circuit any subsequent
770 // checks for the unwind dest of this catchswitch, which would get
771 // confused if they found the outer handler in the callee.
772 FuncletUnwindMap[NewCatchSwitch] = UnwindDestToken;
773 Replacement = NewCatchSwitch;
774 }
775 } else if (!isa<FuncletPadInst>(Val: I)) {
776 llvm_unreachable("unexpected EHPad!");
777 }
778
779 if (Replacement) {
780 Replacement->takeName(V: &*I);
781 I->replaceAllUsesWith(V: Replacement);
782 I->eraseFromParent();
783 UpdatePHINodes(&*BB);
784 }
785 }
786
787 if (InlinedCodeInfo.ContainsCalls)
788 for (Function::iterator BB = FirstNewBlock->getIterator(),
789 E = Caller->end();
790 BB != E; ++BB)
791 if (BasicBlock *NewBB = HandleCallsInBlockInlinedThroughInvoke(
792 BB: &*BB, UnwindEdge: UnwindDest, OriginallyIndirectCalls&: InlinedCodeInfo.OriginallyIndirectCalls,
793 FuncletUnwindMap: &FuncletUnwindMap))
794 // Update any PHI nodes in the exceptional block to indicate that there
795 // is now a new entry in them.
796 UpdatePHINodes(NewBB);
797
798 // Now that everything is happy, we have one final detail. The PHI nodes in
799 // the exception destination block still have entries due to the original
800 // invoke instruction. Eliminate these entries (which might even delete the
801 // PHI node) now.
802 UnwindDest->removePredecessor(Pred: InvokeBB);
803}
804
805static bool haveCommonPrefix(MDNode *MIBStackContext,
806 MDNode *CallsiteStackContext) {
807 assert(MIBStackContext->getNumOperands() > 0 &&
808 CallsiteStackContext->getNumOperands() > 0);
809 // Because of the context trimming performed during matching, the callsite
810 // context could have more stack ids than the MIB. We match up to the end of
811 // the shortest stack context.
812 for (auto MIBStackIter = MIBStackContext->op_begin(),
813 CallsiteStackIter = CallsiteStackContext->op_begin();
814 MIBStackIter != MIBStackContext->op_end() &&
815 CallsiteStackIter != CallsiteStackContext->op_end();
816 MIBStackIter++, CallsiteStackIter++) {
817 auto *Val1 = mdconst::dyn_extract<ConstantInt>(MD: *MIBStackIter);
818 auto *Val2 = mdconst::dyn_extract<ConstantInt>(MD: *CallsiteStackIter);
819 assert(Val1 && Val2);
820 if (Val1->getZExtValue() != Val2->getZExtValue())
821 return false;
822 }
823 return true;
824}
825
826static void removeMemProfMetadata(CallBase *Call) {
827 Call->setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
828}
829
830static void removeCallsiteMetadata(CallBase *Call) {
831 Call->setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
832}
833
834static void updateMemprofMetadata(CallBase *CI,
835 const std::vector<Metadata *> &MIBList,
836 OptimizationRemarkEmitter *ORE) {
837 assert(!MIBList.empty());
838 // Remove existing memprof, which will either be replaced or may not be needed
839 // if we are able to use a single allocation type function attribute.
840 removeMemProfMetadata(Call: CI);
841 CallStackTrie CallStack(ORE);
842 for (Metadata *MIB : MIBList)
843 CallStack.addCallStack(MIB: cast<MDNode>(Val: MIB));
844 bool MemprofMDAttached = CallStack.buildAndAttachMIBMetadata(CI);
845 assert(MemprofMDAttached == CI->hasMetadata(LLVMContext::MD_memprof));
846 if (!MemprofMDAttached)
847 // If we used a function attribute remove the callsite metadata as well.
848 removeCallsiteMetadata(Call: CI);
849}
850
851// Update the metadata on the inlined copy ClonedCall of a call OrigCall in the
852// inlined callee body, based on the callsite metadata InlinedCallsiteMD from
853// the call that was inlined.
854static void propagateMemProfHelper(const CallBase *OrigCall,
855 CallBase *ClonedCall,
856 MDNode *InlinedCallsiteMD,
857 OptimizationRemarkEmitter *ORE) {
858 MDNode *OrigCallsiteMD = ClonedCall->getMetadata(KindID: LLVMContext::MD_callsite);
859 MDNode *ClonedCallsiteMD = nullptr;
860 // Check if the call originally had callsite metadata, and update it for the
861 // new call in the inlined body.
862 if (OrigCallsiteMD) {
863 // The cloned call's context is now the concatenation of the original call's
864 // callsite metadata and the callsite metadata on the call where it was
865 // inlined.
866 ClonedCallsiteMD = MDNode::concatenate(A: OrigCallsiteMD, B: InlinedCallsiteMD);
867 ClonedCall->setMetadata(KindID: LLVMContext::MD_callsite, Node: ClonedCallsiteMD);
868 }
869
870 // Update any memprof metadata on the cloned call.
871 MDNode *OrigMemProfMD = ClonedCall->getMetadata(KindID: LLVMContext::MD_memprof);
872 if (!OrigMemProfMD)
873 return;
874 // We currently expect that allocations with memprof metadata also have
875 // callsite metadata for the allocation's part of the context.
876 assert(OrigCallsiteMD);
877
878 // New call's MIB list.
879 std::vector<Metadata *> NewMIBList;
880
881 // For each MIB metadata, check if its call stack context starts with the
882 // new clone's callsite metadata. If so, that MIB goes onto the cloned call in
883 // the inlined body. If not, it stays on the out-of-line original call.
884 for (auto &MIBOp : OrigMemProfMD->operands()) {
885 MDNode *MIB = dyn_cast<MDNode>(Val: MIBOp);
886 // Stack is first operand of MIB.
887 MDNode *StackMD = getMIBStackNode(MIB);
888 assert(StackMD);
889 // See if the new cloned callsite context matches this profiled context.
890 if (haveCommonPrefix(MIBStackContext: StackMD, CallsiteStackContext: ClonedCallsiteMD))
891 // Add it to the cloned call's MIB list.
892 NewMIBList.push_back(x: MIB);
893 }
894 if (NewMIBList.empty()) {
895 removeMemProfMetadata(Call: ClonedCall);
896 removeCallsiteMetadata(Call: ClonedCall);
897 return;
898 }
899 if (NewMIBList.size() < OrigMemProfMD->getNumOperands())
900 updateMemprofMetadata(CI: ClonedCall, MIBList: NewMIBList, ORE);
901}
902
903// Update memprof related metadata (!memprof and !callsite) based on the
904// inlining of Callee into the callsite at CB. The updates include merging the
905// inlined callee's callsite metadata with that of the inlined call,
906// and moving the subset of any memprof contexts to the inlined callee
907// allocations if they match the new inlined call stack.
908static void
909propagateMemProfMetadata(Function *Callee, CallBase &CB,
910 bool ContainsMemProfMetadata,
911 const ValueMap<const Value *, WeakTrackingVH> &VMap,
912 OptimizationRemarkEmitter *ORE) {
913 MDNode *CallsiteMD = CB.getMetadata(KindID: LLVMContext::MD_callsite);
914 // Only need to update if the inlined callsite had callsite metadata, or if
915 // there was any memprof metadata inlined.
916 if (!CallsiteMD && !ContainsMemProfMetadata)
917 return;
918
919 // Propagate metadata onto the cloned calls in the inlined callee.
920 for (const auto &Entry : VMap) {
921 // See if this is a call that has been inlined and remapped, and not
922 // simplified away in the process.
923 auto *OrigCall = dyn_cast_or_null<CallBase>(Val: Entry.first);
924 auto *ClonedCall = dyn_cast_or_null<CallBase>(Val: Entry.second);
925 if (!OrigCall || !ClonedCall)
926 continue;
927 // If the inlined callsite did not have any callsite metadata, then it isn't
928 // involved in any profiled call contexts, and we can remove any memprof
929 // metadata on the cloned call.
930 if (!CallsiteMD) {
931 removeMemProfMetadata(Call: ClonedCall);
932 removeCallsiteMetadata(Call: ClonedCall);
933 continue;
934 }
935 propagateMemProfHelper(OrigCall, ClonedCall, InlinedCallsiteMD: CallsiteMD, ORE);
936 }
937}
938
939/// Collect all calls that produce RetVal, following only pointer-preserving
940/// instructions (cast, phi, select).
941static void collectPointerReturningCalls(Value *RetVal,
942 SmallVectorImpl<CallBase *> &Out) {
943 SmallVector<Value *, 8> Worklist{RetVal};
944 SmallPtrSet<Value *, 8> Visited;
945 while (!Worklist.empty()) {
946 Value *V = Worklist.pop_back_val();
947 if (!V->getType()->isPointerTy() || !Visited.insert(Ptr: V).second)
948 continue;
949 if (auto *CB = dyn_cast<CallBase>(Val: V))
950 Out.push_back(Elt: CB);
951 else if (isa<BitCastInst, AddrSpaceCastInst>(Val: V))
952 Worklist.push_back(Elt: cast<CastInst>(Val: V)->getOperand(i_nocapture: 0));
953 else if (auto *PN = dyn_cast<PHINode>(Val: V))
954 append_range(C&: Worklist, R: PN->incoming_values());
955 else if (auto *SI = dyn_cast<SelectInst>(Val: V)) {
956 Worklist.push_back(Elt: SI->getTrueValue());
957 Worklist.push_back(Elt: SI->getFalseValue());
958 }
959 }
960}
961
962/// When inlining a call that carries !alloc_token metadata, propagate that
963/// metadata onto calls exposed by inlining the wrapper body. Propagation is
964/// restricted to return-value producing calls, which avoids instrumenting
965/// unrelated calls in the wrapper body.
966static void
967propagateAllocTokenMetadata(Function *CalledFunc, CallBase &CB,
968 const ValueMap<const Value *, WeakTrackingVH> &VMap,
969 ClonedCodeInfo &InlinedFunctionInfo) {
970 MDNode *AllocTokenMD = CB.getMetadata(KindID: LLVMContext::MD_alloc_token);
971 if (!AllocTokenMD)
972 return;
973
974 SmallVector<CallBase *, 2> AllocCalls;
975 for (BasicBlock &BB : *CalledFunc)
976 if (auto *RI = dyn_cast<ReturnInst>(Val: BB.getTerminator()))
977 if (Value *RV = RI->getReturnValue())
978 collectPointerReturningCalls(RetVal: RV, Out&: AllocCalls);
979
980 for (CallBase *OrigCall : AllocCalls) {
981 auto *ClonedCall = dyn_cast_or_null<CallBase>(Val: VMap.lookup(Val: OrigCall));
982 if (!ClonedCall)
983 continue;
984 // Skip calls simplified during inlining; propagation may be incorrect.
985 if (InlinedFunctionInfo.isSimplified(From: OrigCall, To: ClonedCall))
986 continue;
987 // Fill missing only: never overwrite a more specific token the wrapper
988 // already set on an internal allocation. An unknown type (empty type name
989 // with function name) is not more specific.
990 if (MDNode *MD = ClonedCall->getMetadata(KindID: LLVMContext::MD_alloc_token)) {
991 if (MD->getNumOperands() != 3 ||
992 !cast<MDString>(Val: MD->getOperand(I: 0))->getString().empty())
993 continue;
994 }
995 ClonedCall->setMetadata(KindID: LLVMContext::MD_alloc_token, Node: AllocTokenMD);
996 }
997}
998
999/// When inlining a call site that has !llvm.mem.parallel_loop_access,
1000/// !llvm.access.group, !alias.scope or !noalias metadata, that metadata should
1001/// be propagated to all memory-accessing cloned instructions.
1002static void PropagateCallSiteMetadata(CallBase &CB, Function::iterator FStart,
1003 Function::iterator FEnd) {
1004 MDNode *MemParallelLoopAccess =
1005 CB.getMetadata(KindID: LLVMContext::MD_mem_parallel_loop_access);
1006 MDNode *AccessGroup = CB.getMetadata(KindID: LLVMContext::MD_access_group);
1007 MDNode *AliasScope = CB.getMetadata(KindID: LLVMContext::MD_alias_scope);
1008 MDNode *NoAlias = CB.getMetadata(KindID: LLVMContext::MD_noalias);
1009 if (!MemParallelLoopAccess && !AccessGroup && !AliasScope && !NoAlias)
1010 return;
1011
1012 for (BasicBlock &BB : make_range(x: FStart, y: FEnd)) {
1013 for (Instruction &I : BB) {
1014 // This metadata is only relevant for instructions that access memory.
1015 if (!I.mayReadOrWriteMemory())
1016 continue;
1017
1018 if (MemParallelLoopAccess) {
1019 // TODO: This probably should not overwrite MemParalleLoopAccess.
1020 MemParallelLoopAccess = MDNode::concatenate(
1021 A: I.getMetadata(KindID: LLVMContext::MD_mem_parallel_loop_access),
1022 B: MemParallelLoopAccess);
1023 I.setMetadata(KindID: LLVMContext::MD_mem_parallel_loop_access,
1024 Node: MemParallelLoopAccess);
1025 }
1026
1027 if (AccessGroup)
1028 I.setMetadata(KindID: LLVMContext::MD_access_group, Node: uniteAccessGroups(
1029 AccGroups1: I.getMetadata(KindID: LLVMContext::MD_access_group), AccGroups2: AccessGroup));
1030
1031 if (AliasScope)
1032 I.setMetadata(KindID: LLVMContext::MD_alias_scope, Node: MDNode::concatenate(
1033 A: I.getMetadata(KindID: LLVMContext::MD_alias_scope), B: AliasScope));
1034
1035 if (NoAlias)
1036 I.setMetadata(KindID: LLVMContext::MD_noalias, Node: MDNode::concatenate(
1037 A: I.getMetadata(KindID: LLVMContext::MD_noalias), B: NoAlias));
1038 }
1039 }
1040}
1041
1042/// Track inlining chain via inlined.from metadata for dontcall diagnostics.
1043static void PropagateInlinedFromMetadata(CallBase &CB, StringRef CalledFuncName,
1044 StringRef CallerFuncName,
1045 Function::iterator FStart,
1046 Function::iterator FEnd) {
1047 LLVMContext &Ctx = CB.getContext();
1048 uint64_t InlineSiteLoc = 0;
1049 if (auto *MD = CB.getMetadata(Kind: "srcloc"))
1050 if (auto *CI = mdconst::dyn_extract<ConstantInt>(MD: MD->getOperand(I: 0)))
1051 InlineSiteLoc = CI->getZExtValue();
1052
1053 auto *I64Ty = Type::getInt64Ty(C&: Ctx);
1054 auto MakeMDInt = [&](uint64_t V) {
1055 return ConstantAsMetadata::get(C: ConstantInt::get(Ty: I64Ty, V));
1056 };
1057
1058 for (BasicBlock &BB : make_range(x: FStart, y: FEnd)) {
1059 for (Instruction &I : BB) {
1060 auto *CI = dyn_cast<CallInst>(Val: &I);
1061 if (!CI || !CI->getMetadata(Kind: "srcloc"))
1062 continue;
1063 auto *Callee = CI->getCalledFunction();
1064 if (!Callee || (!Callee->hasFnAttribute(Kind: "dontcall-error") &&
1065 !Callee->hasFnAttribute(Kind: "dontcall-warn")))
1066 continue;
1067
1068 SmallVector<Metadata *, 8> Ops;
1069 if (MDNode *Existing = CI->getMetadata(Kind: "inlined.from"))
1070 append_range(C&: Ops, R: Existing->operands());
1071 else {
1072 Ops.push_back(Elt: MDString::get(Context&: Ctx, Str: CalledFuncName));
1073 Ops.push_back(Elt: MakeMDInt(0));
1074 }
1075 Ops.push_back(Elt: MDString::get(Context&: Ctx, Str: CallerFuncName));
1076 Ops.push_back(Elt: MakeMDInt(InlineSiteLoc));
1077 CI->setMetadata(Kind: "inlined.from", Node: MDNode::get(Context&: Ctx, MDs: Ops));
1078 }
1079 }
1080}
1081
1082/// Bundle operands of the inlined function must be added to inlined call sites.
1083static void PropagateOperandBundles(Function::iterator InlinedBB,
1084 Instruction *CallSiteEHPad) {
1085 for (Instruction &II : llvm::make_early_inc_range(Range&: *InlinedBB)) {
1086 CallBase *I = dyn_cast<CallBase>(Val: &II);
1087 if (!I)
1088 continue;
1089 // Skip call sites which already have a "funclet" bundle.
1090 if (I->getOperandBundle(ID: LLVMContext::OB_funclet))
1091 continue;
1092 // Skip call sites which are nounwind intrinsics (as long as they don't
1093 // lower into regular function calls in the course of IR transformations).
1094 auto *CalledFn =
1095 dyn_cast<Function>(Val: I->getCalledOperand()->stripPointerCasts());
1096 if (CalledFn && CalledFn->isIntrinsic() && I->doesNotThrow() &&
1097 !IntrinsicInst::mayLowerToFunctionCall(IID: CalledFn->getIntrinsicID()))
1098 continue;
1099
1100 SmallVector<OperandBundleDef, 1> OpBundles;
1101 I->getOperandBundlesAsDefs(Defs&: OpBundles);
1102 OpBundles.emplace_back(Args: "funclet", Args&: CallSiteEHPad);
1103
1104 Instruction *NewInst = CallBase::Create(CB: I, Bundles: OpBundles, InsertPt: I->getIterator());
1105 NewInst->takeName(V: I);
1106 I->replaceAllUsesWith(V: NewInst);
1107 I->eraseFromParent();
1108 }
1109}
1110
1111namespace {
1112/// Utility for cloning !noalias and !alias.scope metadata. When a code region
1113/// using scoped alias metadata is inlined, the aliasing relationships may not
1114/// hold between the two version. It is necessary to create a deep clone of the
1115/// metadata, putting the two versions in separate scope domains.
1116class ScopedAliasMetadataDeepCloner {
1117 using MetadataMap = DenseMap<const MDNode *, TrackingMDNodeRef>;
1118 SetVector<const MDNode *> MD;
1119 MetadataMap MDMap;
1120 void addRecursiveMetadataUses();
1121
1122public:
1123 ScopedAliasMetadataDeepCloner(const Function *F);
1124
1125 /// Create a new clone of the scoped alias metadata, which will be used by
1126 /// subsequent remap() calls.
1127 void clone();
1128
1129 /// Remap instructions in the given range from the original to the cloned
1130 /// metadata.
1131 void remap(Function::iterator FStart, Function::iterator FEnd);
1132};
1133} // namespace
1134
1135ScopedAliasMetadataDeepCloner::ScopedAliasMetadataDeepCloner(
1136 const Function *F) {
1137 for (const BasicBlock &BB : *F) {
1138 for (const Instruction &I : BB) {
1139 if (const MDNode *M = I.getMetadata(KindID: LLVMContext::MD_alias_scope))
1140 MD.insert(X: M);
1141 if (const MDNode *M = I.getMetadata(KindID: LLVMContext::MD_noalias))
1142 MD.insert(X: M);
1143
1144 // We also need to clone the metadata in noalias intrinsics.
1145 if (const auto *Decl = dyn_cast<NoAliasScopeDeclInst>(Val: &I))
1146 MD.insert(X: Decl->getScopeList());
1147 }
1148 }
1149 addRecursiveMetadataUses();
1150}
1151
1152void ScopedAliasMetadataDeepCloner::addRecursiveMetadataUses() {
1153 SmallVector<const Metadata *, 16> Queue(MD.begin(), MD.end());
1154 while (!Queue.empty()) {
1155 const MDNode *M = cast<MDNode>(Val: Queue.pop_back_val());
1156 for (const Metadata *Op : M->operands())
1157 if (const MDNode *OpMD = dyn_cast<MDNode>(Val: Op))
1158 if (MD.insert(X: OpMD))
1159 Queue.push_back(Elt: OpMD);
1160 }
1161}
1162
1163void ScopedAliasMetadataDeepCloner::clone() {
1164 assert(MDMap.empty() && "clone() already called ?");
1165
1166 SmallVector<TempMDTuple, 16> DummyNodes;
1167 for (const MDNode *I : MD) {
1168 DummyNodes.push_back(Elt: MDTuple::getTemporary(Context&: I->getContext(), MDs: {}));
1169 MDMap[I].reset(MD: DummyNodes.back().get());
1170 }
1171
1172 // Create new metadata nodes to replace the dummy nodes, replacing old
1173 // metadata references with either a dummy node or an already-created new
1174 // node.
1175 SmallVector<Metadata *, 4> NewOps;
1176 for (const MDNode *I : MD) {
1177 for (const Metadata *Op : I->operands()) {
1178 if (const MDNode *M = dyn_cast<MDNode>(Val: Op))
1179 NewOps.push_back(Elt: MDMap[M]);
1180 else
1181 NewOps.push_back(Elt: const_cast<Metadata *>(Op));
1182 }
1183
1184 MDNode *NewM = MDNode::get(Context&: I->getContext(), MDs: NewOps);
1185 MDTuple *TempM = cast<MDTuple>(Val&: MDMap[I]);
1186 assert(TempM->isTemporary() && "Expected temporary node");
1187
1188 TempM->replaceAllUsesWith(MD: NewM);
1189 NewOps.clear();
1190 }
1191}
1192
1193void ScopedAliasMetadataDeepCloner::remap(Function::iterator FStart,
1194 Function::iterator FEnd) {
1195 if (MDMap.empty())
1196 return; // Nothing to do.
1197
1198 for (BasicBlock &BB : make_range(x: FStart, y: FEnd)) {
1199 for (Instruction &I : BB) {
1200 // TODO: The null checks for the MDMap.lookup() results should no longer
1201 // be necessary.
1202 if (MDNode *M = I.getMetadata(KindID: LLVMContext::MD_alias_scope))
1203 if (MDNode *MNew = MDMap.lookup(Val: M))
1204 I.setMetadata(KindID: LLVMContext::MD_alias_scope, Node: MNew);
1205
1206 if (MDNode *M = I.getMetadata(KindID: LLVMContext::MD_noalias))
1207 if (MDNode *MNew = MDMap.lookup(Val: M))
1208 I.setMetadata(KindID: LLVMContext::MD_noalias, Node: MNew);
1209
1210 if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(Val: &I))
1211 if (MDNode *MNew = MDMap.lookup(Val: Decl->getScopeList()))
1212 Decl->setScopeList(MNew);
1213 }
1214 }
1215}
1216
1217/// If the inlined function has noalias arguments,
1218/// then add new alias scopes for each noalias argument, tag the mapped noalias
1219/// parameters with noalias metadata specifying the new scope, and tag all
1220/// non-derived loads, stores and memory intrinsics with the new alias scopes.
1221static void AddAliasScopeMetadata(CallBase &CB, ValueToValueMapTy &VMap,
1222 const DataLayout &DL, AAResults *CalleeAAR,
1223 ClonedCodeInfo &InlinedFunctionInfo) {
1224 if (!EnableNoAliasConversion)
1225 return;
1226
1227 const Function *CalledFunc = CB.getCalledFunction();
1228 SmallVector<const Argument *, 4> NoAliasArgs;
1229
1230 for (const Argument &Arg : CalledFunc->args())
1231 if (CB.paramHasAttr(ArgNo: Arg.getArgNo(), Kind: Attribute::NoAlias) && !Arg.use_empty())
1232 NoAliasArgs.push_back(Elt: &Arg);
1233
1234 if (NoAliasArgs.empty())
1235 return;
1236
1237 // To do a good job, if a noalias variable is captured, we need to know if
1238 // the capture point dominates the particular use we're considering.
1239 DominatorTree DT;
1240 DT.recalculate(Func&: const_cast<Function&>(*CalledFunc));
1241
1242 // noalias indicates that pointer values based on the argument do not alias
1243 // pointer values which are not based on it. So we add a new "scope" for each
1244 // noalias function argument. Accesses using pointers based on that argument
1245 // become part of that alias scope, accesses using pointers not based on that
1246 // argument are tagged as noalias with that scope.
1247
1248 DenseMap<const Argument *, MDNode *> NewScopes;
1249 MDBuilder MDB(CalledFunc->getContext());
1250
1251 // Create a new scope domain for this function. Its scopes are disjoint,
1252 // since accesses based on one noalias argument don't alias accesses based on
1253 // another one.
1254 MDNode *NewDomain = MDB.createAnonymousAliasScopeDomain(
1255 Description: CalledFunc->getName(), /*DisjointScopes=*/true);
1256 for (unsigned i = 0, e = NoAliasArgs.size(); i != e; ++i) {
1257 const Argument *A = NoAliasArgs[i];
1258
1259 std::string Name = std::string(CalledFunc->getName());
1260 if (A->hasName()) {
1261 Name += ": %";
1262 Name += A->getName();
1263 } else {
1264 Name += ": argument ";
1265 Name += utostr(X: i);
1266 }
1267
1268 // Note: We always create a new anonymous root here. This is true regardless
1269 // of the linkage of the callee because the aliasing "scope" is not just a
1270 // property of the callee, but also all control dependencies in the caller.
1271 MDNode *NewScope = MDB.createAnonymousAliasScope(Domain: NewDomain, Name);
1272 NewScopes.insert(KV: std::make_pair(x&: A, y&: NewScope));
1273
1274 if (UseNoAliasIntrinsic) {
1275 // Introduce a llvm.experimental.noalias.scope.decl for the noalias
1276 // argument.
1277 MDNode *AScopeList = MDNode::get(Context&: CalledFunc->getContext(), MDs: NewScope);
1278 auto *NoAliasDecl =
1279 IRBuilder<>(&CB).CreateNoAliasScopeDeclaration(ScopeTag: AScopeList);
1280 // Ignore the result for now. The result will be used when the
1281 // llvm.noalias intrinsic is introduced.
1282 (void)NoAliasDecl;
1283 }
1284 }
1285
1286 // Iterate over all new instructions in the map; for all memory-access
1287 // instructions, add the alias scope metadata.
1288 for (ValueToValueMapTy::iterator VMI = VMap.begin(), VMIE = VMap.end();
1289 VMI != VMIE; ++VMI) {
1290 if (const Instruction *I = dyn_cast<Instruction>(Val: VMI->first)) {
1291 if (!VMI->second)
1292 continue;
1293
1294 Instruction *NI = dyn_cast<Instruction>(Val&: VMI->second);
1295 if (!NI || InlinedFunctionInfo.isSimplified(From: I, To: NI))
1296 continue;
1297
1298 bool IsArgMemOnlyCall = false, IsFuncCall = false;
1299 SmallVector<const Value *, 2> PtrArgs;
1300
1301 if (const LoadInst *LI = dyn_cast<LoadInst>(Val: I))
1302 PtrArgs.push_back(Elt: LI->getPointerOperand());
1303 else if (const StoreInst *SI = dyn_cast<StoreInst>(Val: I))
1304 PtrArgs.push_back(Elt: SI->getPointerOperand());
1305 else if (const VAArgInst *VAAI = dyn_cast<VAArgInst>(Val: I))
1306 PtrArgs.push_back(Elt: VAAI->getPointerOperand());
1307 else if (const AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(Val: I))
1308 PtrArgs.push_back(Elt: CXI->getPointerOperand());
1309 else if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(Val: I))
1310 PtrArgs.push_back(Elt: RMWI->getPointerOperand());
1311 else if (const auto *Call = dyn_cast<CallBase>(Val: I)) {
1312 // If we know that the call does not access memory, then we'll still
1313 // know that about the inlined clone of this call site, and we don't
1314 // need to add metadata.
1315 if (Call->doesNotAccessMemory())
1316 continue;
1317
1318 IsFuncCall = true;
1319 if (CalleeAAR) {
1320 MemoryEffects ME = CalleeAAR->getMemoryEffects(Call);
1321
1322 // We'll retain this knowledge without additional metadata.
1323 if (ME.onlyAccessesInaccessibleMem())
1324 continue;
1325
1326 if (ME.onlyAccessesArgPointees())
1327 IsArgMemOnlyCall = true;
1328 }
1329
1330 for (Value *Arg : Call->args()) {
1331 // Only care about pointer arguments. If a noalias argument is
1332 // accessed through a non-pointer argument, it must be captured
1333 // first (e.g. via ptrtoint), and we protect against captures below.
1334 if (!Arg->getType()->isPointerTy())
1335 continue;
1336
1337 PtrArgs.push_back(Elt: Arg);
1338 }
1339 }
1340
1341 // If we found no pointers, then this instruction is not suitable for
1342 // pairing with an instruction to receive aliasing metadata.
1343 // However, if this is a call, this we might just alias with none of the
1344 // noalias arguments.
1345 if (PtrArgs.empty() && !IsFuncCall)
1346 continue;
1347
1348 // It is possible that there is only one underlying object, but you
1349 // need to go through several PHIs to see it, and thus could be
1350 // repeated in the Objects list.
1351 SmallPtrSet<const Value *, 4> ObjSet;
1352 SmallVector<Metadata *, 4> Scopes, NoAliases;
1353
1354 for (const Value *V : PtrArgs) {
1355 SmallVector<const Value *, 4> Objects;
1356 getUnderlyingObjects(V, Objects, /* LI = */ nullptr);
1357
1358 ObjSet.insert_range(R&: Objects);
1359 }
1360
1361 // Figure out if we're derived from anything that is not a noalias
1362 // argument.
1363 bool RequiresNoCaptureBefore = false, UsesAliasingPtr = false,
1364 UsesUnknownObject = false;
1365 for (const Value *V : ObjSet) {
1366 // Is this value a constant that cannot be derived from any pointer
1367 // value (we need to exclude constant expressions, for example, that
1368 // are formed from arithmetic on global symbols).
1369 bool IsNonPtrConst = isa<ConstantInt>(Val: V) || isa<ConstantFP>(Val: V) ||
1370 isa<ConstantPointerNull>(Val: V) ||
1371 isa<ConstantDataVector>(Val: V) || isa<UndefValue>(Val: V);
1372 if (IsNonPtrConst)
1373 continue;
1374
1375 // If this is anything other than a noalias argument, then we cannot
1376 // completely describe the aliasing properties using alias.scope
1377 // metadata (and, thus, won't add any).
1378 if (const Argument *A = dyn_cast<Argument>(Val: V)) {
1379 if (!CB.paramHasAttr(ArgNo: A->getArgNo(), Kind: Attribute::NoAlias))
1380 UsesAliasingPtr = true;
1381 } else {
1382 UsesAliasingPtr = true;
1383 }
1384
1385 if (isEscapeSource(V)) {
1386 // An escape source can only alias with a noalias argument if it has
1387 // been captured beforehand.
1388 RequiresNoCaptureBefore = true;
1389 } else if (!isa<Argument>(Val: V) && !isIdentifiedObject(V)) {
1390 // If this is neither an escape source, nor some identified object
1391 // (which cannot directly alias a noalias argument), nor some other
1392 // argument (which, by definition, also cannot alias a noalias
1393 // argument), conservatively do not make any assumptions.
1394 UsesUnknownObject = true;
1395 }
1396 }
1397
1398 // Nothing we can do if the used underlying object cannot be reliably
1399 // determined.
1400 if (UsesUnknownObject)
1401 continue;
1402
1403 // A function call can always get captured noalias pointers (via other
1404 // parameters, globals, etc.).
1405 if (IsFuncCall && !IsArgMemOnlyCall)
1406 RequiresNoCaptureBefore = true;
1407
1408 // First, we want to figure out all of the sets to which we might belong.
1409 // We might belong to a set if the noalias argument is in the set of
1410 // underlying objects. If there is some non-noalias argument in our list
1411 // of underlying objects, then we cannot add a scope because the fact
1412 // that some access does not alias with any set of our noalias arguments
1413 // cannot itself guarantee that it does not alias with this access
1414 // (because there is some pointer of unknown origin involved and the
1415 // other access might also depend on this pointer). We also cannot add
1416 // scopes to arbitrary functions unless we know they don't access any
1417 // non-parameter pointer-values.
1418 bool CanAddScopes = !UsesAliasingPtr && (!IsFuncCall || IsArgMemOnlyCall);
1419 if (CanAddScopes) {
1420 for (const Argument *A : NoAliasArgs) {
1421 if (ObjSet.count(Ptr: A))
1422 Scopes.push_back(Elt: NewScopes[A]);
1423 }
1424 }
1425
1426 if (!Scopes.empty()) {
1427 NI->setMetadata(
1428 KindID: LLVMContext::MD_alias_scope,
1429 Node: MDNode::concatenate(A: NI->getMetadata(KindID: LLVMContext::MD_alias_scope),
1430 B: MDNode::get(Context&: CalledFunc->getContext(), MDs: Scopes)));
1431 // Being in these scopes already means being noalias with the rest.
1432 continue;
1433 }
1434
1435 // Next, we want to figure out all of the sets with which we definitely
1436 // don't alias. Iterate over all noalias set, and add those for which:
1437 // 1. The noalias argument is not in the set of objects from which we
1438 // definitely derive.
1439 // 2. The noalias argument has not yet been captured.
1440 // An arbitrary function that might load pointers could see captured
1441 // noalias arguments via other noalias arguments or globals, and so we
1442 // must always check for prior capture.
1443 for (const Argument *A : NoAliasArgs) {
1444 if (ObjSet.contains(Ptr: A))
1445 continue; // May be based on a noalias argument.
1446
1447 // It might be tempting to skip the PointerMayBeCapturedBefore check if
1448 // A->hasNoCaptureAttr() is true, but this is incorrect because
1449 // nocapture only guarantees that no copies outlive the function, not
1450 // that the value cannot be locally captured.
1451 if (!RequiresNoCaptureBefore ||
1452 !capturesAnything(CC: PointerMayBeCapturedBefore(
1453 V: A, /*ReturnCaptures=*/false, I, DT: &DT, /*IncludeI=*/false,
1454 Mask: CaptureComponents::Provenance)))
1455 NoAliases.push_back(Elt: NewScopes[A]);
1456 }
1457
1458 if (!NoAliases.empty())
1459 NI->setMetadata(KindID: LLVMContext::MD_noalias,
1460 Node: MDNode::concatenate(
1461 A: NI->getMetadata(KindID: LLVMContext::MD_noalias),
1462 B: MDNode::get(Context&: CalledFunc->getContext(), MDs: NoAliases)));
1463 }
1464 }
1465}
1466
1467static bool MayContainThrowingOrExitingCallAfterCB(CallBase *Begin,
1468 ReturnInst *End) {
1469
1470 assert(Begin->getParent() == End->getParent() &&
1471 "Expected to be in same basic block!");
1472 auto BeginIt = Begin->getIterator();
1473 assert(BeginIt != End->getIterator() && "Non-empty BB has empty iterator");
1474 return !llvm::isGuaranteedToTransferExecutionToSuccessor(
1475 Begin: ++BeginIt, End: End->getIterator(), ScanLimit: InlinerAttributeWindow + 1);
1476}
1477
1478// Add attributes from CB params and Fn attributes that can always be propagated
1479// to the corresponding argument / inner callbases.
1480static void AddParamAndFnBasicAttributes(const CallBase &CB,
1481 ValueToValueMapTy &VMap,
1482 ClonedCodeInfo &InlinedFunctionInfo) {
1483 auto *CalledFunction = CB.getCalledFunction();
1484 auto &Context = CalledFunction->getContext();
1485
1486 // Collect valid attributes for all params.
1487 SmallVector<AttrBuilder> ValidObjParamAttrs, ValidExactParamAttrs;
1488 bool HasAttrToPropagate = false;
1489
1490 // Attributes we can only propagate if the exact parameter is forwarded.
1491 // We can propagate both poison generating and UB generating attributes
1492 // without any extra checks. The only attribute that is tricky to propagate
1493 // is `noundef` (skipped for now) as that can create new UB where previous
1494 // behavior was just using a poison value.
1495 static const Attribute::AttrKind ExactAttrsToPropagate[] = {
1496 Attribute::Dereferenceable, Attribute::DereferenceableOrNull,
1497 Attribute::NonNull, Attribute::NoFPClass,
1498 Attribute::Alignment, Attribute::Range};
1499
1500 for (unsigned I = 0, E = CB.arg_size(); I < E; ++I) {
1501 ValidObjParamAttrs.emplace_back(Args: AttrBuilder{CB.getContext()});
1502 ValidExactParamAttrs.emplace_back(Args: AttrBuilder{CB.getContext()});
1503 // Access attributes can be propagated to any param with the same underlying
1504 // object as the argument.
1505 if (CB.paramHasAttr(ArgNo: I, Kind: Attribute::ReadNone))
1506 ValidObjParamAttrs.back().addAttribute(Val: Attribute::ReadNone);
1507 if (CB.paramHasAttr(ArgNo: I, Kind: Attribute::ReadOnly))
1508 ValidObjParamAttrs.back().addAttribute(Val: Attribute::ReadOnly);
1509
1510 for (Attribute::AttrKind AK : ExactAttrsToPropagate) {
1511 Attribute Attr = CB.getParamAttr(ArgNo: I, Kind: AK);
1512 if (Attr.isValid())
1513 ValidExactParamAttrs.back().addAttribute(A: Attr);
1514 }
1515
1516 HasAttrToPropagate |= ValidObjParamAttrs.back().hasAttributes();
1517 HasAttrToPropagate |= ValidExactParamAttrs.back().hasAttributes();
1518 }
1519
1520 // Won't be able to propagate anything.
1521 if (!HasAttrToPropagate)
1522 return;
1523
1524 for (BasicBlock &BB : *CalledFunction) {
1525 for (Instruction &Ins : BB) {
1526 const auto *InnerCB = dyn_cast<CallBase>(Val: &Ins);
1527 if (!InnerCB)
1528 continue;
1529 auto *NewInnerCB = dyn_cast_or_null<CallBase>(Val: VMap.lookup(Val: InnerCB));
1530 if (!NewInnerCB)
1531 continue;
1532 // The InnerCB might have be simplified during the inlining
1533 // process which can make propagation incorrect.
1534 if (InlinedFunctionInfo.isSimplified(From: InnerCB, To: NewInnerCB))
1535 continue;
1536
1537 AttributeList AL = NewInnerCB->getAttributes();
1538 for (unsigned I = 0, E = InnerCB->arg_size(); I < E; ++I) {
1539 // It's unsound or requires special handling to propagate
1540 // attributes to byval arguments. Even if CalledFunction
1541 // doesn't e.g. write to the argument (readonly), the call to
1542 // NewInnerCB may write to its by-value copy.
1543 if (NewInnerCB->isByValArgument(ArgNo: I))
1544 continue;
1545
1546 // Don't bother propagating attrs to constants.
1547 if (match(V: NewInnerCB->getArgOperand(i: I),
1548 P: llvm::PatternMatch::m_ImmConstant()))
1549 continue;
1550
1551 // Check if the underlying value for the parameter is an argument.
1552 const Argument *Arg = dyn_cast<Argument>(Val: InnerCB->getArgOperand(i: I));
1553 unsigned ArgNo;
1554 if (Arg) {
1555 ArgNo = Arg->getArgNo();
1556 // For dereferenceable, dereferenceable_or_null, align, etc...
1557 // we don't want to propagate if the existing param has the same
1558 // attribute with "better" constraints. So remove from the
1559 // new AL if the region of the existing param is larger than
1560 // what we can propagate.
1561 AttrBuilder NewAB{
1562 Context, AttributeSet::get(C&: Context, B: ValidExactParamAttrs[ArgNo])};
1563 if (AL.getParamDereferenceableBytes(Index: I) >
1564 NewAB.getDereferenceableBytes())
1565 NewAB.removeAttribute(Val: Attribute::Dereferenceable);
1566 if (AL.getParamDereferenceableOrNullBytes(ArgNo: I) >
1567 NewAB.getDereferenceableOrNullBytes())
1568 NewAB.removeAttribute(Val: Attribute::DereferenceableOrNull);
1569 if (AL.getParamAlignment(ArgNo: I).valueOrOne() >
1570 NewAB.getAlignment().valueOrOne())
1571 NewAB.removeAttribute(Val: Attribute::Alignment);
1572 if (auto ExistingRange = AL.getParamRange(ArgNo: I)) {
1573 if (auto NewRange = NewAB.getRange()) {
1574 ConstantRange CombinedRange =
1575 ExistingRange->intersectWith(CR: *NewRange);
1576 NewAB.removeAttribute(Val: Attribute::Range);
1577 NewAB.addRangeAttr(CR: CombinedRange);
1578 }
1579 }
1580
1581 if (FPClassTest ExistingNoFP = AL.getParamNoFPClass(ArgNo: I))
1582 NewAB.addNoFPClassAttr(NoFPClassMask: ExistingNoFP | NewAB.getNoFPClass());
1583
1584 AL = AL.addParamAttributes(C&: Context, ArgNo: I, B: NewAB);
1585 } else if (NewInnerCB->getArgOperand(i: I)->getType()->isPointerTy()) {
1586 // Check if the underlying value for the parameter is an argument.
1587 const Value *UnderlyingV =
1588 getUnderlyingObject(V: InnerCB->getArgOperand(i: I));
1589 Arg = dyn_cast<Argument>(Val: UnderlyingV);
1590 if (!Arg)
1591 continue;
1592 ArgNo = Arg->getArgNo();
1593 } else {
1594 continue;
1595 }
1596
1597 // If so, propagate its access attributes.
1598 AL = AL.addParamAttributes(C&: Context, ArgNo: I, B: ValidObjParamAttrs[ArgNo]);
1599
1600 // We can have conflicting attributes from the inner callsite and
1601 // to-be-inlined callsite. In that case, choose the most
1602 // restrictive.
1603
1604 // readonly + writeonly means we can never deref so make readnone.
1605 if (AL.hasParamAttr(ArgNo: I, Kind: Attribute::ReadOnly) &&
1606 AL.hasParamAttr(ArgNo: I, Kind: Attribute::WriteOnly))
1607 AL = AL.addParamAttribute(C&: Context, ArgNo: I, Kind: Attribute::ReadNone);
1608
1609 // If have readnone, need to clear readonly/writeonly
1610 if (AL.hasParamAttr(ArgNo: I, Kind: Attribute::ReadNone)) {
1611 AL = AL.removeParamAttribute(C&: Context, ArgNo: I, Kind: Attribute::ReadOnly);
1612 AL = AL.removeParamAttribute(C&: Context, ArgNo: I, Kind: Attribute::WriteOnly);
1613 }
1614
1615 // Writable cannot exist in conjunction w/ readonly/readnone
1616 if (AL.hasParamAttr(ArgNo: I, Kind: Attribute::ReadOnly) ||
1617 AL.hasParamAttr(ArgNo: I, Kind: Attribute::ReadNone))
1618 AL = AL.removeParamAttribute(C&: Context, ArgNo: I, Kind: Attribute::Writable);
1619 }
1620 NewInnerCB->setAttributes(AL);
1621 }
1622 }
1623}
1624
1625// Only allow these white listed attributes to be propagated back to the
1626// callee. This is because other attributes may only be valid on the call
1627// itself, i.e. attributes such as signext and zeroext.
1628
1629// Attributes that are always okay to propagate as if they are violated its
1630// immediate UB.
1631static AttrBuilder IdentifyValidUBGeneratingAttributes(CallBase &CB) {
1632 AttrBuilder Valid(CB.getContext());
1633 if (auto DerefBytes = CB.getRetDereferenceableBytes())
1634 Valid.addDereferenceableAttr(Bytes: DerefBytes);
1635 if (auto DerefOrNullBytes = CB.getRetDereferenceableOrNullBytes())
1636 Valid.addDereferenceableOrNullAttr(Bytes: DerefOrNullBytes);
1637 if (CB.hasRetAttr(Kind: Attribute::NoAlias))
1638 Valid.addAttribute(Val: Attribute::NoAlias);
1639 if (CB.hasRetAttr(Kind: Attribute::NoUndef))
1640 Valid.addAttribute(Val: Attribute::NoUndef);
1641 return Valid;
1642}
1643
1644// Attributes that need additional checks as propagating them may change
1645// behavior or cause new UB.
1646static AttrBuilder IdentifyValidPoisonGeneratingAttributes(CallBase &CB) {
1647 AttrBuilder Valid(CB.getContext());
1648 if (CB.hasRetAttr(Kind: Attribute::NonNull))
1649 Valid.addAttribute(Val: Attribute::NonNull);
1650 if (CB.hasRetAttr(Kind: Attribute::Alignment))
1651 Valid.addAlignmentAttr(Align: CB.getRetAlign());
1652 if (std::optional<ConstantRange> Range = CB.getRange())
1653 Valid.addRangeAttr(CR: *Range);
1654 if (CB.hasRetAttr(Kind: Attribute::NoFPClass))
1655 Valid.addNoFPClassAttr(NoFPClassMask: CB.getRetNoFPClass());
1656 return Valid;
1657}
1658
1659static void AddReturnAttributes(CallBase &CB, ValueToValueMapTy &VMap,
1660 ClonedCodeInfo &InlinedFunctionInfo) {
1661 AttrBuilder CallSiteValidUB = IdentifyValidUBGeneratingAttributes(CB);
1662 AttrBuilder CallSiteValidPG = IdentifyValidPoisonGeneratingAttributes(CB);
1663 if (!CallSiteValidUB.hasAttributes() && !CallSiteValidPG.hasAttributes())
1664 return;
1665 auto *CalledFunction = CB.getCalledFunction();
1666 auto &Context = CalledFunction->getContext();
1667
1668 for (auto &BB : *CalledFunction) {
1669 auto *RI = dyn_cast<ReturnInst>(Val: BB.getTerminator());
1670 if (!RI || !isa<CallBase>(Val: RI->getOperand(i_nocapture: 0)))
1671 continue;
1672 auto *RetVal = cast<CallBase>(Val: RI->getOperand(i_nocapture: 0));
1673 // Check that the cloned RetVal exists and is a call, otherwise we cannot
1674 // add the attributes on the cloned RetVal. Simplification during inlining
1675 // could have transformed the cloned instruction.
1676 auto *NewRetVal = dyn_cast_or_null<CallBase>(Val: VMap.lookup(Val: RetVal));
1677 if (!NewRetVal)
1678 continue;
1679
1680 // The RetVal might have be simplified during the inlining
1681 // process which can make propagation incorrect.
1682 if (InlinedFunctionInfo.isSimplified(From: RetVal, To: NewRetVal))
1683 continue;
1684 // Backward propagation of attributes to the returned value may be incorrect
1685 // if it is control flow dependent.
1686 // Consider:
1687 // @callee {
1688 // %rv = call @foo()
1689 // %rv2 = call @bar()
1690 // if (%rv2 != null)
1691 // return %rv2
1692 // if (%rv == null)
1693 // exit()
1694 // return %rv
1695 // }
1696 // caller() {
1697 // %val = call nonnull @callee()
1698 // }
1699 // Here we cannot add the nonnull attribute on either foo or bar. So, we
1700 // limit the check to both RetVal and RI are in the same basic block and
1701 // there are no throwing/exiting instructions between these instructions.
1702 if (RI->getParent() != RetVal->getParent() ||
1703 MayContainThrowingOrExitingCallAfterCB(Begin: RetVal, End: RI))
1704 continue;
1705 // Add to the existing attributes of NewRetVal, i.e. the cloned call
1706 // instruction.
1707 // NB! When we have the same attribute already existing on NewRetVal, but
1708 // with a differing value, the AttributeList's merge API honours the already
1709 // existing attribute value (i.e. attributes such as dereferenceable,
1710 // dereferenceable_or_null etc). See AttrBuilder::merge for more details.
1711 AttrBuilder ValidUB = IdentifyValidUBGeneratingAttributes(CB);
1712 AttrBuilder ValidPG = IdentifyValidPoisonGeneratingAttributes(CB);
1713 AttributeList AL = NewRetVal->getAttributes();
1714 if (ValidUB.getDereferenceableBytes() < AL.getRetDereferenceableBytes())
1715 ValidUB.removeAttribute(Val: Attribute::Dereferenceable);
1716 if (ValidUB.getDereferenceableOrNullBytes() <
1717 AL.getRetDereferenceableOrNullBytes())
1718 ValidUB.removeAttribute(Val: Attribute::DereferenceableOrNull);
1719 AttributeList NewAL = AL.addRetAttributes(C&: Context, B: ValidUB);
1720 // Attributes that may generate poison returns are a bit tricky. If we
1721 // propagate them, other uses of the callsite might have their behavior
1722 // change or cause UB (if they have noundef) b.c of the new potential
1723 // poison.
1724 // Take the following three cases:
1725 //
1726 // 1)
1727 // define nonnull ptr @foo() {
1728 // %p = call ptr @bar()
1729 // call void @use(ptr %p) willreturn nounwind
1730 // ret ptr %p
1731 // }
1732 //
1733 // 2)
1734 // define noundef nonnull ptr @foo() {
1735 // %p = call ptr @bar()
1736 // call void @use(ptr %p) willreturn nounwind
1737 // ret ptr %p
1738 // }
1739 //
1740 // 3)
1741 // define nonnull ptr @foo() {
1742 // %p = call noundef ptr @bar()
1743 // ret ptr %p
1744 // }
1745 //
1746 // In case 1, we can't propagate nonnull because poison value in @use may
1747 // change behavior or trigger UB.
1748 // In case 2, we don't need to be concerned about propagating nonnull, as
1749 // any new poison at @use will trigger UB anyways.
1750 // In case 3, we can never propagate nonnull because it may create UB due to
1751 // the noundef on @bar.
1752 if (ValidPG.getAlignment().valueOrOne() < AL.getRetAlignment().valueOrOne())
1753 ValidPG.removeAttribute(Val: Attribute::Alignment);
1754 if (ValidPG.hasAttributes()) {
1755 Attribute CBRange = ValidPG.getAttribute(Kind: Attribute::Range);
1756 if (CBRange.isValid()) {
1757 Attribute NewRange = AL.getRetAttr(Kind: Attribute::Range);
1758 if (NewRange.isValid()) {
1759 ValidPG.addRangeAttr(
1760 CR: CBRange.getRange().intersectWith(CR: NewRange.getRange()));
1761 }
1762 }
1763
1764 Attribute CBNoFPClass = ValidPG.getAttribute(Kind: Attribute::NoFPClass);
1765 if (CBNoFPClass.isValid() && AL.hasRetAttr(Kind: Attribute::NoFPClass)) {
1766 ValidPG.addNoFPClassAttr(
1767 NoFPClassMask: CBNoFPClass.getNoFPClass() |
1768 AL.getRetAttr(Kind: Attribute::NoFPClass).getNoFPClass());
1769 }
1770
1771 // Three checks.
1772 // If the callsite has `noundef`, then a poison due to violating the
1773 // return attribute will create UB anyways so we can always propagate.
1774 // Otherwise, if the return value (callee to be inlined) has `noundef`, we
1775 // can't propagate as a new poison return will cause UB.
1776 // Finally, check if the return value has no uses whose behavior may
1777 // change/may cause UB if we potentially return poison. At the moment this
1778 // is implemented overly conservatively with a single-use check.
1779 // TODO: Update the single-use check to iterate through uses and only bail
1780 // if we have a potentially dangerous use.
1781
1782 if (CB.hasRetAttr(Kind: Attribute::NoUndef) ||
1783 (RetVal->hasOneUse() && !RetVal->hasRetAttr(Kind: Attribute::NoUndef)))
1784 NewAL = NewAL.addRetAttributes(C&: Context, B: ValidPG);
1785 }
1786 NewRetVal->setAttributes(NewAL);
1787 }
1788}
1789
1790/// If the inlined function has non-byval align arguments, then
1791/// add @llvm.assume-based alignment assumptions to preserve this information.
1792static void AddAlignmentAssumptions(CallBase &CB, InlineFunctionInfo &IFI) {
1793 if (!PreserveAlignmentAssumptions || !IFI.GetAssumptionCache)
1794 return;
1795
1796 AssumptionCache *AC = &IFI.GetAssumptionCache(*CB.getCaller());
1797 auto &DL = CB.getDataLayout();
1798
1799 // To avoid inserting redundant assumptions, we should check for assumptions
1800 // already in the caller. To do this, we might need a DT of the caller.
1801 DominatorTree DT;
1802 bool DTCalculated = false;
1803
1804 Function *CalledFunc = CB.getCalledFunction();
1805 for (Argument &Arg : CalledFunc->args()) {
1806 if (!Arg.getType()->isPointerTy() || Arg.hasPassPointeeByValueCopyAttr() ||
1807 Arg.use_empty())
1808 continue;
1809 MaybeAlign Alignment = Arg.getParamAlign();
1810 if (!Alignment)
1811 continue;
1812
1813 if (!DTCalculated) {
1814 DT.recalculate(Func&: *CB.getCaller());
1815 DTCalculated = true;
1816 }
1817 // If we can already prove the asserted alignment in the context of the
1818 // caller, then don't bother inserting the assumption.
1819 Value *ArgVal = CB.getArgOperand(i: Arg.getArgNo());
1820 if (getKnownAlignment(V: ArgVal, DL, CtxI: &CB, AC, DT: &DT) >= *Alignment)
1821 continue;
1822
1823 CallInst *NewAsmp = IRBuilder<>(&CB).CreateAlignmentAssumption(
1824 DL, PtrValue: ArgVal, Alignment: Alignment->value());
1825 AC->registerAssumption(CI: cast<AssumeInst>(Val: NewAsmp));
1826 }
1827}
1828
1829static void HandleByValArgumentInit(Type *ByValType, Value *Dst, Value *Src,
1830 MaybeAlign SrcAlign, Module *M,
1831 BasicBlock *InsertBlock,
1832 InlineFunctionInfo &IFI,
1833 Function *CalledFunc) {
1834 IRBuilder<> Builder(InsertBlock->begin());
1835
1836 Value *Size =
1837 Builder.getInt64(C: M->getDataLayout().getTypeStoreSize(Ty: ByValType));
1838
1839 Align DstAlign = Dst->getPointerAlignment(DL: M->getDataLayout());
1840
1841 // Generate a memcpy with the correct alignments.
1842 CallInst *CI = Builder.CreateMemCpy(Dst, DstAlign, Src, SrcAlign, Size);
1843
1844 // The verifier requires that all calls of debug-info-bearing functions
1845 // from debug-info-bearing functions have a debug location (for inlining
1846 // purposes). Assign a dummy location to satisfy the constraint.
1847 if (!CI->getDebugLoc() && InsertBlock->getParent()->getSubprogram())
1848 if (DISubprogram *SP = CalledFunc->getSubprogram())
1849 CI->setDebugLoc(DILocation::get(Context&: SP->getContext(), Line: 0, Column: 0, Scope: SP));
1850}
1851
1852/// When inlining a call site that has a byval argument,
1853/// we have to make the implicit memcpy explicit by adding it.
1854static Value *HandleByValArgument(Type *ByValType, Value *Arg,
1855 Instruction *TheCall,
1856 const Function *CalledFunc,
1857 InlineFunctionInfo &IFI,
1858 MaybeAlign ByValAlignment) {
1859 Function *Caller = TheCall->getFunction();
1860 const DataLayout &DL = Caller->getDataLayout();
1861
1862 // If the called function is readonly, then it could not mutate the caller's
1863 // copy of the byval'd memory. In this case, it is safe to elide the copy and
1864 // temporary.
1865 if (CalledFunc->onlyReadsMemory()) {
1866 // If the byval argument has a specified alignment that is greater than the
1867 // passed in pointer, then we either have to round up the input pointer or
1868 // give up on this transformation.
1869 if (ByValAlignment.valueOrOne() == 1)
1870 return Arg;
1871
1872 AssumptionCache *AC =
1873 IFI.GetAssumptionCache ? &IFI.GetAssumptionCache(*Caller) : nullptr;
1874
1875 // If the pointer is already known to be sufficiently aligned, or if we can
1876 // round it up to a larger alignment, then we don't need a temporary.
1877 if (getOrEnforceKnownAlignment(V: Arg, PrefAlign: *ByValAlignment, DL, CtxI: TheCall, AC) >=
1878 *ByValAlignment)
1879 return Arg;
1880
1881 // Otherwise, we have to make a memcpy to get a safe alignment. This is bad
1882 // for code quality, but rarely happens and is required for correctness.
1883 }
1884
1885 // Create the alloca. If we have DataLayout, use nice alignment.
1886 Align Alignment = DL.getPrefTypeAlign(Ty: ByValType);
1887
1888 // If the byval had an alignment specified, we *must* use at least that
1889 // alignment, as it is required by the byval argument (and uses of the
1890 // pointer inside the callee).
1891 if (ByValAlignment)
1892 Alignment = std::max(a: Alignment, b: *ByValAlignment);
1893
1894 AllocaInst *NewAlloca =
1895 new AllocaInst(ByValType, Arg->getType()->getPointerAddressSpace(),
1896 nullptr, Alignment, Arg->getName());
1897 NewAlloca->setDebugLoc(DebugLoc::getCompilerGenerated());
1898 NewAlloca->insertBefore(InsertPos: Caller->begin()->begin());
1899 IFI.StaticAllocas.push_back(Elt: NewAlloca);
1900
1901 // Uses of the argument in the function should use our new alloca
1902 // instead.
1903 return NewAlloca;
1904}
1905
1906// Check whether this Value is used by a lifetime intrinsic.
1907static bool isUsedByLifetimeMarker(Value *V) {
1908 for (User *U : V->users())
1909 if (isa<LifetimeIntrinsic>(Val: U))
1910 return true;
1911 return false;
1912}
1913
1914// Check whether the given alloca already has
1915// lifetime.start or lifetime.end intrinsics.
1916static bool hasLifetimeMarkers(AllocaInst *AI) {
1917 Type *Ty = AI->getType();
1918 Type *Int8PtrTy =
1919 PointerType::get(C&: Ty->getContext(), AddressSpace: Ty->getPointerAddressSpace());
1920 if (Ty == Int8PtrTy)
1921 return isUsedByLifetimeMarker(V: AI);
1922
1923 // Do a scan to find all the casts to i8*.
1924 for (User *U : AI->users()) {
1925 if (U->getType() != Int8PtrTy) continue;
1926 if (U->stripPointerCasts() != AI) continue;
1927 if (isUsedByLifetimeMarker(V: U))
1928 return true;
1929 }
1930 return false;
1931}
1932
1933/// Return the result of AI->isStaticAlloca() if AI were moved to the entry
1934/// block. Allocas used in inalloca calls and allocas of dynamic array size
1935/// cannot be static.
1936static bool allocaWouldBeStaticInEntry(const AllocaInst *AI ) {
1937 return isa<Constant>(Val: AI->getArraySize()) && !AI->isUsedWithInAlloca();
1938}
1939
1940/// Returns a DebugLoc for a new DILocation which is a clone of \p OrigDL
1941/// inlined at \p InlinedAt. \p IANodes is an inlined-at cache.
1942static DebugLoc inlineDebugLoc(
1943 DebugLoc OrigDL, DILocation *InlinedAt, LLVMContext &Ctx,
1944 DenseMap<const MDNode *, MDNode *> &IANodes,
1945 SmallDenseMap<const DILocation *, DILocation *, 16> &InlineLocs) {
1946 if (DILocation *Cached = InlineLocs.lookup(Val: OrigDL.get()))
1947 return DebugLoc(Cached);
1948 DILocation *IA =
1949 OrigDL->getInlinedAt()
1950 ? DebugLoc::appendInlinedAt(DL: OrigDL, InlinedAt, Ctx, Cache&: IANodes).get()
1951 : InlinedAt;
1952 DILocation *Result = DILocation::getDistinct(
1953 Context&: Ctx, Line: OrigDL.getLine(), Column: OrigDL.getCol(), Scope: OrigDL.getScope(), InlinedAt: IA,
1954 ImplicitCode: OrigDL.isImplicitCode(), AtomGroup: OrigDL->getAtomGroup(), AtomRank: OrigDL->getAtomRank());
1955 InlineLocs[OrigDL.get()] = Result;
1956 return DebugLoc(Result);
1957}
1958
1959/// Update inlined instructions' line numbers to
1960/// to encode location where these instructions are inlined.
1961static void fixupLineNumbers(Function *Fn, Function::iterator FI,
1962 Instruction *TheCall, bool CalleeHasDebugInfo) {
1963 if (!TheCall->getDebugLoc())
1964 return;
1965
1966 // Don't propagate the source location atom from the call to inlined nodebug
1967 // instructions, and avoid putting it in the InlinedAt field of inlined
1968 // not-nodebug instructions. FIXME: Possibly worth transferring/generating
1969 // an atom for the returned value, otherwise we miss stepping on inlined
1970 // nodebug functions (which is different to existing behaviour).
1971 DebugLoc TheCallDL = TheCall->getDebugLoc()->getWithoutAtom();
1972
1973 // A call receiving the outer callsite's fallback location has no usable
1974 // callsite probe of its own. Do not let it inherit the outer call's probe
1975 // discriminator, but preserve any packed DWARF base discriminator.
1976 DebugLoc TheCallDLForInlinedCall = TheCallDL;
1977 uint32_t Discriminator = TheCallDLForInlinedCall->getDiscriminator();
1978 if (DILocation::isPseudoProbeDiscriminator(Discriminator)) {
1979 std::optional<uint32_t> DwarfDiscriminator =
1980 PseudoProbeDwarfDiscriminator::extractDwarfBaseDiscriminator(
1981 Value: Discriminator);
1982 TheCallDLForInlinedCall = TheCallDLForInlinedCall->cloneWithDiscriminator(
1983 Discriminator: DwarfDiscriminator.value_or(u: 0));
1984 }
1985
1986 auto &Ctx = Fn->getContext();
1987 DILocation *InlinedAtNode = TheCallDL;
1988
1989 // Create a unique call site, not to be confused with any other call from the
1990 // same location.
1991 InlinedAtNode = DILocation::getDistinct(
1992 Context&: Ctx, Line: InlinedAtNode->getLine(), Column: InlinedAtNode->getColumn(),
1993 Scope: InlinedAtNode->getScope(), InlinedAt: InlinedAtNode->getInlinedAt());
1994
1995 // Cache the inlined-at nodes as they're built so they are reused, without
1996 // this every instruction's inlined-at chain would become distinct from each
1997 // other.
1998 DenseMap<const MDNode *, MDNode *> IANodes;
1999 SmallDenseMap<const DILocation *, DILocation *, 16> InlineLocs;
2000
2001 // Check if we are not generating inline line tables and want to use
2002 // the call site location instead.
2003 bool NoInlineLineTables = Fn->hasFnAttribute(Kind: "no-inline-line-tables");
2004
2005 // Helper-util for updating the metadata attached to an instruction.
2006 auto UpdateInst = [&](Instruction &I) {
2007 // Loop metadata needs to be updated so that the start and end locs
2008 // reference inlined-at locations.
2009 auto updateLoopInfoLoc = [&Ctx, &InlinedAtNode, &IANodes,
2010 &InlineLocs](Metadata *MD) -> Metadata * {
2011 if (auto *Loc = dyn_cast_or_null<DILocation>(Val: MD))
2012 return inlineDebugLoc(OrigDL: Loc, InlinedAt: InlinedAtNode, Ctx, IANodes, InlineLocs)
2013 .get();
2014 return MD;
2015 };
2016 updateLoopMetadataDebugLocations(I, Updater: updateLoopInfoLoc);
2017
2018 if (!NoInlineLineTables)
2019 if (DebugLoc DL = I.getDebugLoc()) {
2020 DebugLoc IDL = inlineDebugLoc(OrigDL: DL, InlinedAt: InlinedAtNode, Ctx&: I.getContext(),
2021 IANodes, InlineLocs);
2022 I.setDebugLoc(IDL);
2023 return;
2024 }
2025
2026 if (CalleeHasDebugInfo && !NoInlineLineTables)
2027 return;
2028
2029 // If the inlined instruction has no line number, or if inline info
2030 // is not being generated, make it look as if it originates from the call
2031 // location. This is important for ((__always_inline, __nodebug__))
2032 // functions which must use caller location for all instructions in their
2033 // function body.
2034
2035 // Don't update static allocas, as they may get moved later.
2036 if (auto *AI = dyn_cast<AllocaInst>(Val: &I))
2037 if (allocaWouldBeStaticInEntry(AI))
2038 return;
2039
2040 // Do not force a debug loc for pseudo probes, since they do not need to
2041 // be debuggable, and also they are expected to have a zero/null dwarf
2042 // discriminator at this point which could be violated otherwise.
2043 if (isa<PseudoProbeInst>(Val: I))
2044 return;
2045
2046 I.setDebugLoc(isa<CallBase>(Val: I) ? TheCallDLForInlinedCall : TheCallDL);
2047 };
2048
2049 // Helper-util for updating debug-info records attached to instructions.
2050 auto UpdateDVR = [&](DbgRecord *DVR) {
2051 assert(DVR->getDebugLoc() && "Debug Value must have debug loc");
2052 if (NoInlineLineTables) {
2053 DVR->setDebugLoc(TheCallDL);
2054 return;
2055 }
2056 DebugLoc DL = DVR->getDebugLoc();
2057 DebugLoc IDL = inlineDebugLoc(OrigDL: DL, InlinedAt: InlinedAtNode,
2058 Ctx&: DVR->getMarker()->getParent()->getContext(),
2059 IANodes, InlineLocs);
2060 DVR->setDebugLoc(IDL);
2061 };
2062
2063 // Iterate over all instructions, updating metadata and debug-info records.
2064 for (; FI != Fn->end(); ++FI) {
2065 for (Instruction &I : *FI) {
2066 UpdateInst(I);
2067 for (DbgRecord &DVR : I.getDbgRecordRange()) {
2068 UpdateDVR(&DVR);
2069 }
2070 }
2071
2072 // Remove debug info records if we're not keeping inline info.
2073 if (NoInlineLineTables) {
2074 BasicBlock::iterator BI = FI->begin();
2075 while (BI != FI->end()) {
2076 BI->dropDbgRecords();
2077 ++BI;
2078 }
2079 }
2080 }
2081}
2082
2083#undef DEBUG_TYPE
2084#define DEBUG_TYPE "assignment-tracking"
2085/// Find Alloca and linked DbgAssignIntrinsic for locals escaped by \p CB.
2086static at::StorageToVarsMap collectEscapedLocals(const DataLayout &DL,
2087 const CallBase &CB) {
2088 at::StorageToVarsMap EscapedLocals;
2089 SmallPtrSet<const Value *, 4> SeenBases;
2090
2091 LLVM_DEBUG(
2092 errs() << "# Finding caller local variables escaped by callee\n");
2093 for (const Value *Arg : CB.args()) {
2094 LLVM_DEBUG(errs() << "INSPECT: " << *Arg << "\n");
2095 if (!Arg->getType()->isPointerTy()) {
2096 LLVM_DEBUG(errs() << " | SKIP: Not a pointer\n");
2097 continue;
2098 }
2099
2100 const Instruction *I = dyn_cast<Instruction>(Val: Arg);
2101 if (!I) {
2102 LLVM_DEBUG(errs() << " | SKIP: Not result of instruction\n");
2103 continue;
2104 }
2105
2106 // Walk back to the base storage.
2107 assert(Arg->getType()->isPtrOrPtrVectorTy());
2108 APInt TmpOffset(DL.getIndexTypeSizeInBits(Ty: Arg->getType()), 0, false);
2109 const AllocaInst *Base = dyn_cast<AllocaInst>(
2110 Val: Arg->stripAndAccumulateConstantOffsets(DL, Offset&: TmpOffset, AllowNonInbounds: true));
2111 if (!Base) {
2112 LLVM_DEBUG(errs() << " | SKIP: Couldn't walk back to base storage\n");
2113 continue;
2114 }
2115
2116 assert(Base);
2117 LLVM_DEBUG(errs() << " | BASE: " << *Base << "\n");
2118 // We only need to process each base address once - skip any duplicates.
2119 if (!SeenBases.insert(Ptr: Base).second)
2120 continue;
2121
2122 // Find all local variables associated with the backing storage.
2123 auto CollectAssignsForStorage = [&](DbgVariableRecord *DbgAssign) {
2124 // Skip variables from inlined functions - they are not local variables.
2125 if (DbgAssign->getDebugLoc().getInlinedAt())
2126 return;
2127 LLVM_DEBUG(errs() << " > DEF : " << *DbgAssign << "\n");
2128 EscapedLocals[Base].insert(X: at::VarRecord(DbgAssign));
2129 };
2130 for_each(Range: at::getDVRAssignmentMarkers(Inst: Base), F: CollectAssignsForStorage);
2131 }
2132 return EscapedLocals;
2133}
2134
2135static void trackInlinedStores(Function::iterator Start, Function::iterator End,
2136 const CallBase &CB) {
2137 LLVM_DEBUG(errs() << "trackInlinedStores into "
2138 << Start->getParent()->getName() << " from "
2139 << CB.getCalledFunction()->getName() << "\n");
2140 const DataLayout &DL = CB.getDataLayout();
2141 at::trackAssignments(Start, End, Vars: collectEscapedLocals(DL, CB), DL);
2142}
2143
2144/// Update inlined instructions' DIAssignID metadata. We need to do this
2145/// otherwise a function inlined more than once into the same function
2146/// will cause DIAssignID to be shared by many instructions.
2147static void fixupAssignments(Function::iterator Start, Function::iterator End) {
2148 DenseMap<DIAssignID *, DIAssignID *> Map;
2149 // Loop over all the inlined instructions. If we find a DIAssignID
2150 // attachment or use, replace it with a new version.
2151 for (auto BBI = Start; BBI != End; ++BBI) {
2152 for (Instruction &I : *BBI)
2153 at::remapAssignID(Map, I);
2154 }
2155}
2156#undef DEBUG_TYPE
2157#define DEBUG_TYPE "inline-function"
2158
2159/// Update the block frequencies of the caller after a callee has been inlined.
2160///
2161/// Each block cloned into the caller has its block frequency scaled by the
2162/// ratio of CallSiteFreq/CalleeEntryFreq. This ensures that the cloned copy of
2163/// callee's entry block gets the same frequency as the callsite block and the
2164/// relative frequencies of all cloned blocks remain the same after cloning.
2165static void updateCallerBFI(BasicBlock *CallSiteBlock,
2166 const ValueToValueMapTy &VMap,
2167 BlockFrequencyInfo *CallerBFI,
2168 BlockFrequencyInfo *CalleeBFI,
2169 const BasicBlock &CalleeEntryBlock) {
2170 SmallPtrSet<BasicBlock *, 16> ClonedBBs;
2171 for (auto Entry : VMap) {
2172 if (!isa<BasicBlock>(Val: Entry.first) || !Entry.second)
2173 continue;
2174 auto *OrigBB = cast<BasicBlock>(Val: Entry.first);
2175 auto *ClonedBB = cast<BasicBlock>(Val: Entry.second);
2176 BlockFrequency Freq = CalleeBFI->getBlockFreq(BB: OrigBB);
2177 if (!ClonedBBs.insert(Ptr: ClonedBB).second) {
2178 // Multiple blocks in the callee might get mapped to one cloned block in
2179 // the caller since we prune the callee as we clone it. When that happens,
2180 // we want to use the maximum among the original blocks' frequencies.
2181 BlockFrequency NewFreq = CallerBFI->getBlockFreq(BB: ClonedBB);
2182 if (NewFreq > Freq)
2183 Freq = NewFreq;
2184 }
2185 CallerBFI->setBlockFreq(BB: ClonedBB, Freq);
2186 }
2187 BasicBlock *EntryClone = cast<BasicBlock>(Val: VMap.lookup(Val: &CalleeEntryBlock));
2188 CallerBFI->setBlockFreqAndScale(
2189 ReferenceBB: EntryClone, Freq: CallerBFI->getBlockFreq(BB: CallSiteBlock), BlocksToScale&: ClonedBBs);
2190}
2191
2192/// Update the branch metadata for cloned call instructions.
2193static void updateCallProfile(Function *Callee, const ValueToValueMapTy &VMap,
2194 const uint64_t &CalleeEntryCount,
2195 const CallBase &TheCall, ProfileSummaryInfo *PSI,
2196 BlockFrequencyInfo *CallerBFI) {
2197 if (CalleeEntryCount < 1)
2198 return;
2199 auto CallSiteCount =
2200 PSI ? PSI->getProfileCount(CallInst: TheCall, BFI: CallerBFI) : std::nullopt;
2201 int64_t CallCount = std::min(a: CallSiteCount.value_or(u: 0), b: CalleeEntryCount);
2202 updateProfileCallee(Callee, EntryDelta: -CallCount, VMap: &VMap);
2203}
2204
2205void llvm::updateProfileCallee(
2206 Function *Callee, int64_t EntryDelta,
2207 const ValueMap<const Value *, WeakTrackingVH> *VMap) {
2208 auto CalleeCount = Callee->getEntryCount();
2209 if (!CalleeCount)
2210 return;
2211
2212 // Since CallSiteCount is an estimate, it could exceed the original callee
2213 // count and has to be set to 0 so guard against underflow.
2214 const uint64_t NewEntryCount =
2215 (EntryDelta < 0 && static_cast<uint64_t>(-EntryDelta) > *CalleeCount)
2216 ? 0
2217 : *CalleeCount + EntryDelta;
2218
2219 auto updateVTableProfWeight = [](CallBase *CB, const uint64_t NewEntryCount,
2220 const uint64_t PriorEntryCount) {
2221 Instruction *VPtr = PGOIndirectCallVisitor::tryGetVTableInstruction(CB);
2222 if (VPtr)
2223 scaleProfData(I&: *VPtr, S: NewEntryCount, T: PriorEntryCount);
2224 };
2225
2226 // During inlining ?
2227 if (VMap) {
2228 uint64_t CloneEntryCount = *CalleeCount - NewEntryCount;
2229 for (auto Entry : *VMap) {
2230 if (isa<CallInst>(Val: Entry.first))
2231 if (auto *CI = dyn_cast_or_null<CallInst>(Val: Entry.second)) {
2232 CI->updateProfWeight(S: CloneEntryCount, T: *CalleeCount);
2233 updateVTableProfWeight(CI, CloneEntryCount, *CalleeCount);
2234 }
2235
2236 if (isa<InvokeInst>(Val: Entry.first))
2237 if (auto *II = dyn_cast_or_null<InvokeInst>(Val: Entry.second)) {
2238 II->updateProfWeight(S: CloneEntryCount, T: *CalleeCount);
2239 updateVTableProfWeight(II, CloneEntryCount, *CalleeCount);
2240 }
2241 }
2242 }
2243
2244 if (EntryDelta) {
2245 Callee->setEntryCount(Count: NewEntryCount);
2246
2247 for (BasicBlock &BB : *Callee)
2248 // No need to update the callsite if it is pruned during inlining.
2249 if (!VMap || VMap->count(Val: &BB))
2250 for (Instruction &I : BB) {
2251 if (CallInst *CI = dyn_cast<CallInst>(Val: &I)) {
2252 CI->updateProfWeight(S: NewEntryCount, T: *CalleeCount);
2253 updateVTableProfWeight(CI, NewEntryCount, *CalleeCount);
2254 }
2255 if (InvokeInst *II = dyn_cast<InvokeInst>(Val: &I)) {
2256 II->updateProfWeight(S: NewEntryCount, T: *CalleeCount);
2257 updateVTableProfWeight(II, NewEntryCount, *CalleeCount);
2258 }
2259 }
2260 }
2261}
2262
2263/// An operand bundle "clang.arc.attachedcall" on a call indicates the call
2264/// result is implicitly consumed by a call to retainRV or claimRV immediately
2265/// after the call. This function inlines the retainRV/claimRV calls.
2266///
2267/// There are three cases to consider:
2268///
2269/// 1. If there is a call to autoreleaseRV that takes a pointer to the returned
2270/// object in the callee return block, the autoreleaseRV call and the
2271/// retainRV/claimRV call in the caller cancel out. If the call in the caller
2272/// is a claimRV call, a call to objc_release is emitted.
2273///
2274/// 2. If there is a call in the callee return block that doesn't have operand
2275/// bundle "clang.arc.attachedcall", the operand bundle on the original call
2276/// is transferred to the call in the callee.
2277///
2278/// 3. Otherwise, a call to objc_retain is inserted if the call in the caller is
2279/// a retainRV call.
2280static void
2281inlineRetainOrClaimRVCalls(CallBase &CB, objcarc::ARCInstKind RVCallKind,
2282 const SmallVectorImpl<ReturnInst *> &Returns) {
2283 assert(objcarc::isRetainOrClaimRV(RVCallKind) && "unexpected ARC function");
2284 bool IsRetainRV = RVCallKind == objcarc::ARCInstKind::RetainRV,
2285 IsUnsafeClaimRV = !IsRetainRV;
2286
2287 for (auto *RI : Returns) {
2288 Value *RetOpnd = objcarc::GetRCIdentityRoot(V: RI->getOperand(i_nocapture: 0));
2289 bool InsertRetainCall = IsRetainRV;
2290 IRBuilder<> Builder(*RI->getModule());
2291
2292 // Walk backwards through the basic block looking for either a matching
2293 // autoreleaseRV call or an unannotated call.
2294 auto InstRange = llvm::make_range(x: ++(RI->getIterator().getReverse()),
2295 y: RI->getParent()->rend());
2296 for (Instruction &I : llvm::make_early_inc_range(Range&: InstRange)) {
2297 // Ignore casts.
2298 if (isa<CastInst>(Val: I))
2299 continue;
2300
2301 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I)) {
2302 if (II->getIntrinsicID() != Intrinsic::objc_autoreleaseReturnValue ||
2303 !II->use_empty() ||
2304 objcarc::GetRCIdentityRoot(V: II->getOperand(i_nocapture: 0)) != RetOpnd)
2305 break;
2306
2307 // If we've found a matching authoreleaseRV call:
2308 // - If claimRV is attached to the call, insert a call to objc_release
2309 // and erase the autoreleaseRV call.
2310 // - If retainRV is attached to the call, just erase the autoreleaseRV
2311 // call.
2312 if (IsUnsafeClaimRV) {
2313 Builder.SetInsertPoint(II);
2314 Builder.CreateIntrinsic(ID: Intrinsic::objc_release, Args: RetOpnd);
2315 }
2316 II->eraseFromParent();
2317 InsertRetainCall = false;
2318 break;
2319 }
2320
2321 auto *CI = dyn_cast<CallInst>(Val: &I);
2322
2323 if (!CI)
2324 break;
2325
2326 if (objcarc::GetRCIdentityRoot(V: CI) != RetOpnd ||
2327 objcarc::hasAttachedCallOpBundle(CB: CI))
2328 break;
2329
2330 // If we've found an unannotated call that defines RetOpnd, add a
2331 // "clang.arc.attachedcall" operand bundle.
2332 Value *BundleArgs[] = {*objcarc::getAttachedARCFunction(CB: &CB)};
2333 OperandBundleDef OB("clang.arc.attachedcall", BundleArgs);
2334 auto *NewCall = CallBase::addOperandBundle(
2335 CB: CI, ID: LLVMContext::OB_clang_arc_attachedcall, OB, InsertPt: CI->getIterator());
2336 NewCall->copyMetadata(SrcInst: *CI);
2337 CI->replaceAllUsesWith(V: NewCall);
2338 CI->eraseFromParent();
2339 InsertRetainCall = false;
2340 break;
2341 }
2342
2343 if (InsertRetainCall) {
2344 // The retainRV is attached to the call and we've failed to find a
2345 // matching autoreleaseRV or an annotated call in the callee. Emit a call
2346 // to objc_retain.
2347 Builder.SetInsertPoint(RI);
2348 Builder.CreateIntrinsic(ID: Intrinsic::objc_retain, Args: RetOpnd);
2349 }
2350 }
2351}
2352
2353// In contextual profiling, when an inline succeeds, we want to remap the
2354// indices of the callee into the index space of the caller. We can't just leave
2355// them as-is because the same callee may appear in other places in this caller
2356// (other callsites), and its (callee's) counters and sub-contextual profile
2357// tree would be potentially different.
2358// Not all BBs of the callee may survive the opportunistic DCE InlineFunction
2359// does (same goes for callsites in the callee).
2360// We will return a pair of vectors, one for basic block IDs and one for
2361// callsites. For such a vector V, V[Idx] will be -1 if the callee
2362// instrumentation with index Idx did not survive inlining, and a new value
2363// otherwise.
2364// This function will update the caller's instrumentation intrinsics
2365// accordingly, mapping indices as described above. We also replace the "name"
2366// operand because we use it to distinguish between "own" instrumentation and
2367// "from callee" instrumentation when performing the traversal of the CFG of the
2368// caller. We traverse depth-first from the callsite's BB and up to the point we
2369// hit BBs owned by the caller.
2370// The return values will be then used to update the contextual
2371// profile. Note: we only update the "name" and "index" operands in the
2372// instrumentation intrinsics, we leave the hash and total nr of indices as-is,
2373// it's not worth updating those.
2374static std::pair<std::vector<int64_t>, std::vector<int64_t>>
2375remapIndices(Function &Caller, BasicBlock *StartBB,
2376 PGOContextualProfile &CtxProf, uint32_t CalleeCounters,
2377 uint32_t CalleeCallsites) {
2378 // We'll allocate a new ID to imported callsite counters and callsites. We're
2379 // using -1 to indicate a counter we delete. Most likely the entry ID, for
2380 // example, will be deleted - we don't want 2 IDs in the same BB, and the
2381 // entry would have been cloned in the callsite's old BB.
2382 std::vector<int64_t> CalleeCounterMap;
2383 std::vector<int64_t> CalleeCallsiteMap;
2384 CalleeCounterMap.resize(new_size: CalleeCounters, x: -1);
2385 CalleeCallsiteMap.resize(new_size: CalleeCallsites, x: -1);
2386
2387 auto RewriteInstrIfNeeded = [&](InstrProfIncrementInst &Ins) -> bool {
2388 if (Ins.getNameValue() == &Caller)
2389 return false;
2390 const auto OldID = static_cast<uint32_t>(Ins.getIndex()->getZExtValue());
2391 if (CalleeCounterMap[OldID] == -1)
2392 CalleeCounterMap[OldID] = CtxProf.allocateNextCounterIndex(F: Caller);
2393 const auto NewID = static_cast<uint32_t>(CalleeCounterMap[OldID]);
2394
2395 Ins.setNameValue(&Caller);
2396 Ins.setIndex(NewID);
2397 return true;
2398 };
2399
2400 auto RewriteCallsiteInsIfNeeded = [&](InstrProfCallsite &Ins) -> bool {
2401 if (Ins.getNameValue() == &Caller)
2402 return false;
2403 const auto OldID = static_cast<uint32_t>(Ins.getIndex()->getZExtValue());
2404 if (CalleeCallsiteMap[OldID] == -1)
2405 CalleeCallsiteMap[OldID] = CtxProf.allocateNextCallsiteIndex(F: Caller);
2406 const auto NewID = static_cast<uint32_t>(CalleeCallsiteMap[OldID]);
2407
2408 Ins.setNameValue(&Caller);
2409 Ins.setIndex(NewID);
2410 return true;
2411 };
2412
2413 std::deque<BasicBlock *> Worklist;
2414 DenseSet<const BasicBlock *> Seen;
2415 // We will traverse the BBs starting from the callsite BB. The callsite BB
2416 // will have at least a BB ID - maybe its own, and in any case the one coming
2417 // from the cloned function's entry BB. The other BBs we'll start seeing from
2418 // there on may or may not have BB IDs. BBs with IDs belonging to our caller
2419 // are definitely not coming from the imported function and form a boundary
2420 // past which we don't need to traverse anymore. BBs may have no
2421 // instrumentation (because we originally inserted instrumentation as per
2422 // MST), in which case we'll traverse past them. An invariant we'll keep is
2423 // that a BB will have at most 1 BB ID. For example, in the callsite BB, we
2424 // will delete the callee BB's instrumentation. This doesn't result in
2425 // information loss: the entry BB of the callee will have the same count as
2426 // the callsite's BB. At the end of this traversal, all the callee's
2427 // instrumentation would be mapped into the caller's instrumentation index
2428 // space. Some of the callee's counters may be deleted (as mentioned, this
2429 // should result in no loss of information).
2430 Worklist.push_back(x: StartBB);
2431 while (!Worklist.empty()) {
2432 auto *BB = Worklist.front();
2433 Worklist.pop_front();
2434 bool Changed = false;
2435 auto *BBID = CtxProfAnalysis::getBBInstrumentation(BB&: *BB);
2436 if (BBID) {
2437 Changed |= RewriteInstrIfNeeded(*BBID);
2438 // this may be the entryblock from the inlined callee, coming into a BB
2439 // that didn't have instrumentation because of MST decisions. Let's make
2440 // sure it's placed accordingly. This is a noop elsewhere.
2441 BBID->moveBefore(InsertPos: BB->getFirstInsertionPt());
2442 }
2443 for (auto &I : llvm::make_early_inc_range(Range&: *BB)) {
2444 if (auto *Inc = dyn_cast<InstrProfIncrementInst>(Val: &I)) {
2445 if (isa<InstrProfIncrementInstStep>(Val: Inc)) {
2446 // Step instrumentation is used for select instructions. Inlining may
2447 // have propagated a constant resulting in the condition of the select
2448 // being resolved, case in which function cloning resolves the value
2449 // of the select, and elides the select instruction. If that is the
2450 // case, the step parameter of the instrumentation will reflect that.
2451 // We can delete the instrumentation in that case.
2452 if (isa<Constant>(Val: Inc->getStep())) {
2453 assert(!Inc->getNextNode() || !isa<SelectInst>(Inc->getNextNode()));
2454 Inc->eraseFromParent();
2455 } else {
2456 assert(isa_and_nonnull<SelectInst>(Inc->getNextNode()));
2457 RewriteInstrIfNeeded(*Inc);
2458 }
2459 } else if (Inc != BBID) {
2460 // If we're here it means that the BB had more than 1 IDs, presumably
2461 // some coming from the callee. We "made up our mind" to keep the
2462 // first one (which may or may not have been originally the caller's).
2463 // All the others are superfluous and we delete them.
2464 Inc->eraseFromParent();
2465 Changed = true;
2466 }
2467 } else if (auto *CS = dyn_cast<InstrProfCallsite>(Val: &I)) {
2468 Changed |= RewriteCallsiteInsIfNeeded(*CS);
2469 }
2470 }
2471 if (!BBID || Changed)
2472 for (auto *Succ : successors(BB))
2473 if (Seen.insert(V: Succ).second)
2474 Worklist.push_back(x: Succ);
2475 }
2476
2477 assert(!llvm::is_contained(CalleeCounterMap, 0) &&
2478 "Counter index mapping should be either to -1 or to non-zero index, "
2479 "because the 0 "
2480 "index corresponds to the entry BB of the caller");
2481 assert(!llvm::is_contained(CalleeCallsiteMap, 0) &&
2482 "Callsite index mapping should be either to -1 or to non-zero index, "
2483 "because there should have been at least a callsite - the inlined one "
2484 "- which would have had a 0 index.");
2485
2486 return {std::move(CalleeCounterMap), std::move(CalleeCallsiteMap)};
2487}
2488
2489// Inline. If successful, update the contextual profile (if a valid one is
2490// given).
2491// The contextual profile data is organized in trees, as follows:
2492// - each node corresponds to a function
2493// - the root of each tree corresponds to an "entrypoint" - e.g.
2494// RPC handler for server side
2495// - the path from the root to a node is a particular call path
2496// - the counters stored in a node are counter values observed in that
2497// particular call path ("context")
2498// - the edges between nodes are annotated with callsite IDs.
2499//
2500// Updating the contextual profile after an inlining means, at a high level,
2501// copying over the data of the callee, **intentionally without any value
2502// scaling**, and copying over the callees of the inlined callee.
2503llvm::InlineResult
2504llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
2505 PGOContextualProfile &CtxProf, bool MergeAttributes,
2506 AAResults *CalleeAAR, bool InsertLifetime,
2507 bool TrackInlineHistory, Function *ForwardVarArgsTo,
2508 OptimizationRemarkEmitter *ORE) {
2509 if (!CtxProf.isInSpecializedModule())
2510 return InlineFunction(CB, IFI, MergeAttributes, CalleeAAR, InsertLifetime,
2511 TrackInlineHistory, ForwardVarArgsTo, ORE);
2512
2513 auto &Caller = *CB.getCaller();
2514 auto &Callee = *CB.getCalledFunction();
2515 auto *StartBB = CB.getParent();
2516
2517 // Get some preliminary data about the callsite before it might get inlined.
2518 // Inlining shouldn't delete the callee, but it's cleaner (and low-cost) to
2519 // get this data upfront and rely less on InlineFunction's behavior.
2520 const auto CalleeGUID = Callee.getGUID();
2521 auto *CallsiteIDIns = CtxProfAnalysis::getCallsiteInstrumentation(CB);
2522 const auto CallsiteID =
2523 static_cast<uint32_t>(CallsiteIDIns->getIndex()->getZExtValue());
2524
2525 const auto NumCalleeCounters = CtxProf.getNumCounters(F: Callee);
2526 const auto NumCalleeCallsites = CtxProf.getNumCallsites(F: Callee);
2527
2528 auto Ret = InlineFunction(CB, IFI, MergeAttributes, CalleeAAR, InsertLifetime,
2529 TrackInlineHistory, ForwardVarArgsTo, ORE);
2530 if (!Ret.isSuccess())
2531 return Ret;
2532
2533 // Inlining succeeded, we don't need the instrumentation of the inlined
2534 // callsite.
2535 CallsiteIDIns->eraseFromParent();
2536
2537 // Assinging Maps and then capturing references into it in the lambda because
2538 // captured structured bindings are a C++20 extension. We do also need a
2539 // capture here, though.
2540 const auto IndicesMaps = remapIndices(Caller, StartBB, CtxProf,
2541 CalleeCounters: NumCalleeCounters, CalleeCallsites: NumCalleeCallsites);
2542 const uint32_t NewCountersSize = CtxProf.getNumCounters(F: Caller);
2543
2544 auto Updater = [&](PGOCtxProfContext &Ctx) {
2545 assert(Ctx.guid() == Caller.getGUID());
2546 const auto &[CalleeCounterMap, CalleeCallsiteMap] = IndicesMaps;
2547 assert(
2548 (Ctx.counters().size() +
2549 llvm::count_if(CalleeCounterMap, [](auto V) { return V != -1; }) ==
2550 NewCountersSize) &&
2551 "The caller's counters size should have grown by the number of new "
2552 "distinct counters inherited from the inlined callee.");
2553 Ctx.resizeCounters(Size: NewCountersSize);
2554 // If the callsite wasn't exercised in this context, the value of the
2555 // counters coming from it is 0 - which it is right now, after resizing them
2556 // - and so we're done.
2557 auto CSIt = Ctx.callsites().find(x: CallsiteID);
2558 if (CSIt == Ctx.callsites().end())
2559 return;
2560 auto CalleeCtxIt = CSIt->second.find(x: CalleeGUID);
2561 // The callsite was exercised, but not with this callee (so presumably this
2562 // is an indirect callsite). Again, we're done here.
2563 if (CalleeCtxIt == CSIt->second.end())
2564 return;
2565
2566 // Let's pull in the counter values and the subcontexts coming from the
2567 // inlined callee.
2568 auto &CalleeCtx = CalleeCtxIt->second;
2569 assert(CalleeCtx.guid() == CalleeGUID);
2570
2571 for (auto I = 0U; I < CalleeCtx.counters().size(); ++I) {
2572 const int64_t NewIndex = CalleeCounterMap[I];
2573 if (NewIndex >= 0) {
2574 assert(NewIndex != 0 && "counter index mapping shouldn't happen to a 0 "
2575 "index, that's the caller's entry BB");
2576 Ctx.counters()[NewIndex] = CalleeCtx.counters()[I];
2577 }
2578 }
2579 for (auto &[I, OtherSet] : CalleeCtx.callsites()) {
2580 const int64_t NewCSIdx = CalleeCallsiteMap[I];
2581 if (NewCSIdx >= 0) {
2582 assert(NewCSIdx != 0 &&
2583 "callsite index mapping shouldn't happen to a 0 index, the "
2584 "caller must've had at least one callsite (with such an index)");
2585 Ctx.ingestAllContexts(CSId: NewCSIdx, Other: std::move(OtherSet));
2586 }
2587 }
2588 // We know the traversal is preorder, so it wouldn't have yet looked at the
2589 // sub-contexts of this context that it's currently visiting. Meaning, the
2590 // erase below invalidates no iterators.
2591 auto Deleted = Ctx.callsites().erase(x: CallsiteID);
2592 assert(Deleted);
2593 (void)Deleted;
2594 };
2595 CtxProf.update(Updater, F: Caller);
2596 return Ret;
2597}
2598
2599llvm::InlineResult llvm::CanInlineCallSite(const CallBase &CB,
2600 InlineFunctionInfo &IFI) {
2601 assert(CB.getParent() && CB.getFunction() && "Instruction not in function!");
2602
2603 // FIXME: we don't inline callbr yet.
2604 if (isa<CallBrInst>(Val: CB))
2605 return InlineResult::failure(Reason: "We don't inline callbr yet.");
2606
2607 // If IFI has any state in it, zap it before we fill it in.
2608 IFI.reset();
2609
2610 Function *CalledFunc = CB.getCalledFunction();
2611 if (!CalledFunc || // Can't inline external function or indirect
2612 CalledFunc->isDeclaration()) // call!
2613 return InlineResult::failure(Reason: "external or indirect");
2614
2615 // Don't inline if we've already inlined this callee through this call site
2616 // before to prevent infinite inlining through mutually recursive functions.
2617 if (MDNode *InlineHistory = CB.getMetadata(KindID: LLVMContext::MD_inline_history)) {
2618 for (const auto &Op : InlineHistory->operands()) {
2619 if (auto *MD = dyn_cast_or_null<ValueAsMetadata>(Val: Op)) {
2620 if (MD->getValue() == CalledFunc) {
2621 return InlineResult::failure(Reason: "inline history");
2622 }
2623 }
2624 }
2625 }
2626
2627 // The inliner does not know how to inline through calls with operand bundles
2628 // in general ...
2629 if (CB.hasOperandBundles()) {
2630 for (int i = 0, e = CB.getNumOperandBundles(); i != e; ++i) {
2631 auto OBUse = CB.getOperandBundleAt(Index: i);
2632 uint32_t Tag = OBUse.getTagID();
2633 // ... but it knows how to inline through "deopt" operand bundles ...
2634 if (Tag == LLVMContext::OB_deopt)
2635 continue;
2636 // ... and "funclet" operand bundles.
2637 if (Tag == LLVMContext::OB_funclet)
2638 continue;
2639 if (Tag == LLVMContext::OB_clang_arc_attachedcall)
2640 continue;
2641 if (Tag == LLVMContext::OB_kcfi)
2642 continue;
2643 if (Tag == LLVMContext::OB_convergencectrl) {
2644 IFI.ConvergenceControlToken = OBUse.Inputs[0].get();
2645 continue;
2646 }
2647
2648 return InlineResult::failure(Reason: "unsupported operand bundle");
2649 }
2650 }
2651
2652 // FIXME: The check below is redundant and incomplete. According to spec, if a
2653 // convergent call is missing a token, then the caller is using uncontrolled
2654 // convergence. If the callee has an entry intrinsic, then the callee is using
2655 // controlled convergence, and the call cannot be inlined. A proper
2656 // implemenation of this check requires a whole new analysis that identifies
2657 // convergence in every function. For now, we skip that and just do this one
2658 // cursory check. The underlying assumption is that in a compiler flow that
2659 // fully implements convergence control tokens, there is no mixing of
2660 // controlled and uncontrolled convergent operations in the whole program.
2661 if (CB.isConvergent()) {
2662 if (!IFI.ConvergenceControlToken &&
2663 getConvergenceEntry(BB&: CalledFunc->getEntryBlock())) {
2664 return InlineResult::failure(
2665 Reason: "convergent call needs convergencectrl operand");
2666 }
2667 }
2668
2669 const BasicBlock *OrigBB = CB.getParent();
2670 const Function *Caller = OrigBB->getParent();
2671
2672 // GC poses two hazards to inlining, which only occur when the callee has GC:
2673 // 1. If the caller has no GC, then the callee's GC must be propagated to the
2674 // caller.
2675 // 2. If the caller has a differing GC, it is invalid to inline.
2676 if (CalledFunc->hasGC()) {
2677 if (Caller->hasGC() && CalledFunc->getGC() != Caller->getGC())
2678 return InlineResult::failure(Reason: "incompatible GC");
2679 }
2680
2681 // Get the personality function from the callee if it contains a landing pad.
2682 Constant *CalledPersonality =
2683 CalledFunc->hasPersonalityFn()
2684 ? CalledFunc->getPersonalityFn()->stripPointerCasts()
2685 : nullptr;
2686
2687 // Find the personality function used by the landing pads of the caller. If it
2688 // exists, then check to see that it matches the personality function used in
2689 // the callee.
2690 Constant *CallerPersonality =
2691 Caller->hasPersonalityFn()
2692 ? Caller->getPersonalityFn()->stripPointerCasts()
2693 : nullptr;
2694 if (CalledPersonality) {
2695 // If the personality functions match, then we can perform the
2696 // inlining. Otherwise, we can't inline.
2697 // TODO: This isn't 100% true. Some personality functions are proper
2698 // supersets of others and can be used in place of the other.
2699 if (CallerPersonality && CalledPersonality != CallerPersonality)
2700 return InlineResult::failure(Reason: "incompatible personality");
2701 }
2702
2703 // We need to figure out which funclet the callsite was in so that we may
2704 // properly nest the callee.
2705 if (CallerPersonality) {
2706 EHPersonality Personality = classifyEHPersonality(Pers: CallerPersonality);
2707 if (isScopedEHPersonality(Pers: Personality)) {
2708 std::optional<OperandBundleUse> ParentFunclet =
2709 CB.getOperandBundle(ID: LLVMContext::OB_funclet);
2710 if (ParentFunclet)
2711 IFI.CallSiteEHPad = cast<FuncletPadInst>(Val: ParentFunclet->Inputs.front());
2712
2713 // OK, the inlining site is legal. What about the target function?
2714
2715 if (IFI.CallSiteEHPad) {
2716 if (Personality == EHPersonality::MSVC_CXX) {
2717 // The MSVC personality cannot tolerate catches getting inlined into
2718 // cleanup funclets.
2719 if (isa<CleanupPadInst>(Val: IFI.CallSiteEHPad)) {
2720 // Ok, the call site is within a cleanuppad. Let's check the callee
2721 // for catchpads.
2722 for (const BasicBlock &CalledBB : *CalledFunc) {
2723 if (isa<CatchSwitchInst>(Val: CalledBB.getFirstNonPHIIt()))
2724 return InlineResult::failure(Reason: "catch in cleanup funclet");
2725 }
2726 }
2727 } else if (isAsynchronousEHPersonality(Pers: Personality)) {
2728 // SEH is even less tolerant, there may not be any sort of exceptional
2729 // funclet in the callee.
2730 for (const BasicBlock &CalledBB : *CalledFunc) {
2731 if (CalledBB.isEHPad())
2732 return InlineResult::failure(Reason: "SEH in cleanup funclet");
2733 }
2734 }
2735 }
2736 }
2737 }
2738
2739 return InlineResult::success();
2740}
2741
2742/// This function inlines the called function into the basic block of the
2743/// caller. This returns false if it is not possible to inline this call.
2744/// The program is still in a well defined state if this occurs though.
2745///
2746/// Note that this only does one level of inlining. For example, if the
2747/// instruction 'call B' is inlined, and 'B' calls 'C', then the call to 'C' now
2748/// exists in the instruction stream. Similarly this will inline a recursive
2749/// function by one level.
2750void llvm::InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI,
2751 bool MergeAttributes, AAResults *CalleeAAR,
2752 bool InsertLifetime, bool TrackInlineHistory,
2753 Function *ForwardVarArgsTo,
2754 OptimizationRemarkEmitter *ORE) {
2755 BasicBlock *OrigBB = CB.getParent();
2756 Function *Caller = OrigBB->getParent();
2757 Function *CalledFunc = CB.getCalledFunction();
2758 assert(CalledFunc && !CalledFunc->isDeclaration() &&
2759 "CanInlineCallSite should have verified direct call to definition");
2760
2761 // Determine if we are dealing with a call in an EHPad which does not unwind
2762 // to caller.
2763 bool EHPadForCallUnwindsLocally = false;
2764 if (IFI.CallSiteEHPad && isa<CallInst>(Val: CB)) {
2765 UnwindDestMemoTy FuncletUnwindMap;
2766 Value *CallSiteUnwindDestToken =
2767 getUnwindDestToken(EHPad: IFI.CallSiteEHPad, MemoMap&: FuncletUnwindMap);
2768
2769 EHPadForCallUnwindsLocally =
2770 CallSiteUnwindDestToken &&
2771 !isa<ConstantTokenNone>(Val: CallSiteUnwindDestToken);
2772 }
2773
2774 // Get an iterator to the last basic block in the function, which will have
2775 // the new function inlined after it.
2776 Function::iterator LastBlock = --Caller->end();
2777
2778 // Make sure to capture all of the return instructions from the cloned
2779 // function.
2780 SmallVector<ReturnInst*, 8> Returns;
2781 ClonedCodeInfo InlinedFunctionInfo;
2782 Function::iterator FirstNewBlock;
2783
2784 // GC poses two hazards to inlining, which only occur when the callee has GC:
2785 // 1. If the caller has no GC, then the callee's GC must be propagated to the
2786 // caller.
2787 // 2. If the caller has a differing GC, it is invalid to inline.
2788 if (CalledFunc->hasGC()) {
2789 if (!Caller->hasGC())
2790 Caller->setGC(CalledFunc->getGC());
2791 else {
2792 assert(CalledFunc->getGC() == Caller->getGC() &&
2793 "CanInlineCallSite should have verified compatible GCs");
2794 }
2795 }
2796
2797 if (CalledFunc->hasPersonalityFn()) {
2798 Constant *CalledPersonality =
2799 CalledFunc->getPersonalityFn()->stripPointerCasts();
2800 if (!Caller->hasPersonalityFn()) {
2801 Caller->setPersonalityFn(CalledPersonality);
2802 } else
2803 assert(Caller->getPersonalityFn()->stripPointerCasts() ==
2804 CalledPersonality &&
2805 "CanInlineCallSite should have verified compatible personality");
2806 }
2807
2808 { // Scope to destroy VMap after cloning.
2809 ValueToValueMapTy VMap;
2810 struct ByValInit {
2811 Value *Dst;
2812 Value *Src;
2813 MaybeAlign SrcAlign;
2814 Type *Ty;
2815 };
2816 // Keep a list of tuples (dst, src, src_align) to emit byval
2817 // initializations. Src Alignment is only available though the callbase,
2818 // therefore has to be saved.
2819 SmallVector<ByValInit, 4> ByValInits;
2820
2821 // When inlining a function that contains noalias scope metadata,
2822 // this metadata needs to be cloned so that the inlined blocks
2823 // have different "unique scopes" at every call site.
2824 // Track the metadata that must be cloned. Do this before other changes to
2825 // the function, so that we do not get in trouble when inlining caller ==
2826 // callee.
2827 ScopedAliasMetadataDeepCloner SAMetadataCloner(CB.getCalledFunction());
2828
2829 auto &DL = Caller->getDataLayout();
2830
2831 // Calculate the vector of arguments to pass into the function cloner, which
2832 // matches up the formal to the actual argument values.
2833 auto AI = CB.arg_begin();
2834 unsigned ArgNo = 0;
2835 for (Function::arg_iterator I = CalledFunc->arg_begin(),
2836 E = CalledFunc->arg_end(); I != E; ++I, ++AI, ++ArgNo) {
2837 Value *ActualArg = *AI;
2838
2839 // When byval arguments actually inlined, we need to make the copy implied
2840 // by them explicit. However, we don't do this if the callee is readonly
2841 // or readnone, because the copy would be unneeded: the callee doesn't
2842 // modify the struct.
2843 if (CB.isByValArgument(ArgNo)) {
2844 ActualArg = HandleByValArgument(ByValType: CB.getParamByValType(ArgNo), Arg: ActualArg,
2845 TheCall: &CB, CalledFunc, IFI,
2846 ByValAlignment: CalledFunc->getParamAlign(ArgNo));
2847 if (ActualArg != *AI)
2848 ByValInits.push_back(Elt: {.Dst: ActualArg, .Src: (Value *)*AI,
2849 .SrcAlign: CB.getParamAlign(ArgNo),
2850 .Ty: CB.getParamByValType(ArgNo)});
2851 }
2852
2853 VMap[&*I] = ActualArg;
2854 }
2855
2856 // TODO: Remove this when users have been updated to the assume bundles.
2857 // Add alignment assumptions if necessary. We do this before the inlined
2858 // instructions are actually cloned into the caller so that we can easily
2859 // check what will be known at the start of the inlined code.
2860 AddAlignmentAssumptions(CB, IFI);
2861
2862 AssumptionCache *AC =
2863 IFI.GetAssumptionCache ? &IFI.GetAssumptionCache(*Caller) : nullptr;
2864
2865 /// Preserve all attributes on of the call and its parameters.
2866 salvageKnowledge(I: &CB, AC);
2867
2868 // We want the inliner to prune the code as it copies. We would LOVE to
2869 // have no dead or constant instructions leftover after inlining occurs
2870 // (which can happen, e.g., because an argument was constant), but we'll be
2871 // happy with whatever the cloner can do.
2872 CloneAndPruneFunctionInto(NewFunc: Caller, OldFunc: CalledFunc, VMap,
2873 /*ModuleLevelChanges=*/false, Returns, NameSuffix: ".i",
2874 CodeInfo&: InlinedFunctionInfo);
2875 // Remember the first block that is newly cloned over.
2876 FirstNewBlock = LastBlock; ++FirstNewBlock;
2877
2878 // Insert retainRV/clainRV runtime calls.
2879 objcarc::ARCInstKind RVCallKind = objcarc::getAttachedARCFunctionKind(CB: &CB);
2880 if (RVCallKind != objcarc::ARCInstKind::None)
2881 inlineRetainOrClaimRVCalls(CB, RVCallKind, Returns);
2882
2883 // Updated caller/callee profiles only when requested. For sample loader
2884 // inlining, the context-sensitive inlinee profile doesn't need to be
2885 // subtracted from callee profile, and the inlined clone also doesn't need
2886 // to be scaled based on call site count.
2887 if (IFI.UpdateProfile) {
2888 if (IFI.CallerBFI != nullptr && IFI.CalleeBFI != nullptr)
2889 // Update the BFI of blocks cloned into the caller.
2890 updateCallerBFI(CallSiteBlock: OrigBB, VMap, CallerBFI: IFI.CallerBFI, CalleeBFI: IFI.CalleeBFI,
2891 CalleeEntryBlock: CalledFunc->front());
2892
2893 if (auto Profile = CalledFunc->getEntryCount())
2894 updateCallProfile(Callee: CalledFunc, VMap, CalleeEntryCount: *Profile, TheCall: CB, PSI: IFI.PSI,
2895 CallerBFI: IFI.CallerBFI);
2896 }
2897
2898 // Inject byval arguments initialization.
2899 for (ByValInit &Init : ByValInits)
2900 HandleByValArgumentInit(ByValType: Init.Ty, Dst: Init.Dst, Src: Init.Src, SrcAlign: Init.SrcAlign,
2901 M: Caller->getParent(), InsertBlock: &*FirstNewBlock, IFI,
2902 CalledFunc);
2903
2904 std::optional<OperandBundleUse> ParentDeopt =
2905 CB.getOperandBundle(ID: LLVMContext::OB_deopt);
2906 if (ParentDeopt) {
2907 SmallVector<OperandBundleDef, 2> OpDefs;
2908
2909 for (auto &VH : InlinedFunctionInfo.OperandBundleCallSites) {
2910 CallBase *ICS = dyn_cast_or_null<CallBase>(Val&: VH);
2911 if (!ICS)
2912 continue; // instruction was DCE'd or RAUW'ed to undef
2913
2914 OpDefs.clear();
2915
2916 OpDefs.reserve(N: ICS->getNumOperandBundles());
2917
2918 for (unsigned COBi = 0, COBe = ICS->getNumOperandBundles(); COBi < COBe;
2919 ++COBi) {
2920 auto ChildOB = ICS->getOperandBundleAt(Index: COBi);
2921 if (ChildOB.getTagID() != LLVMContext::OB_deopt) {
2922 // If the inlined call has other operand bundles, let them be
2923 OpDefs.emplace_back(Args&: ChildOB);
2924 continue;
2925 }
2926
2927 // It may be useful to separate this logic (of handling operand
2928 // bundles) out to a separate "policy" component if this gets crowded.
2929 // Prepend the parent's deoptimization continuation to the newly
2930 // inlined call's deoptimization continuation.
2931 std::vector<Value *> MergedDeoptArgs;
2932 MergedDeoptArgs.reserve(n: ParentDeopt->Inputs.size() +
2933 ChildOB.Inputs.size());
2934
2935 llvm::append_range(C&: MergedDeoptArgs, R&: ParentDeopt->Inputs);
2936 llvm::append_range(C&: MergedDeoptArgs, R&: ChildOB.Inputs);
2937
2938 OpDefs.emplace_back(Args: "deopt", Args: std::move(MergedDeoptArgs));
2939 }
2940
2941 Instruction *NewI = CallBase::Create(CB: ICS, Bundles: OpDefs, InsertPt: ICS->getIterator());
2942
2943 // Note: the RAUW does the appropriate fixup in VMap, so we need to do
2944 // this even if the call returns void.
2945 ICS->replaceAllUsesWith(V: NewI);
2946
2947 VH = nullptr;
2948 ICS->eraseFromParent();
2949 }
2950 }
2951
2952 // For 'nodebug' functions, the associated DISubprogram is always null.
2953 // Conservatively avoid propagating the callsite debug location to
2954 // instructions inlined from a function whose DISubprogram is not null.
2955 fixupLineNumbers(Fn: Caller, FI: FirstNewBlock, TheCall: &CB,
2956 CalleeHasDebugInfo: CalledFunc->getSubprogram() != nullptr);
2957
2958 if (isAssignmentTrackingEnabled(M: *Caller->getParent())) {
2959 // Interpret inlined stores to caller-local variables as assignments.
2960 trackInlinedStores(Start: FirstNewBlock, End: Caller->end(), CB);
2961
2962 // Update DIAssignID metadata attachments and uses so that they are
2963 // unique to this inlined instance.
2964 fixupAssignments(Start: FirstNewBlock, End: Caller->end());
2965 }
2966
2967 // Now clone the inlined noalias scope metadata.
2968 SAMetadataCloner.clone();
2969 SAMetadataCloner.remap(FStart: FirstNewBlock, FEnd: Caller->end());
2970
2971 // Add noalias metadata if necessary.
2972 AddAliasScopeMetadata(CB, VMap, DL, CalleeAAR, InlinedFunctionInfo);
2973
2974 // Clone return attributes on the callsite into the calls within the inlined
2975 // function which feed into its return value.
2976 AddReturnAttributes(CB, VMap, InlinedFunctionInfo);
2977
2978 // Clone attributes on the params of the callsite to calls within the
2979 // inlined function which use the same param.
2980 AddParamAndFnBasicAttributes(CB, VMap, InlinedFunctionInfo);
2981
2982 propagateMemProfMetadata(
2983 Callee: CalledFunc, CB, ContainsMemProfMetadata: InlinedFunctionInfo.ContainsMemProfMetadata, VMap, ORE);
2984
2985 // Propagate metadata on the callsite if necessary.
2986 PropagateCallSiteMetadata(CB, FStart: FirstNewBlock, FEnd: Caller->end());
2987
2988 // Propagate an allocation wrapper's !alloc_token if necessary.
2989 propagateAllocTokenMetadata(CalledFunc, CB, VMap, InlinedFunctionInfo);
2990
2991 // Propagate implicit ref metadata.
2992 if (CalledFunc->hasMetadata(KindID: LLVMContext::MD_implicit_ref)) {
2993 SmallVector<MDNode *> MDs;
2994 CalledFunc->getMetadata(KindID: LLVMContext::MD_implicit_ref, MDs);
2995 for (MDNode *MD : MDs) {
2996 Caller->addMetadata(KindID: LLVMContext::MD_implicit_ref, MD&: *MD);
2997 }
2998 }
2999
3000 // Propagate inlined.from metadata for dontcall diagnostics.
3001 PropagateInlinedFromMetadata(CB, CalledFuncName: CalledFunc->getName(), CallerFuncName: Caller->getName(),
3002 FStart: FirstNewBlock, FEnd: Caller->end());
3003
3004 // Register any cloned assumptions.
3005 if (IFI.GetAssumptionCache)
3006 for (BasicBlock &NewBlock :
3007 make_range(x: FirstNewBlock->getIterator(), y: Caller->end()))
3008 for (Instruction &I : NewBlock)
3009 if (auto *II = dyn_cast<AssumeInst>(Val: &I))
3010 IFI.GetAssumptionCache(*Caller).registerAssumption(CI: II);
3011 }
3012
3013 if (IFI.ConvergenceControlToken) {
3014 IntrinsicInst *IntrinsicCall = getConvergenceEntry(BB&: *FirstNewBlock);
3015 if (IntrinsicCall) {
3016 IntrinsicCall->replaceAllUsesWith(V: IFI.ConvergenceControlToken);
3017 IntrinsicCall->eraseFromParent();
3018 }
3019 }
3020
3021 // If there are any alloca instructions in the block that used to be the entry
3022 // block for the callee, move them to the entry block of the caller. First
3023 // calculate which instruction they should be inserted before. We insert the
3024 // instructions at the end of the current alloca list.
3025 {
3026 BasicBlock::iterator InsertPoint = Caller->begin()->begin();
3027 for (BasicBlock::iterator I = FirstNewBlock->begin(),
3028 E = FirstNewBlock->end(); I != E; ) {
3029 AllocaInst *AI = dyn_cast<AllocaInst>(Val: I++);
3030 if (!AI) continue;
3031
3032 // If the alloca is now dead, remove it. This often occurs due to code
3033 // specialization.
3034 if (AI->use_empty()) {
3035 AI->eraseFromParent();
3036 continue;
3037 }
3038
3039 if (!allocaWouldBeStaticInEntry(AI))
3040 continue;
3041
3042 // Keep track of the static allocas that we inline into the caller.
3043 IFI.StaticAllocas.push_back(Elt: AI);
3044
3045 // Scan for the block of allocas that we can move over, and move them
3046 // all at once.
3047 while (isa<AllocaInst>(Val: I) &&
3048 !cast<AllocaInst>(Val&: I)->use_empty() &&
3049 allocaWouldBeStaticInEntry(AI: cast<AllocaInst>(Val&: I))) {
3050 IFI.StaticAllocas.push_back(Elt: cast<AllocaInst>(Val&: I));
3051 ++I;
3052 }
3053
3054 // Transfer all of the allocas over in a block. Using splice means
3055 // that the instructions aren't removed from the symbol table, then
3056 // reinserted.
3057 I.setTailBit(true);
3058 Caller->getEntryBlock().splice(ToIt: InsertPoint, FromBB: &*FirstNewBlock,
3059 FromBeginIt: AI->getIterator(), FromEndIt: I);
3060 }
3061 }
3062
3063 // If the call to the callee cannot throw, set the 'nounwind' flag on any
3064 // calls that we inline.
3065 bool MarkNoUnwind = CB.doesNotThrow();
3066
3067 SmallVector<Value*,4> VarArgsToForward;
3068 SmallVector<AttributeSet, 4> VarArgsAttrs;
3069 for (unsigned i = CalledFunc->getFunctionType()->getNumParams();
3070 i < CB.arg_size(); i++) {
3071 VarArgsToForward.push_back(Elt: CB.getArgOperand(i));
3072 VarArgsAttrs.push_back(Elt: CB.getAttributes().getParamAttrs(ArgNo: i));
3073 }
3074
3075 bool InlinedMustTailCalls = false, InlinedDeoptimizeCalls = false;
3076 if (InlinedFunctionInfo.ContainsCalls) {
3077 CallInst::TailCallKind CallSiteTailKind = CallInst::TCK_None;
3078 if (CallInst *CI = dyn_cast<CallInst>(Val: &CB))
3079 CallSiteTailKind = CI->getTailCallKind();
3080
3081 // For inlining purposes, the "notail" marker is the same as no marker.
3082 if (CallSiteTailKind == CallInst::TCK_NoTail)
3083 CallSiteTailKind = CallInst::TCK_None;
3084
3085 for (Function::iterator BB = FirstNewBlock, E = Caller->end(); BB != E;
3086 ++BB) {
3087 for (Instruction &I : llvm::make_early_inc_range(Range&: *BB)) {
3088 CallInst *CI = dyn_cast<CallInst>(Val: &I);
3089 if (!CI)
3090 continue;
3091
3092 // Forward varargs from inlined call site to calls to the
3093 // ForwardVarArgsTo function, if requested, and to musttail calls.
3094 if (!VarArgsToForward.empty() &&
3095 ((ForwardVarArgsTo &&
3096 CI->getCalledFunction() == ForwardVarArgsTo) ||
3097 CI->isMustTailCall())) {
3098 // Collect attributes for non-vararg parameters.
3099 AttributeList Attrs = CI->getAttributes();
3100 SmallVector<AttributeSet, 8> ArgAttrs;
3101 if (!Attrs.isEmpty() || !VarArgsAttrs.empty()) {
3102 for (unsigned ArgNo = 0;
3103 ArgNo < CI->getFunctionType()->getNumParams(); ++ArgNo)
3104 ArgAttrs.push_back(Elt: Attrs.getParamAttrs(ArgNo));
3105 }
3106
3107 // Add VarArg attributes.
3108 ArgAttrs.append(in_start: VarArgsAttrs.begin(), in_end: VarArgsAttrs.end());
3109 Attrs = AttributeList::get(C&: CI->getContext(), FnAttrs: Attrs.getFnAttrs(),
3110 RetAttrs: Attrs.getRetAttrs(), ArgAttrs);
3111 // Add VarArgs to existing parameters.
3112 SmallVector<Value *, 6> Params(CI->args());
3113 Params.append(in_start: VarArgsToForward.begin(), in_end: VarArgsToForward.end());
3114 CallInst *NewCI = CallInst::Create(
3115 Ty: CI->getFunctionType(), Func: CI->getCalledOperand(), Args: Params, NameStr: "", InsertBefore: CI->getIterator());
3116 NewCI->setDebugLoc(CI->getDebugLoc());
3117 NewCI->setAttributes(Attrs);
3118 NewCI->setCallingConv(CI->getCallingConv());
3119 CI->replaceAllUsesWith(V: NewCI);
3120 CI->eraseFromParent();
3121 CI = NewCI;
3122 }
3123
3124 if (Function *F = CI->getCalledFunction())
3125 InlinedDeoptimizeCalls |=
3126 F->getIntrinsicID() == Intrinsic::experimental_deoptimize;
3127
3128 // We need to reduce the strength of any inlined tail calls. For
3129 // musttail, we have to avoid introducing potential unbounded stack
3130 // growth. For example, if functions 'f' and 'g' are mutually recursive
3131 // with musttail, we can inline 'g' into 'f' so long as we preserve
3132 // musttail on the cloned call to 'f'. If either the inlined call site
3133 // or the cloned call site is *not* musttail, the program already has
3134 // one frame of stack growth, so it's safe to remove musttail. Here is
3135 // a table of example transformations:
3136 //
3137 // f -> musttail g -> musttail f ==> f -> musttail f
3138 // f -> musttail g -> tail f ==> f -> tail f
3139 // f -> g -> musttail f ==> f -> f
3140 // f -> g -> tail f ==> f -> f
3141 //
3142 // Inlined notail calls should remain notail calls.
3143 CallInst::TailCallKind ChildTCK = CI->getTailCallKind();
3144 if (ChildTCK != CallInst::TCK_NoTail)
3145 ChildTCK = std::min(a: CallSiteTailKind, b: ChildTCK);
3146 CI->setTailCallKind(ChildTCK);
3147 InlinedMustTailCalls |= CI->isMustTailCall();
3148
3149 // Call sites inlined through a 'nounwind' call site should be
3150 // 'nounwind' as well. However, avoid marking call sites explicitly
3151 // where possible. This helps expose more opportunities for CSE after
3152 // inlining, commonly when the callee is an intrinsic.
3153 if (MarkNoUnwind && !CI->doesNotThrow())
3154 CI->setDoesNotThrow();
3155 }
3156 }
3157 }
3158
3159 // Leave lifetime markers for the static alloca's, scoping them to the
3160 // function we just inlined.
3161 // We need to insert lifetime intrinsics even at O0 to avoid invalid
3162 // access caused by multithreaded coroutines. The check
3163 // `Caller->isPresplitCoroutine()` would affect AlwaysInliner at O0 only.
3164 if ((InsertLifetime || Caller->isPresplitCoroutine()) &&
3165 !IFI.StaticAllocas.empty()) {
3166 IRBuilder<> builder(FirstNewBlock->begin());
3167 for (AllocaInst *AI : IFI.StaticAllocas) {
3168 // Don't mark swifterror allocas. They can't have bitcast uses.
3169 if (AI->isSwiftError())
3170 continue;
3171
3172 // If the alloca is already scoped to something smaller than the whole
3173 // function then there's no need to add redundant, less accurate markers.
3174 if (hasLifetimeMarkers(AI))
3175 continue;
3176
3177 std::optional<TypeSize> Size = AI->getAllocationSize(DL: AI->getDataLayout());
3178 if (Size && Size->isZero())
3179 continue;
3180
3181 builder.CreateLifetimeStart(Ptr: AI);
3182 for (ReturnInst *RI : Returns) {
3183 // Don't insert llvm.lifetime.end calls between a musttail or deoptimize
3184 // call and a return. The return kills all local allocas.
3185 if (InlinedMustTailCalls &&
3186 RI->getParent()->getTerminatingMustTailCall())
3187 continue;
3188 if (InlinedDeoptimizeCalls &&
3189 RI->getParent()->getTerminatingDeoptimizeCall())
3190 continue;
3191 IRBuilder<>(RI).CreateLifetimeEnd(Ptr: AI);
3192 }
3193 }
3194 }
3195
3196 // If the inlined code contained dynamic alloca instructions, wrap the inlined
3197 // code with llvm.stacksave/llvm.stackrestore intrinsics.
3198 if (InlinedFunctionInfo.ContainsDynamicAllocas) {
3199 // Insert the llvm.stacksave.
3200 CallInst *SavedPtr =
3201 IRBuilder<>(FirstNewBlock->begin()).CreateStackSave(Name: "savedstack");
3202
3203 // Insert a call to llvm.stackrestore before any return instructions in the
3204 // inlined function.
3205 for (ReturnInst *RI : Returns) {
3206 // Don't insert llvm.stackrestore calls between a musttail or deoptimize
3207 // call and a return. The return will restore the stack pointer.
3208 if (InlinedMustTailCalls && RI->getParent()->getTerminatingMustTailCall())
3209 continue;
3210 if (InlinedDeoptimizeCalls && RI->getParent()->getTerminatingDeoptimizeCall())
3211 continue;
3212 IRBuilder<>(RI).CreateStackRestore(Ptr: SavedPtr);
3213 }
3214 }
3215
3216 // If we are inlining for an invoke instruction, we must make sure to rewrite
3217 // any call instructions into invoke instructions. This is sensitive to which
3218 // funclet pads were top-level in the inlinee, so must be done before
3219 // rewriting the "parent pad" links.
3220 if (auto *II = dyn_cast<InvokeInst>(Val: &CB)) {
3221 BasicBlock *UnwindDest = II->getUnwindDest();
3222 BasicBlock::iterator FirstNonPHI = UnwindDest->getFirstNonPHIIt();
3223 if (isa<LandingPadInst>(Val: FirstNonPHI)) {
3224 HandleInlinedLandingPad(II, FirstNewBlock: &*FirstNewBlock, InlinedCodeInfo&: InlinedFunctionInfo);
3225 } else {
3226 HandleInlinedEHPad(II, FirstNewBlock: &*FirstNewBlock, InlinedCodeInfo&: InlinedFunctionInfo);
3227 }
3228 }
3229
3230 // Update the lexical scopes of the new funclets and callsites.
3231 // Anything that had 'none' as its parent is now nested inside the callsite's
3232 // EHPad.
3233 if (IFI.CallSiteEHPad) {
3234 for (Function::iterator BB = FirstNewBlock->getIterator(),
3235 E = Caller->end();
3236 BB != E; ++BB) {
3237 // Add bundle operands to inlined call sites.
3238 PropagateOperandBundles(InlinedBB: BB, CallSiteEHPad: IFI.CallSiteEHPad);
3239
3240 // It is problematic if the inlinee has a cleanupret which unwinds to
3241 // caller and we inline it into a call site which doesn't unwind but into
3242 // an EH pad that does. Such an edge must be dynamically unreachable.
3243 // As such, we replace the cleanupret with unreachable.
3244 if (auto *CleanupRet = dyn_cast<CleanupReturnInst>(Val: BB->getTerminator()))
3245 if (CleanupRet->unwindsToCaller() && EHPadForCallUnwindsLocally)
3246 changeToUnreachable(I: CleanupRet);
3247
3248 BasicBlock::iterator I = BB->getFirstNonPHIIt();
3249 if (!I->isEHPad())
3250 continue;
3251
3252 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val&: I)) {
3253 if (isa<ConstantTokenNone>(Val: CatchSwitch->getParentPad()))
3254 CatchSwitch->setParentPad(IFI.CallSiteEHPad);
3255 } else {
3256 auto *FPI = cast<FuncletPadInst>(Val&: I);
3257 if (isa<ConstantTokenNone>(Val: FPI->getParentPad()))
3258 FPI->setParentPad(IFI.CallSiteEHPad);
3259 }
3260 }
3261 }
3262
3263 if (InlinedDeoptimizeCalls) {
3264 // We need to at least remove the deoptimizing returns from the Return set,
3265 // so that the control flow from those returns does not get merged into the
3266 // caller (but terminate it instead). If the caller's return type does not
3267 // match the callee's return type, we also need to change the return type of
3268 // the intrinsic.
3269 if (Caller->getReturnType() == CB.getType()) {
3270 llvm::erase_if(C&: Returns, P: [](ReturnInst *RI) {
3271 return RI->getParent()->getTerminatingDeoptimizeCall() != nullptr;
3272 });
3273 } else {
3274 SmallVector<ReturnInst *, 8> NormalReturns;
3275 Function *NewDeoptIntrinsic = Intrinsic::getOrInsertDeclaration(
3276 M: Caller->getParent(), id: Intrinsic::experimental_deoptimize,
3277 OverloadTys: {Caller->getReturnType()});
3278
3279 for (ReturnInst *RI : Returns) {
3280 CallInst *DeoptCall = RI->getParent()->getTerminatingDeoptimizeCall();
3281 if (!DeoptCall) {
3282 NormalReturns.push_back(Elt: RI);
3283 continue;
3284 }
3285
3286 // The calling convention on the deoptimize call itself may be bogus,
3287 // since the code we're inlining may have undefined behavior (and may
3288 // never actually execute at runtime); but all
3289 // @llvm.experimental.deoptimize declarations have to have the same
3290 // calling convention in a well-formed module.
3291 auto CallingConv = DeoptCall->getCalledFunction()->getCallingConv();
3292 NewDeoptIntrinsic->setCallingConv(CallingConv);
3293 auto *CurBB = RI->getParent();
3294 RI->eraseFromParent();
3295
3296 SmallVector<Value *, 4> CallArgs(DeoptCall->args());
3297
3298 SmallVector<OperandBundleDef, 1> OpBundles;
3299 DeoptCall->getOperandBundlesAsDefs(Defs&: OpBundles);
3300 auto DeoptAttributes = DeoptCall->getAttributes();
3301 DeoptCall->eraseFromParent();
3302 assert(!OpBundles.empty() &&
3303 "Expected at least the deopt operand bundle");
3304
3305 IRBuilder<> Builder(CurBB);
3306 CallInst *NewDeoptCall =
3307 Builder.CreateCall(Callee: NewDeoptIntrinsic, Args: CallArgs, OpBundles);
3308 NewDeoptCall->setCallingConv(CallingConv);
3309 NewDeoptCall->setAttributes(DeoptAttributes);
3310 if (NewDeoptCall->getType()->isVoidTy())
3311 Builder.CreateRetVoid();
3312 else
3313 Builder.CreateRet(V: NewDeoptCall);
3314 // Since the ret type is changed, remove the incompatible attributes.
3315 NewDeoptCall->removeRetAttrs(AttrsToRemove: AttributeFuncs::typeIncompatible(
3316 Ty: NewDeoptCall->getType(), AS: NewDeoptCall->getRetAttributes()));
3317 }
3318
3319 // Leave behind the normal returns so we can merge control flow.
3320 std::swap(LHS&: Returns, RHS&: NormalReturns);
3321 }
3322 }
3323
3324 // Handle any inlined musttail call sites. In order for a new call site to be
3325 // musttail, the source of the clone and the inlined call site must have been
3326 // musttail. Therefore it's safe to return without merging control into the
3327 // phi below.
3328 if (InlinedMustTailCalls) {
3329 // Handle the returns preceded by musttail calls separately.
3330 SmallVector<ReturnInst *, 8> NormalReturns;
3331 for (ReturnInst *RI : Returns) {
3332 CallInst *ReturnedMustTail =
3333 RI->getParent()->getTerminatingMustTailCall();
3334 if (!ReturnedMustTail)
3335 NormalReturns.push_back(Elt: RI);
3336 }
3337
3338 // Leave behind the normal returns so we can merge control flow.
3339 std::swap(LHS&: Returns, RHS&: NormalReturns);
3340 }
3341
3342 // Now that all of the transforms on the inlined code have taken place but
3343 // before we splice the inlined code into the CFG and lose track of which
3344 // blocks were actually inlined, collect the call sites. We only do this if
3345 // call graph updates weren't requested, as those provide value handle based
3346 // tracking of inlined call sites instead. Calls to intrinsics are not
3347 // collected because they are not inlineable.
3348 if (InlinedFunctionInfo.ContainsCalls) {
3349 // Otherwise just collect the raw call sites that were inlined.
3350 for (BasicBlock &NewBB :
3351 make_range(x: FirstNewBlock->getIterator(), y: Caller->end()))
3352 for (Instruction &I : NewBB)
3353 if (auto *CB = dyn_cast<CallBase>(Val: &I))
3354 if (!(CB->getCalledFunction() &&
3355 CB->getCalledFunction()->isIntrinsic()))
3356 IFI.InlinedCallSites.push_back(Elt: CB);
3357 }
3358
3359 for (CallBase *ICB : IFI.InlinedCallSites) {
3360 // We only track inline history if requested, or if the inlined call site
3361 // was originally an indirect call (it may have become a direct call
3362 // during inlining).
3363 if (TrackInlineHistory ||
3364 InlinedFunctionInfo.OriginallyIndirectCalls.contains(key: ICB)) {
3365 // !inline_history is {Callee, CB.inline_history, ICB.inline_history}.
3366 // Metadata nodes may be null if the referenced function was erased from
3367 // the module.
3368 SmallVector<Metadata *, 4> History;
3369 History.push_back(Elt: ValueAsMetadata::get(V: CalledFunc));
3370 if (MDNode *CBHistory = CB.getMetadata(KindID: LLVMContext::MD_inline_history)) {
3371 for (const auto &Op : CBHistory->operands()) {
3372 if (Op)
3373 History.push_back(Elt: Op.get());
3374 }
3375 }
3376 if (MDNode *CBHistory =
3377 ICB->getMetadata(KindID: LLVMContext::MD_inline_history)) {
3378 for (const auto &Op : CBHistory->operands()) {
3379 if (Op)
3380 History.push_back(Elt: Op.get());
3381 }
3382 }
3383 MDNode *NewHistory = MDNode::get(Context&: Caller->getContext(), MDs: History);
3384 ICB->setMetadata(KindID: LLVMContext::MD_inline_history, Node: NewHistory);
3385 }
3386 }
3387
3388 // If we cloned in _exactly one_ basic block, and if that block ends in a
3389 // return instruction, we splice the body of the inlined callee directly into
3390 // the calling basic block.
3391 if (Returns.size() == 1 && std::distance(first: FirstNewBlock, last: Caller->end()) == 1) {
3392 // Move all of the instructions right before the call.
3393 OrigBB->splice(ToIt: CB.getIterator(), FromBB: &*FirstNewBlock, FromBeginIt: FirstNewBlock->begin(),
3394 FromEndIt: FirstNewBlock->end());
3395 // Remove the cloned basic block.
3396 Caller->back().eraseFromParent();
3397
3398 // If the call site was an invoke instruction, add a branch to the normal
3399 // destination.
3400 if (InvokeInst *II = dyn_cast<InvokeInst>(Val: &CB)) {
3401 UncondBrInst *NewBr =
3402 UncondBrInst::Create(Target: II->getNormalDest(), InsertBefore: CB.getIterator());
3403 NewBr->setDebugLoc(Returns[0]->getDebugLoc());
3404 }
3405
3406 // If the return instruction returned a value, replace uses of the call with
3407 // uses of the returned value.
3408 if (!CB.use_empty()) {
3409 ReturnInst *R = Returns[0];
3410 if (&CB == R->getReturnValue())
3411 CB.replaceAllUsesWith(V: PoisonValue::get(T: CB.getType()));
3412 else
3413 CB.replaceAllUsesWith(V: R->getReturnValue());
3414 }
3415 // Since we are now done with the Call/Invoke, we can delete it.
3416 CB.eraseFromParent();
3417
3418 // Since we are now done with the return instruction, delete it also.
3419 Returns[0]->eraseFromParent();
3420
3421 if (MergeAttributes)
3422 AttributeFuncs::mergeAttributesForInlining(Caller&: *Caller, Callee: *CalledFunc);
3423
3424 // We are now done with the inlining.
3425 return;
3426 }
3427
3428 // Otherwise, we have the normal case, of more than one block to inline or
3429 // multiple return sites.
3430
3431 // We want to clone the entire callee function into the hole between the
3432 // "starter" and "ender" blocks. How we accomplish this depends on whether
3433 // this is an invoke instruction or a call instruction.
3434 BasicBlock *AfterCallBB;
3435 UncondBrInst *CreatedBranchToNormalDest = nullptr;
3436 if (InvokeInst *II = dyn_cast<InvokeInst>(Val: &CB)) {
3437
3438 // Add an unconditional branch to make this look like the CallInst case...
3439 CreatedBranchToNormalDest =
3440 UncondBrInst::Create(Target: II->getNormalDest(), InsertBefore: CB.getIterator());
3441 // We intend to replace this DebugLoc with another later.
3442 CreatedBranchToNormalDest->setDebugLoc(DebugLoc::getTemporary());
3443
3444 // Split the basic block. This guarantees that no PHI nodes will have to be
3445 // updated due to new incoming edges, and make the invoke case more
3446 // symmetric to the call case.
3447 AfterCallBB =
3448 OrigBB->splitBasicBlock(I: CreatedBranchToNormalDest->getIterator(),
3449 BBName: CalledFunc->getName() + ".exit");
3450
3451 } else { // It's a call
3452 // If this is a call instruction, we need to split the basic block that
3453 // the call lives in.
3454 //
3455 AfterCallBB = OrigBB->splitBasicBlock(I: CB.getIterator(),
3456 BBName: CalledFunc->getName() + ".exit");
3457 }
3458
3459 if (IFI.CallerBFI) {
3460 // Copy original BB's block frequency to AfterCallBB
3461 IFI.CallerBFI->setBlockFreq(BB: AfterCallBB,
3462 Freq: IFI.CallerBFI->getBlockFreq(BB: OrigBB));
3463 }
3464
3465 // Change the branch that used to go to AfterCallBB to branch to the first
3466 // basic block of the inlined function.
3467 //
3468 UncondBrInst *Br = cast<UncondBrInst>(Val: OrigBB->getTerminator());
3469 Br->setSuccessor(&*FirstNewBlock);
3470
3471 // Now that the function is correct, make it a little bit nicer. In
3472 // particular, move the basic blocks inserted from the end of the function
3473 // into the space made by splitting the source basic block.
3474 Caller->splice(ToIt: AfterCallBB->getIterator(), FromF: Caller, FromBeginIt: FirstNewBlock,
3475 FromEndIt: Caller->end());
3476
3477 // Handle all of the return instructions that we just cloned in, and eliminate
3478 // any users of the original call/invoke instruction.
3479 Type *RTy = CalledFunc->getReturnType();
3480
3481 PHINode *PHI = nullptr;
3482 if (Returns.size() > 1) {
3483 // The PHI node should go at the front of the new basic block to merge all
3484 // possible incoming values.
3485 if (!CB.use_empty()) {
3486 PHI = PHINode::Create(Ty: RTy, NumReservedValues: Returns.size(), NameStr: CB.getName());
3487 PHI->insertBefore(InsertPos: AfterCallBB->begin());
3488 // Anything that used the result of the function call should now use the
3489 // PHI node as their operand.
3490 CB.replaceAllUsesWith(V: PHI);
3491 }
3492
3493 // Loop over all of the return instructions adding entries to the PHI node
3494 // as appropriate.
3495 if (PHI) {
3496 for (ReturnInst *RI : Returns) {
3497 assert(RI->getReturnValue()->getType() == PHI->getType() &&
3498 "Ret value not consistent in function!");
3499 PHI->addIncoming(V: RI->getReturnValue(), BB: RI->getParent());
3500 }
3501 }
3502
3503 // Add a branch to the merge points and remove return instructions.
3504 DebugLoc Loc;
3505 for (ReturnInst *RI : Returns) {
3506 UncondBrInst *BI = UncondBrInst::Create(Target: AfterCallBB, InsertBefore: RI->getIterator());
3507 Loc = RI->getDebugLoc();
3508 BI->setDebugLoc(Loc);
3509 RI->eraseFromParent();
3510 }
3511 // We need to set the debug location to *somewhere* inside the
3512 // inlined function. The line number may be nonsensical, but the
3513 // instruction will at least be associated with the right
3514 // function.
3515 if (CreatedBranchToNormalDest)
3516 CreatedBranchToNormalDest->setDebugLoc(Loc);
3517 } else if (!Returns.empty()) {
3518 // Otherwise, if there is exactly one return value, just replace anything
3519 // using the return value of the call with the computed value.
3520 if (!CB.use_empty()) {
3521 if (&CB == Returns[0]->getReturnValue())
3522 CB.replaceAllUsesWith(V: PoisonValue::get(T: CB.getType()));
3523 else
3524 CB.replaceAllUsesWith(V: Returns[0]->getReturnValue());
3525 }
3526
3527 // Update PHI nodes that use the ReturnBB to use the AfterCallBB.
3528 BasicBlock *ReturnBB = Returns[0]->getParent();
3529 ReturnBB->replaceAllUsesWith(V: AfterCallBB);
3530
3531 // Splice the code from the return block into the block that it will return
3532 // to, which contains the code that was after the call.
3533 AfterCallBB->splice(ToIt: AfterCallBB->begin(), FromBB: ReturnBB);
3534
3535 if (CreatedBranchToNormalDest)
3536 CreatedBranchToNormalDest->setDebugLoc(Returns[0]->getDebugLoc());
3537
3538 // Delete the return instruction now and empty ReturnBB now.
3539 Returns[0]->eraseFromParent();
3540 ReturnBB->eraseFromParent();
3541 } else if (!CB.use_empty()) {
3542 // In this case there are no returns to use, so there is no clear source
3543 // location for the "return".
3544 // FIXME: It may be correct to use the scope end line of the function here,
3545 // since this likely means we are falling out of the function.
3546 if (CreatedBranchToNormalDest)
3547 CreatedBranchToNormalDest->setDebugLoc(DebugLoc::getUnknown());
3548 // No returns, but something is using the return value of the call. Just
3549 // nuke the result.
3550 CB.replaceAllUsesWith(V: PoisonValue::get(T: CB.getType()));
3551 }
3552
3553 // Since we are now done with the Call/Invoke, we can delete it.
3554 CB.eraseFromParent();
3555
3556 // If we inlined any musttail calls and the original return is now
3557 // unreachable, delete it. It can only contain a ret.
3558 if (InlinedMustTailCalls && pred_empty(BB: AfterCallBB))
3559 AfterCallBB->eraseFromParent();
3560
3561 // We should always be able to fold the entry block of the function into the
3562 // single predecessor of the block...
3563 BasicBlock *CalleeEntry = Br->getSuccessor();
3564
3565 // Splice the code entry block into calling block, right before the
3566 // unconditional branch.
3567 CalleeEntry->replaceAllUsesWith(V: OrigBB); // Update PHI nodes
3568 OrigBB->splice(ToIt: Br->getIterator(), FromBB: CalleeEntry);
3569
3570 // Remove the unconditional branch.
3571 Br->eraseFromParent();
3572
3573 // Now we can remove the CalleeEntry block, which is now empty.
3574 CalleeEntry->eraseFromParent();
3575
3576 // If we inserted a phi node, check to see if it has a single value (e.g. all
3577 // the entries are the same or undef). If so, remove the PHI so it doesn't
3578 // block other optimizations.
3579 if (PHI) {
3580 AssumptionCache *AC =
3581 IFI.GetAssumptionCache ? &IFI.GetAssumptionCache(*Caller) : nullptr;
3582 auto &DL = Caller->getDataLayout();
3583 if (Value *V = simplifyInstruction(I: PHI, Q: {DL, nullptr, nullptr, AC})) {
3584 PHI->replaceAllUsesWith(V);
3585 PHI->eraseFromParent();
3586 }
3587 }
3588
3589 if (MergeAttributes)
3590 AttributeFuncs::mergeAttributesForInlining(Caller&: *Caller, Callee: *CalledFunc);
3591}
3592
3593llvm::InlineResult llvm::InlineFunction(
3594 CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes,
3595 AAResults *CalleeAAR, bool InsertLifetime, bool TrackInlineHistory,
3596 Function *ForwardVarArgsTo, OptimizationRemarkEmitter *ORE) {
3597 llvm::InlineResult Result = CanInlineCallSite(CB, IFI);
3598 if (Result.isSuccess()) {
3599 InlineFunctionImpl(CB, IFI, MergeAttributes, CalleeAAR, InsertLifetime,
3600 TrackInlineHistory, ForwardVarArgsTo, ORE);
3601 }
3602
3603 return Result;
3604}
3605