1//===- CloneFunction.cpp - Clone a function into another function ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the CloneFunctionInto interface, which is used as the
10// low-level function cloner. This is used by the CloneFunction and function
11// inliner to do the dirty work of copying the body of a function around.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/ADT/SmallVector.h"
16#include "llvm/ADT/SmallVectorExtras.h"
17#include "llvm/ADT/Statistic.h"
18#include "llvm/Analysis/ConstantFolding.h"
19#include "llvm/Analysis/DomTreeUpdater.h"
20#include "llvm/Analysis/InstructionSimplify.h"
21#include "llvm/Analysis/LoopInfo.h"
22#include "llvm/IR/AttributeMask.h"
23#include "llvm/IR/CFG.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DebugInfo.h"
26#include "llvm/IR/DerivedTypes.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/InstIterator.h"
29#include "llvm/IR/Instructions.h"
30#include "llvm/IR/IntrinsicInst.h"
31#include "llvm/IR/LLVMContext.h"
32#include "llvm/IR/MDBuilder.h"
33#include "llvm/IR/Metadata.h"
34#include "llvm/IR/Module.h"
35#include "llvm/Transforms/Utils/BasicBlockUtils.h"
36#include "llvm/Transforms/Utils/Cloning.h"
37#include "llvm/Transforms/Utils/Local.h"
38#include "llvm/Transforms/Utils/ValueMapper.h"
39#include <cstdint>
40#include <map>
41#include <optional>
42using namespace llvm;
43
44#define DEBUG_TYPE "clone-function"
45
46STATISTIC(RemappedAtomMax, "Highest global NextAtomGroup (after mapping)");
47
48void llvm::mapAtomInstance(const DebugLoc &DL, ValueToValueMapTy &VMap) {
49 uint64_t CurGroup = DL->getAtomGroup();
50 if (!CurGroup)
51 return;
52
53 // Try inserting a new entry. If there's already a mapping for this atom
54 // then there's nothing to do.
55 auto [It, Inserted] = VMap.AtomMap.insert(KV: {{DL.getInlinedAt(), CurGroup}, 0});
56 if (!Inserted)
57 return;
58
59 // Map entry to a new atom group.
60 uint64_t NewGroup = DL->getContext().incNextDILocationAtomGroup();
61 assert(NewGroup > CurGroup && "Next should always be greater than current");
62 It->second = NewGroup;
63
64 RemappedAtomMax = std::max<uint64_t>(a: NewGroup, b: RemappedAtomMax);
65}
66
67static void collectDebugInfoFromInstructions(const Function &F,
68 DebugInfoFinder &DIFinder) {
69 const Module *M = F.getParent();
70 if (!M)
71 return;
72 // Inspect instructions to process e.g. DILexicalBlocks of inlined functions
73 for (const Instruction &I : instructions(F))
74 DIFinder.processInstruction(M: *M, I);
75}
76
77// Create a predicate that matches the metadata that should be identity mapped
78// during function cloning.
79static MetadataPredicate
80createIdentityMDPredicate(const Function &F, CloneFunctionChangeType Changes) {
81 if (Changes >= CloneFunctionChangeType::DifferentModule)
82 return [](const Metadata *MD) { return false; };
83
84 DISubprogram *SPClonedWithinModule = F.getSubprogram();
85
86 // Don't clone inlined subprograms.
87 auto ShouldKeep = [SPClonedWithinModule](const DISubprogram *SP) -> bool {
88 return SP != SPClonedWithinModule;
89 };
90
91 return [=](const Metadata *MD) {
92 // Avoid cloning compile units.
93 if (isa<DICompileUnit>(Val: MD))
94 return true;
95
96 if (auto *SP = dyn_cast<DISubprogram>(Val: MD))
97 return ShouldKeep(SP);
98
99 // If a subprogram isn't going to be cloned skip its lexical blocks as well.
100 if (auto *LScope = dyn_cast<DILocalScope>(Val: MD))
101 return ShouldKeep(LScope->getSubprogram());
102
103 // Avoid cloning local variables of subprograms that won't be cloned.
104 if (auto *DV = dyn_cast<DILocalVariable>(Val: MD))
105 if (auto *S = dyn_cast_or_null<DILocalScope>(Val: DV->getScope()))
106 return ShouldKeep(S->getSubprogram());
107
108 // Clone types that are local to subprograms being cloned.
109 // Avoid cloning other types.
110 auto *Type = dyn_cast<DIType>(Val: MD);
111 if (!Type)
112 return false;
113
114 // No need to clone types if subprograms are not cloned.
115 if (SPClonedWithinModule == nullptr)
116 return true;
117
118 // Scopeless types may be derived from local types (e.g. pointers to local
119 // types). They may need cloning.
120 if (const DIDerivedType *DTy = dyn_cast_or_null<DIDerivedType>(Val: Type);
121 DTy && !DTy->getScope())
122 return false;
123
124 auto *LScope = dyn_cast_or_null<DILocalScope>(Val: Type->getScope());
125 if (!LScope)
126 return true;
127
128 if (ShouldKeep(LScope->getSubprogram()))
129 return true;
130
131 return false;
132 };
133}
134
135/// See comments in Cloning.h.
136BasicBlock *llvm::CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap,
137 const Twine &NameSuffix, Function *F,
138 ClonedCodeInfo *CodeInfo, bool MapAtoms) {
139 BasicBlock *NewBB = BasicBlock::Create(Context&: BB->getContext(), Name: "", Parent: F);
140 if (BB->hasName())
141 NewBB->setName(BB->getName() + NameSuffix);
142
143 bool hasCalls = false, hasDynamicAllocas = false, hasMemProfMetadata = false;
144
145 // Loop over all instructions, and copy them over.
146 for (const Instruction &I : *BB) {
147 Instruction *NewInst = I.clone();
148 if (I.hasName())
149 NewInst->setName(I.getName() + NameSuffix);
150
151 NewInst->insertBefore(BB&: *NewBB, InsertPos: NewBB->end());
152 NewInst->cloneDebugInfoFrom(From: &I);
153
154 VMap[&I] = NewInst; // Add instruction map to value.
155
156 if (MapAtoms) {
157 if (const DebugLoc &DL = NewInst->getDebugLoc())
158 mapAtomInstance(DL: DL.get(), VMap);
159 }
160
161 if (isa<CallInst>(Val: I) && !I.isDebugOrPseudoInst()) {
162 hasCalls = true;
163 hasMemProfMetadata |= I.hasMetadata(KindID: LLVMContext::MD_memprof);
164 hasMemProfMetadata |= I.hasMetadata(KindID: LLVMContext::MD_callsite);
165 }
166 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Val: &I)) {
167 if (!AI->isStaticAlloca()) {
168 hasDynamicAllocas = true;
169 }
170 }
171 }
172
173 if (CodeInfo) {
174 CodeInfo->ContainsCalls |= hasCalls;
175 CodeInfo->ContainsMemProfMetadata |= hasMemProfMetadata;
176 CodeInfo->ContainsDynamicAllocas |= hasDynamicAllocas;
177 }
178 return NewBB;
179}
180
181void llvm::CloneFunctionAttributesInto(Function *NewFunc,
182 const Function *OldFunc,
183 ValueToValueMapTy &VMap,
184 bool ModuleLevelChanges,
185 ValueMapTypeRemapper *TypeMapper,
186 ValueMaterializer *Materializer) {
187 // Copy all attributes other than those stored in Function's AttributeList
188 // which holds e.g. parameters and return value attributes.
189 AttributeList NewAttrs = NewFunc->getAttributes();
190 NewFunc->copyAttributesFrom(Src: OldFunc);
191 NewFunc->setAttributes(NewAttrs);
192
193 const RemapFlags FuncGlobalRefFlags =
194 ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges;
195
196 // Fix up the personality function that got copied over.
197 if (OldFunc->hasPersonalityFn())
198 NewFunc->setPersonalityFn(MapValue(V: OldFunc->getPersonalityFn(), VM&: VMap,
199 Flags: FuncGlobalRefFlags, TypeMapper,
200 Materializer));
201
202 if (OldFunc->hasPrefixData()) {
203 NewFunc->setPrefixData(MapValue(V: OldFunc->getPrefixData(), VM&: VMap,
204 Flags: FuncGlobalRefFlags, TypeMapper,
205 Materializer));
206 }
207
208 if (OldFunc->hasPrologueData()) {
209 NewFunc->setPrologueData(MapValue(V: OldFunc->getPrologueData(), VM&: VMap,
210 Flags: FuncGlobalRefFlags, TypeMapper,
211 Materializer));
212 }
213
214 SmallVector<AttributeSet, 4> NewArgAttrs(NewFunc->arg_size());
215 AttributeList OldAttrs = OldFunc->getAttributes();
216
217 // Clone any argument attributes that are present in the VMap.
218 for (const Argument &OldArg : OldFunc->args()) {
219 if (Argument *NewArg = dyn_cast<Argument>(Val&: VMap[&OldArg])) {
220 // Remap the parameter indices.
221 NewArgAttrs[NewArg->getArgNo()] =
222 OldAttrs.getParamAttrs(ArgNo: OldArg.getArgNo());
223 }
224 }
225
226 NewFunc->setAttributes(
227 AttributeList::get(C&: NewFunc->getContext(), FnAttrs: OldAttrs.getFnAttrs(),
228 RetAttrs: OldAttrs.getRetAttrs(), ArgAttrs: NewArgAttrs));
229}
230
231void llvm::CloneFunctionMetadataInto(Function &NewFunc, const Function &OldFunc,
232 ValueToValueMapTy &VMap,
233 RemapFlags RemapFlag,
234 ValueMapTypeRemapper *TypeMapper,
235 ValueMaterializer *Materializer,
236 const MetadataPredicate *IdentityMD) {
237 SmallVector<std::pair<unsigned, MDNode *>, 1> MDs;
238 OldFunc.getAllMetadata(MDs);
239 for (const auto &[Kind, MD] : MDs) {
240 NewFunc.addMetadata(KindID: Kind, MD&: *MapMetadata(MD, VM&: VMap, Flags: RemapFlag, TypeMapper,
241 Materializer, IdentityMD));
242 }
243}
244
245void llvm::CloneFunctionBodyInto(Function &NewFunc, const Function &OldFunc,
246 ValueToValueMapTy &VMap, RemapFlags RemapFlag,
247 SmallVectorImpl<ReturnInst *> &Returns,
248 const char *NameSuffix,
249 ClonedCodeInfo *CodeInfo,
250 ValueMapTypeRemapper *TypeMapper,
251 ValueMaterializer *Materializer,
252 const MetadataPredicate *IdentityMD) {
253 if (OldFunc.isDeclaration())
254 return;
255
256 // Loop over all of the basic blocks in the function, cloning them as
257 // appropriate. Note that we save BE this way in order to handle cloning of
258 // recursive functions into themselves.
259 for (const BasicBlock &BB : OldFunc) {
260 // Create a new basic block and copy instructions into it!
261 BasicBlock *CBB =
262 CloneBasicBlock(BB: &BB, VMap, NameSuffix, F: &NewFunc, CodeInfo);
263
264 // Add basic block mapping.
265 VMap[&BB] = CBB;
266
267 // It is only legal to clone a function if a block address within that
268 // function is never referenced outside of the function. Given that, we
269 // want to map block addresses from the old function to block addresses in
270 // the clone. (This is different from the generic ValueMapper
271 // implementation, which generates an invalid blockaddress when
272 // cloning a function.)
273 if (BB.hasAddressTaken()) {
274 Constant *OldBBAddr = BlockAddress::get(F: const_cast<Function *>(&OldFunc),
275 BB: const_cast<BasicBlock *>(&BB));
276 VMap[OldBBAddr] = BlockAddress::get(F: &NewFunc, BB: CBB);
277 }
278
279 // Note return instructions for the caller.
280 if (ReturnInst *RI = dyn_cast<ReturnInst>(Val: CBB->getTerminator()))
281 Returns.push_back(Elt: RI);
282 }
283
284 // Loop over all of the instructions in the new function, fixing up operand
285 // references as we go. This uses VMap to do all the hard work.
286 for (Function::iterator
287 BB = cast<BasicBlock>(Val&: VMap[&OldFunc.front()])->getIterator(),
288 BE = NewFunc.end();
289 BB != BE; ++BB)
290 // Loop over all instructions, fixing each one as we find it, and any
291 // attached debug-info records.
292 for (Instruction &II : *BB) {
293 RemapInstruction(I: &II, VM&: VMap, Flags: RemapFlag, TypeMapper, Materializer,
294 IdentityMD);
295 RemapDbgRecordRange(M: II.getModule(), Range: II.getDbgRecordRange(), VM&: VMap,
296 Flags: RemapFlag, TypeMapper, Materializer, IdentityMD);
297 }
298}
299
300// Clone OldFunc into NewFunc, transforming the old arguments into references to
301// VMap values.
302void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
303 ValueToValueMapTy &VMap,
304 CloneFunctionChangeType Changes,
305 SmallVectorImpl<ReturnInst *> &Returns,
306 const char *NameSuffix, ClonedCodeInfo *CodeInfo,
307 ValueMapTypeRemapper *TypeMapper,
308 ValueMaterializer *Materializer) {
309 assert(NameSuffix && "NameSuffix cannot be null!");
310
311#ifndef NDEBUG
312 for (const Argument &I : OldFunc->args())
313 assert(VMap.count(&I) && "No mapping from source argument specified!");
314#endif
315
316 bool ModuleLevelChanges = Changes > CloneFunctionChangeType::LocalChangesOnly;
317
318 CloneFunctionAttributesInto(NewFunc, OldFunc, VMap, ModuleLevelChanges,
319 TypeMapper, Materializer);
320
321 // Everything else beyond this point deals with function instructions,
322 // so if we are dealing with a function declaration, we're done.
323 if (OldFunc->isDeclaration())
324 return;
325
326 if (Changes < CloneFunctionChangeType::DifferentModule) {
327 assert((NewFunc->getParent() == nullptr ||
328 NewFunc->getParent() == OldFunc->getParent()) &&
329 "Expected NewFunc to have the same parent, or no parent");
330 } else {
331 assert((NewFunc->getParent() == nullptr ||
332 NewFunc->getParent() != OldFunc->getParent()) &&
333 "Expected NewFunc to have different parents, or no parent");
334
335 if (Changes == CloneFunctionChangeType::DifferentModule) {
336 assert(NewFunc->getParent() &&
337 "Need parent of new function to maintain debug info invariants");
338 }
339 }
340
341 MetadataPredicate IdentityMD = createIdentityMDPredicate(F: *OldFunc, Changes);
342
343 // Cloning is always a Module level operation, since Metadata needs to be
344 // cloned.
345 const RemapFlags RemapFlag = RF_None;
346
347 CloneFunctionMetadataInto(NewFunc&: *NewFunc, OldFunc: *OldFunc, VMap, RemapFlag, TypeMapper,
348 Materializer, IdentityMD: &IdentityMD);
349
350 CloneFunctionBodyInto(NewFunc&: *NewFunc, OldFunc: *OldFunc, VMap, RemapFlag, Returns,
351 NameSuffix, CodeInfo, TypeMapper, Materializer,
352 IdentityMD: &IdentityMD);
353
354 // Only update !llvm.dbg.cu for DifferentModule (not CloneModule). In the
355 // same module, the compile unit will already be listed (or not). When
356 // cloning a module, CloneModule() will handle creating the named metadata.
357 if (Changes != CloneFunctionChangeType::DifferentModule)
358 return;
359
360 // Update !llvm.dbg.cu with compile units added to the new module if this
361 // function is being cloned in isolation.
362 //
363 // FIXME: This is making global / module-level changes, which doesn't seem
364 // like the right encapsulation Consider dropping the requirement to update
365 // !llvm.dbg.cu (either obsoleting the node, or restricting it to
366 // non-discardable compile units) instead of discovering compile units by
367 // visiting the metadata attached to global values, which would allow this
368 // code to be deleted. Alternatively, perhaps give responsibility for this
369 // update to CloneFunctionInto's callers.
370 Module *NewModule = NewFunc->getParent();
371 NamedMDNode *NMD = NewModule->getOrInsertNamedMetadata(Name: "llvm.dbg.cu");
372 // Avoid multiple insertions of the same DICompileUnit to NMD.
373 SmallPtrSet<const void *, 8> Visited(llvm::from_range, NMD->operands());
374
375 // Collect and clone all the compile units referenced from the instructions in
376 // the function (e.g. as instructions' scope).
377 DebugInfoFinder DIFinder;
378 collectDebugInfoFromInstructions(F: *OldFunc, DIFinder);
379 for (DICompileUnit *Unit : DIFinder.compile_units()) {
380 MDNode *MappedUnit =
381 MapMetadata(MD: Unit, VM&: VMap, Flags: RF_None, TypeMapper, Materializer);
382 if (Visited.insert(Ptr: MappedUnit).second)
383 NMD->addOperand(M: MappedUnit);
384 }
385}
386
387/// Return a copy of the specified function and add it to that function's
388/// module. Also, any references specified in the VMap are changed to refer to
389/// their mapped value instead of the original one. If any of the arguments to
390/// the function are in the VMap, the arguments are deleted from the resultant
391/// function. The VMap is updated to include mappings from all of the
392/// instructions and basicblocks in the function from their old to new values.
393///
394Function *llvm::CloneFunction(Function *F, ValueToValueMapTy &VMap,
395 ClonedCodeInfo *CodeInfo) {
396 std::vector<Type *> ArgTypes;
397
398 // The user might be deleting arguments to the function by specifying them in
399 // the VMap. If so, we need to not add the arguments to the arg ty vector
400 //
401 for (const Argument &I : F->args())
402 if (VMap.count(Val: &I) == 0) // Haven't mapped the argument to anything yet?
403 ArgTypes.push_back(x: I.getType());
404
405 // Create a new function type...
406 FunctionType *FTy =
407 FunctionType::get(Result: F->getFunctionType()->getReturnType(), Params: ArgTypes,
408 isVarArg: F->getFunctionType()->isVarArg());
409
410 // Create the new function...
411 Function *NewF = Function::Create(Ty: FTy, Linkage: F->getLinkage(), AddrSpace: F->getAddressSpace(),
412 N: F->getName(), M: F->getParent());
413
414 // Loop over the arguments, copying the names of the mapped arguments over...
415 Function::arg_iterator DestI = NewF->arg_begin();
416 for (const Argument &I : F->args())
417 if (VMap.count(Val: &I) == 0) { // Is this argument preserved?
418 DestI->setName(I.getName()); // Copy the name over...
419 VMap[&I] = &*DestI++; // Add mapping to VMap
420 }
421
422 SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned.
423 CloneFunctionInto(NewFunc: NewF, OldFunc: F, VMap, Changes: CloneFunctionChangeType::LocalChangesOnly,
424 Returns, NameSuffix: "", CodeInfo);
425
426 return NewF;
427}
428
429namespace {
430/// This is a private class used to implement CloneAndPruneFunctionInto.
431struct PruningFunctionCloner {
432 Function *NewFunc;
433 const Function *OldFunc;
434 ValueToValueMapTy &VMap;
435 bool ModuleLevelChanges;
436 const char *NameSuffix;
437 ClonedCodeInfo &CodeInfo;
438 bool HostFuncIsStrictFP;
439
440 Instruction *cloneInstruction(BasicBlock::const_iterator II);
441
442public:
443 PruningFunctionCloner(Function *newFunc, const Function *oldFunc,
444 ValueToValueMapTy &valueMap, bool moduleLevelChanges,
445 const char *nameSuffix, ClonedCodeInfo &codeInfo)
446 : NewFunc(newFunc), OldFunc(oldFunc), VMap(valueMap),
447 ModuleLevelChanges(moduleLevelChanges), NameSuffix(nameSuffix),
448 CodeInfo(codeInfo) {
449 HostFuncIsStrictFP =
450 newFunc->getAttributes().hasFnAttr(Kind: Attribute::StrictFP);
451 }
452
453 /// The specified block is found to be reachable, clone it and
454 /// anything that it can reach.
455 void CloneBlock(const BasicBlock *BB, BasicBlock::const_iterator StartingInst,
456 std::vector<const BasicBlock *> &ToClone);
457};
458} // namespace
459
460Instruction *
461PruningFunctionCloner::cloneInstruction(BasicBlock::const_iterator II) {
462 if (!HostFuncIsStrictFP)
463 return II->clone();
464
465 const Instruction &OldInst = *II;
466 Intrinsic::ID CIID = getConstrainedIntrinsicID(Instr: OldInst);
467 if (CIID == Intrinsic::not_intrinsic)
468 return II->clone();
469
470 // Instead of cloning the instruction, a call to constrained intrinsic should
471 // be created. Assume the first arguments of constrained intrinsics are the
472 // same as the operands of original instruction.
473
474 // Create intrinsic call.
475 LLVMContext &Ctx = NewFunc->getContext();
476 SmallVector<Value *, 4> Args;
477 unsigned NumOperands = OldInst.getNumOperands();
478 if (isa<CallInst>(Val: OldInst))
479 --NumOperands;
480 for (unsigned I = 0; I < NumOperands; ++I)
481 Args.push_back(Elt: OldInst.getOperand(i: I));
482
483 if (const auto *CmpI = dyn_cast<FCmpInst>(Val: &OldInst)) {
484 FCmpInst::Predicate Pred = CmpI->getPredicate();
485 StringRef PredName = FCmpInst::getPredicateName(P: Pred);
486 Args.push_back(Elt: MetadataAsValue::get(Context&: Ctx, MD: MDString::get(Context&: Ctx, Str: PredName)));
487 }
488
489 // The last arguments of a constrained intrinsic are metadata that represent
490 // rounding mode (absent in some intrinsics) and exception behavior. The
491 // inlined function uses default settings.
492 if (Intrinsic::hasConstrainedFPRoundingModeOperand(QID: CIID))
493 Args.push_back(
494 Elt: MetadataAsValue::get(Context&: Ctx, MD: MDString::get(Context&: Ctx, Str: "round.tonearest")));
495 Args.push_back(
496 Elt: MetadataAsValue::get(Context&: Ctx, MD: MDString::get(Context&: Ctx, Str: "fpexcept.ignore")));
497
498 SmallVector<Type *> ArgTys = llvm::map_to_vector(C&: Args, F: &Value::getType);
499 Function *IFn = Intrinsic::getOrInsertDeclaration(M: NewFunc->getParent(), IID: CIID,
500 RetTy: OldInst.getType(), ArgTys);
501 return CallInst::Create(Func: IFn, Args, NameStr: OldInst.getName() + ".strict");
502}
503
504/// The specified block is found to be reachable, clone it and
505/// anything that it can reach.
506void PruningFunctionCloner::CloneBlock(
507 const BasicBlock *BB, BasicBlock::const_iterator StartingInst,
508 std::vector<const BasicBlock *> &ToClone) {
509 WeakTrackingVH &BBEntry = VMap[BB];
510
511 // Have we already cloned this block?
512 if (BBEntry)
513 return;
514
515 // Nope, clone it now.
516 BasicBlock *NewBB;
517 Twine NewName(BB->hasName() ? Twine(BB->getName()) + NameSuffix : "");
518 BBEntry = NewBB = BasicBlock::Create(Context&: BB->getContext(), Name: NewName, Parent: NewFunc);
519
520 // It is only legal to clone a function if a block address within that
521 // function is never referenced outside of the function. Given that, we
522 // want to map block addresses from the old function to block addresses in
523 // the clone. (This is different from the generic ValueMapper
524 // implementation, which generates an invalid blockaddress when
525 // cloning a function.)
526 //
527 // Note that we don't need to fix the mapping for unreachable blocks;
528 // the default mapping there is safe.
529 if (BB->hasAddressTaken()) {
530 Constant *OldBBAddr = BlockAddress::get(F: const_cast<Function *>(OldFunc),
531 BB: const_cast<BasicBlock *>(BB));
532 VMap[OldBBAddr] = BlockAddress::get(F: NewFunc, BB: NewBB);
533 }
534
535 bool hasCalls = false, hasDynamicAllocas = false, hasStaticAllocas = false;
536 bool hasMemProfMetadata = false;
537
538 // Keep a cursor pointing at the last place we cloned debug-info records from.
539 BasicBlock::const_iterator DbgCursor = StartingInst;
540 auto CloneDbgRecordsToHere =
541 [&DbgCursor](Instruction *NewInst, BasicBlock::const_iterator II) {
542 // Clone debug-info records onto this instruction. Iterate through any
543 // source-instructions we've cloned and then subsequently optimised
544 // away, so that their debug-info doesn't go missing.
545 for (; DbgCursor != II; ++DbgCursor)
546 NewInst->cloneDebugInfoFrom(From: &*DbgCursor, FromHere: std::nullopt, InsertAtHead: false);
547 NewInst->cloneDebugInfoFrom(From: &*II);
548 DbgCursor = std::next(x: II);
549 };
550
551 // Loop over all instructions, and copy them over, DCE'ing as we go. This
552 // loop doesn't include the terminator.
553 for (BasicBlock::const_iterator II = StartingInst, IE = --BB->end(); II != IE;
554 ++II) {
555
556 // Don't clone fake_use as it may suppress many optimizations
557 // due to inlining, especially SROA.
558 if (auto *IntrInst = dyn_cast<IntrinsicInst>(Val&: II))
559 if (IntrInst->getIntrinsicID() == Intrinsic::fake_use)
560 continue;
561
562 Instruction *NewInst = cloneInstruction(II);
563 NewInst->insertInto(ParentBB: NewBB, It: NewBB->end());
564
565 if (HostFuncIsStrictFP) {
566 // All function calls in the inlined function must get 'strictfp'
567 // attribute to prevent undesirable optimizations.
568 if (auto *Call = dyn_cast<CallInst>(Val: NewInst))
569 Call->addFnAttr(Kind: Attribute::StrictFP);
570 }
571
572 // Eagerly remap operands to the newly cloned instruction, except for PHI
573 // nodes for which we defer processing until we update the CFG.
574 if (!isa<PHINode>(Val: NewInst)) {
575 RemapInstruction(I: NewInst, VM&: VMap,
576 Flags: ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges);
577
578 // Eagerly constant fold the newly cloned instruction. If successful, add
579 // a mapping to the new value. Non-constant operands may be incomplete at
580 // this stage, thus instruction simplification is performed after
581 // processing phi-nodes.
582 if (Value *V = ConstantFoldInstruction(
583 I: NewInst, DL: BB->getDataLayout())) {
584 if (isInstructionTriviallyDead(I: NewInst)) {
585 VMap[&*II] = V;
586 NewInst->eraseFromParent();
587 continue;
588 }
589 }
590 }
591
592 if (auto *CB = dyn_cast<CallBase>(Val&: II); CB && CB->isIndirectCall())
593 CodeInfo.OriginallyIndirectCalls.insert(X: NewInst);
594
595 if (II->hasName())
596 NewInst->setName(II->getName() + NameSuffix);
597 VMap[&*II] = NewInst; // Add instruction map to value.
598 if (isa<CallInst>(Val: II) && !II->isDebugOrPseudoInst()) {
599 hasCalls = true;
600 hasMemProfMetadata |= II->hasMetadata(KindID: LLVMContext::MD_memprof);
601 hasMemProfMetadata |= II->hasMetadata(KindID: LLVMContext::MD_callsite);
602 }
603
604 CloneDbgRecordsToHere(NewInst, II);
605
606 CodeInfo.OrigVMap[&*II] = NewInst;
607 if (auto *CB = dyn_cast<CallBase>(Val: &*II))
608 if (CB->hasOperandBundles())
609 CodeInfo.OperandBundleCallSites.push_back(x: NewInst);
610
611 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Val&: II)) {
612 if (isa<ConstantInt>(Val: AI->getArraySize()))
613 hasStaticAllocas = true;
614 else
615 hasDynamicAllocas = true;
616 }
617 }
618
619 // Finally, clone over the terminator.
620 const Instruction *OldTI = BB->getTerminator();
621 bool TerminatorDone = false;
622 if (const CondBrInst *BI = dyn_cast<CondBrInst>(Val: OldTI)) {
623 // If the condition was a known constant in the callee...
624 ConstantInt *Cond = dyn_cast<ConstantInt>(Val: BI->getCondition());
625 // Or is a known constant in the caller...
626 if (!Cond) {
627 Value *V = VMap.lookup(Val: BI->getCondition());
628 Cond = dyn_cast_or_null<ConstantInt>(Val: V);
629 }
630
631 // Constant fold to uncond branch!
632 if (Cond) {
633 BasicBlock *Dest = BI->getSuccessor(i: !Cond->getZExtValue());
634 auto *NewBI = UncondBrInst::Create(Target: Dest, InsertBefore: NewBB);
635 NewBI->setDebugLoc(BI->getDebugLoc());
636 VMap[OldTI] = NewBI;
637 ToClone.push_back(x: Dest);
638 TerminatorDone = true;
639 }
640 } else if (const SwitchInst *SI = dyn_cast<SwitchInst>(Val: OldTI)) {
641 // If switching on a value known constant in the caller.
642 ConstantInt *Cond = dyn_cast<ConstantInt>(Val: SI->getCondition());
643 if (!Cond) { // Or known constant after constant prop in the callee...
644 Value *V = VMap.lookup(Val: SI->getCondition());
645 Cond = dyn_cast_or_null<ConstantInt>(Val: V);
646 }
647 if (Cond) { // Constant fold to uncond branch!
648 SwitchInst::ConstCaseHandle Case = *SI->findCaseValue(C: Cond);
649 BasicBlock *Dest = const_cast<BasicBlock *>(Case.getCaseSuccessor());
650 auto *NewBI = UncondBrInst::Create(Target: Dest, InsertBefore: NewBB);
651 NewBI->setDebugLoc(SI->getDebugLoc());
652 VMap[OldTI] = NewBI;
653 ToClone.push_back(x: Dest);
654 TerminatorDone = true;
655 }
656 }
657
658 if (!TerminatorDone) {
659 Instruction *NewInst = OldTI->clone();
660 if (OldTI->hasName())
661 NewInst->setName(OldTI->getName() + NameSuffix);
662 NewInst->insertInto(ParentBB: NewBB, It: NewBB->end());
663
664 CloneDbgRecordsToHere(NewInst, OldTI->getIterator());
665
666 VMap[OldTI] = NewInst; // Add instruction map to value.
667
668 CodeInfo.OrigVMap[OldTI] = NewInst;
669 if (auto *CB = dyn_cast<CallBase>(Val: OldTI))
670 if (CB->hasOperandBundles())
671 CodeInfo.OperandBundleCallSites.push_back(x: NewInst);
672
673 // Recursively clone any reachable successor blocks.
674 append_range(C&: ToClone, R: successors(I: BB->getTerminator()));
675 } else {
676 // If we didn't create a new terminator, clone DbgVariableRecords from the
677 // old terminator onto the new terminator.
678 Instruction *NewInst = NewBB->getTerminator();
679 assert(NewInst);
680
681 CloneDbgRecordsToHere(NewInst, OldTI->getIterator());
682 }
683
684 CodeInfo.ContainsCalls |= hasCalls;
685 CodeInfo.ContainsMemProfMetadata |= hasMemProfMetadata;
686 CodeInfo.ContainsDynamicAllocas |= hasDynamicAllocas;
687 CodeInfo.ContainsDynamicAllocas |=
688 hasStaticAllocas && BB != &BB->getParent()->front();
689}
690
691/// This works like CloneAndPruneFunctionInto, except that it does not clone the
692/// entire function. Instead it starts at an instruction provided by the caller
693/// and copies (and prunes) only the code reachable from that instruction.
694void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
695 const Instruction *StartingInst,
696 ValueToValueMapTy &VMap,
697 bool ModuleLevelChanges,
698 SmallVectorImpl<ReturnInst *> &Returns,
699 const char *NameSuffix,
700 ClonedCodeInfo &CodeInfo) {
701 assert(NameSuffix && "NameSuffix cannot be null!");
702
703 ValueMapTypeRemapper *TypeMapper = nullptr;
704 ValueMaterializer *Materializer = nullptr;
705
706#ifndef NDEBUG
707 // If the cloning starts at the beginning of the function, verify that
708 // the function arguments are mapped.
709 if (!StartingInst)
710 for (const Argument &II : OldFunc->args())
711 assert(VMap.count(&II) && "No mapping from source argument specified!");
712#endif
713
714 PruningFunctionCloner PFC(NewFunc, OldFunc, VMap, ModuleLevelChanges,
715 NameSuffix, CodeInfo);
716 const BasicBlock *StartingBB;
717 if (StartingInst)
718 StartingBB = StartingInst->getParent();
719 else {
720 StartingBB = &OldFunc->getEntryBlock();
721 StartingInst = &StartingBB->front();
722 }
723
724 // Clone the entry block, and anything recursively reachable from it.
725 std::vector<const BasicBlock *> CloneWorklist;
726 PFC.CloneBlock(BB: StartingBB, StartingInst: StartingInst->getIterator(), ToClone&: CloneWorklist);
727 while (!CloneWorklist.empty()) {
728 const BasicBlock *BB = CloneWorklist.back();
729 CloneWorklist.pop_back();
730 PFC.CloneBlock(BB, StartingInst: BB->begin(), ToClone&: CloneWorklist);
731 }
732
733 // Loop over all of the basic blocks in the old function. If the block was
734 // reachable, we have cloned it and the old block is now in the value map:
735 // insert it into the new function in the right order. If not, ignore it.
736 //
737 // Defer PHI resolution until rest of function is resolved.
738 SmallVector<const PHINode *, 16> PHIToResolve;
739 for (const BasicBlock &BI : *OldFunc) {
740 Value *V = VMap.lookup(Val: &BI);
741 BasicBlock *NewBB = cast_or_null<BasicBlock>(Val: V);
742 if (!NewBB)
743 continue; // Dead block.
744
745 // Move the new block to preserve the order in the original function.
746 NewBB->moveBefore(MovePos: NewFunc->end());
747
748 // Handle PHI nodes specially, as we have to remove references to dead
749 // blocks.
750 for (const PHINode &PN : BI.phis()) {
751 // PHI nodes may have been remapped to non-PHI nodes by the caller or
752 // during the cloning process.
753 if (isa<PHINode>(Val: VMap[&PN]))
754 PHIToResolve.push_back(Elt: &PN);
755 else
756 break;
757 }
758
759 // Finally, remap the terminator instructions, as those can't be remapped
760 // until all BBs are mapped.
761 RemapInstruction(I: NewBB->getTerminator(), VM&: VMap,
762 Flags: ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges,
763 TypeMapper, Materializer);
764 }
765
766 // Defer PHI resolution until rest of function is resolved, PHI resolution
767 // requires the CFG to be up-to-date.
768 for (unsigned phino = 0, e = PHIToResolve.size(); phino != e;) {
769 const PHINode *OPN = PHIToResolve[phino];
770 unsigned NumPreds = OPN->getNumIncomingValues();
771 const BasicBlock *OldBB = OPN->getParent();
772 BasicBlock *NewBB = cast<BasicBlock>(Val&: VMap[OldBB]);
773
774 // Map operands for blocks that are live and remove operands for blocks
775 // that are dead.
776 for (; phino != PHIToResolve.size() &&
777 PHIToResolve[phino]->getParent() == OldBB;
778 ++phino) {
779 OPN = PHIToResolve[phino];
780 PHINode *PN = cast<PHINode>(Val&: VMap[OPN]);
781 for (int64_t pred = NumPreds - 1; pred >= 0; --pred) {
782 Value *V = VMap.lookup(Val: PN->getIncomingBlock(i: pred));
783 if (BasicBlock *MappedBlock = cast_or_null<BasicBlock>(Val: V)) {
784 Value *InVal =
785 MapValue(V: PN->getIncomingValue(i: pred), VM&: VMap,
786 Flags: ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges);
787 assert(InVal && "Unknown input value?");
788 PN->setIncomingValue(i: pred, V: InVal);
789 PN->setIncomingBlock(i: pred, BB: MappedBlock);
790 continue;
791 }
792 PN->removeIncomingValue(Idx: pred, DeletePHIIfEmpty: false);
793 }
794 }
795
796 // The loop above has removed PHI entries for those blocks that are dead
797 // and has updated others. However, if a block is live (i.e. copied over)
798 // but its terminator has been changed to not go to this block, then our
799 // phi nodes will have invalid entries. Update the PHI nodes in this
800 // case.
801 PHINode *PN = cast<PHINode>(Val: NewBB->begin());
802 NumPreds = pred_size(BB: NewBB);
803 if (NumPreds != PN->getNumIncomingValues()) {
804 assert(NumPreds < PN->getNumIncomingValues());
805 // Count how many times each predecessor comes to this block.
806 DenseMap<BasicBlock *, unsigned> PredCount;
807 for (BasicBlock *Pred : predecessors(BB: NewBB))
808 ++PredCount[Pred];
809
810 BasicBlock::iterator I = NewBB->begin();
811 DenseMap<BasicBlock *, unsigned> SeenPredCount;
812 SeenPredCount.reserve(NumEntries: PredCount.size());
813 for (; (PN = dyn_cast<PHINode>(Val&: I)); ++I) {
814 SeenPredCount.clear();
815 PN->removeIncomingValueIf(
816 Predicate: [&](unsigned Idx) {
817 BasicBlock *IncomingBlock = PN->getIncomingBlock(i: Idx);
818 auto It = PredCount.find(Val: IncomingBlock);
819 if (It == PredCount.end())
820 return true;
821 unsigned &SeenCount = SeenPredCount[IncomingBlock];
822 if (SeenCount < It->second) {
823 SeenCount++;
824 return false;
825 }
826 return true;
827 },
828 DeletePHIIfEmpty: false);
829 }
830 }
831
832 // If the loops above have made these phi nodes have 0 or 1 operand,
833 // replace them with poison or the input value. We must do this for
834 // correctness, because 0-operand phis are not valid.
835 PN = cast<PHINode>(Val: NewBB->begin());
836 if (PN->getNumIncomingValues() == 0) {
837 BasicBlock::iterator I = NewBB->begin();
838 BasicBlock::const_iterator OldI = OldBB->begin();
839 while ((PN = dyn_cast<PHINode>(Val: I++))) {
840 Value *NV = PoisonValue::get(T: PN->getType());
841 PN->replaceAllUsesWith(V: NV);
842 assert(VMap[&*OldI] == PN && "VMap mismatch");
843 VMap[&*OldI] = NV;
844 PN->eraseFromParent();
845 ++OldI;
846 }
847 }
848 }
849
850 // Drop all incompatible return attributes that cannot be applied to NewFunc
851 // during cloning, so as to allow instruction simplification to reason on the
852 // old state of the function. The original attributes are restored later.
853 AttributeList Attrs = NewFunc->getAttributes();
854 AttributeMask IncompatibleAttrs = AttributeFuncs::typeIncompatible(
855 Ty: OldFunc->getReturnType(), AS: Attrs.getRetAttrs());
856 NewFunc->removeRetAttrs(Attrs: IncompatibleAttrs);
857
858 // As phi-nodes have been now remapped, allow incremental simplification of
859 // newly-cloned instructions.
860 const DataLayout &DL = NewFunc->getDataLayout();
861 for (const BasicBlock &BB : *OldFunc) {
862 for (const Instruction &I : BB) {
863 auto *NewI = dyn_cast_or_null<Instruction>(Val: VMap.lookup(Val: &I));
864 if (!NewI)
865 continue;
866
867 if (Value *V = simplifyInstruction(I: NewI, Q: DL)) {
868 NewI->replaceAllUsesWith(V);
869
870 if (isInstructionTriviallyDead(I: NewI)) {
871 NewI->eraseFromParent();
872 } else {
873 // Did not erase it? Restore the new instruction into VMap previously
874 // dropped by `ValueIsRAUWd`.
875 VMap[&I] = NewI;
876 }
877 }
878 }
879 }
880
881 // Restore attributes.
882 NewFunc->setAttributes(Attrs);
883
884 // Remap debug records operands now that all values have been mapped.
885 // Doing this now (late) preserves use-before-defs in debug records. If
886 // we didn't do this, ValueAsMetadata(use-before-def) operands would be
887 // replaced by empty metadata. This would signal later cleanup passes to
888 // remove the debug records, potentially causing incorrect locations.
889 Function::iterator Begin = cast<BasicBlock>(Val&: VMap[StartingBB])->getIterator();
890 for (BasicBlock &BB : make_range(x: Begin, y: NewFunc->end())) {
891 for (Instruction &I : BB) {
892 RemapDbgRecordRange(M: I.getModule(), Range: I.getDbgRecordRange(), VM&: VMap,
893 Flags: ModuleLevelChanges ? RF_None
894 : RF_NoModuleLevelChanges,
895 TypeMapper, Materializer);
896 }
897 }
898
899 // Simplify conditional branches and switches with a constant operand. We try
900 // to prune these out when cloning, but if the simplification required
901 // looking through PHI nodes, those are only available after forming the full
902 // basic block. That may leave some here, and we still want to prune the dead
903 // code as early as possible.
904 for (BasicBlock &BB : make_range(x: Begin, y: NewFunc->end()))
905 ConstantFoldTerminator(BB: &BB);
906
907 // Some blocks may have become unreachable as a result. Find and delete them.
908 {
909 SmallPtrSet<BasicBlock *, 16> ReachableBlocks;
910 SmallVector<BasicBlock *, 16> Worklist;
911 Worklist.push_back(Elt: &*Begin);
912 while (!Worklist.empty()) {
913 BasicBlock *BB = Worklist.pop_back_val();
914 if (ReachableBlocks.insert(Ptr: BB).second)
915 append_range(C&: Worklist, R: successors(BB));
916 }
917
918 SmallVector<BasicBlock *, 16> UnreachableBlocks;
919 for (BasicBlock &BB : make_range(x: Begin, y: NewFunc->end()))
920 if (!ReachableBlocks.contains(Ptr: &BB))
921 UnreachableBlocks.push_back(Elt: &BB);
922 DeleteDeadBlocks(BBs: UnreachableBlocks);
923 }
924
925 // Now that the inlined function body has been fully constructed, go through
926 // and zap unconditional fall-through branches. This happens all the time when
927 // specializing code: code specialization turns conditional branches into
928 // uncond branches, and this code folds them.
929 Function::iterator I = Begin;
930 while (I != NewFunc->end()) {
931 UncondBrInst *BI = dyn_cast<UncondBrInst>(Val: I->getTerminator());
932 if (!BI) {
933 ++I;
934 continue;
935 }
936
937 BasicBlock *Dest = BI->getSuccessor();
938 if (!Dest->getSinglePredecessor() || Dest->hasAddressTaken()) {
939 ++I;
940 continue;
941 }
942
943 // We shouldn't be able to get single-entry PHI nodes here, as instsimplify
944 // above should have zapped all of them..
945 assert(!isa<PHINode>(Dest->begin()));
946
947 // We know all single-entry PHI nodes in the inlined function have been
948 // removed, so we just need to splice the blocks.
949 BI->eraseFromParent();
950
951 // Make all PHI nodes that referred to Dest now refer to I as their source.
952 Dest->replaceAllUsesWith(V: &*I);
953
954 // Move all the instructions in the succ to the pred.
955 I->splice(ToIt: I->end(), FromBB: Dest);
956
957 // Remove the dest block.
958 Dest->eraseFromParent();
959
960 // Do not increment I, iteratively merge all things this block branches to.
961 }
962
963 // Make a final pass over the basic blocks from the old function to gather
964 // any return instructions which survived folding. We have to do this here
965 // because we can iteratively remove and merge returns above.
966 for (Function::iterator I = cast<BasicBlock>(Val&: VMap[StartingBB])->getIterator(),
967 E = NewFunc->end();
968 I != E; ++I)
969 if (ReturnInst *RI = dyn_cast<ReturnInst>(Val: I->getTerminator()))
970 Returns.push_back(Elt: RI);
971}
972
973/// This works exactly like CloneFunctionInto,
974/// except that it does some simple constant prop and DCE on the fly. The
975/// effect of this is to copy significantly less code in cases where (for
976/// example) a function call with constant arguments is inlined, and those
977/// constant arguments cause a significant amount of code in the callee to be
978/// dead. Since this doesn't produce an exact copy of the input, it can't be
979/// used for things like CloneFunction or CloneModule.
980void llvm::CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc,
981 ValueToValueMapTy &VMap,
982 bool ModuleLevelChanges,
983 SmallVectorImpl<ReturnInst *> &Returns,
984 const char *NameSuffix,
985 ClonedCodeInfo &CodeInfo) {
986 CloneAndPruneIntoFromInst(NewFunc, OldFunc, StartingInst: &OldFunc->front().front(), VMap,
987 ModuleLevelChanges, Returns, NameSuffix, CodeInfo);
988}
989
990/// Remaps instructions in \p Blocks using the mapping in \p VMap.
991void llvm::remapInstructionsInBlocks(ArrayRef<BasicBlock *> Blocks,
992 ValueToValueMapTy &VMap) {
993 // Rewrite the code to refer to itself.
994 for (BasicBlock *BB : Blocks) {
995 for (Instruction &Inst : *BB) {
996 RemapDbgRecordRange(M: Inst.getModule(), Range: Inst.getDbgRecordRange(), VM&: VMap,
997 Flags: RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
998 RemapInstruction(I: &Inst, VM&: VMap,
999 Flags: RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
1000 }
1001 }
1002}
1003
1004/// Clones a loop \p OrigLoop. Returns the loop and the blocks in \p
1005/// Blocks.
1006///
1007/// Updates LoopInfo and DominatorTree assuming the loop is dominated by block
1008/// \p LoopDomBB. Insert the new blocks before block specified in \p Before.
1009Loop *llvm::cloneLoopWithPreheader(BasicBlock *Before, BasicBlock *LoopDomBB,
1010 Loop *OrigLoop, ValueToValueMapTy &VMap,
1011 const Twine &NameSuffix, LoopInfo *LI,
1012 DominatorTree *DT,
1013 SmallVectorImpl<BasicBlock *> &Blocks) {
1014 Function *F = OrigLoop->getHeader()->getParent();
1015 Loop *ParentLoop = OrigLoop->getParentLoop();
1016 DenseMap<Loop *, Loop *> LMap;
1017
1018 Loop *NewLoop = LI->AllocateLoop();
1019 LMap[OrigLoop] = NewLoop;
1020 if (ParentLoop)
1021 ParentLoop->addChildLoop(NewChild: NewLoop);
1022 else
1023 LI->addTopLevelLoop(New: NewLoop);
1024
1025 BasicBlock *OrigPH = OrigLoop->getLoopPreheader();
1026 assert(OrigPH && "No preheader");
1027 BasicBlock *NewPH = CloneBasicBlock(BB: OrigPH, VMap, NameSuffix, F);
1028 // To rename the loop PHIs.
1029 VMap[OrigPH] = NewPH;
1030 Blocks.push_back(Elt: NewPH);
1031
1032 // Update LoopInfo.
1033 if (ParentLoop)
1034 ParentLoop->addBasicBlockToLoop(NewBB: NewPH, LI&: *LI);
1035
1036 // Update DominatorTree.
1037 DT->addNewBlock(BB: NewPH, DomBB: LoopDomBB);
1038
1039 for (Loop *CurLoop : OrigLoop->getLoopsInPreorder()) {
1040 Loop *&NewLoop = LMap[CurLoop];
1041 if (!NewLoop) {
1042 NewLoop = LI->AllocateLoop();
1043
1044 // Establish the parent/child relationship.
1045 Loop *OrigParent = CurLoop->getParentLoop();
1046 assert(OrigParent && "Could not find the original parent loop");
1047 Loop *NewParentLoop = LMap[OrigParent];
1048 assert(NewParentLoop && "Could not find the new parent loop");
1049
1050 NewParentLoop->addChildLoop(NewChild: NewLoop);
1051 }
1052 }
1053
1054 for (BasicBlock *BB : OrigLoop->getBlocks()) {
1055 Loop *CurLoop = LI->getLoopFor(BB);
1056 Loop *&NewLoop = LMap[CurLoop];
1057 assert(NewLoop && "Expecting new loop to be allocated");
1058
1059 BasicBlock *NewBB = CloneBasicBlock(BB, VMap, NameSuffix, F);
1060 VMap[BB] = NewBB;
1061
1062 // Update LoopInfo.
1063 NewLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
1064
1065 // Add DominatorTree node. After seeing all blocks, update to correct
1066 // IDom.
1067 DT->addNewBlock(BB: NewBB, DomBB: NewPH);
1068
1069 Blocks.push_back(Elt: NewBB);
1070 }
1071
1072 for (BasicBlock *BB : OrigLoop->getBlocks()) {
1073 // Update loop headers.
1074 Loop *CurLoop = LI->getLoopFor(BB);
1075 if (BB == CurLoop->getHeader())
1076 LMap[CurLoop]->moveToHeader(BB: cast<BasicBlock>(Val&: VMap[BB]));
1077
1078 // Update DominatorTree.
1079 BasicBlock *IDomBB = DT->getNode(BB)->getIDom()->getBlock();
1080 DT->changeImmediateDominator(BB: cast<BasicBlock>(Val&: VMap[BB]),
1081 NewBB: cast<BasicBlock>(Val&: VMap[IDomBB]));
1082 }
1083
1084 // Move them physically from the end of the block list.
1085 F->splice(ToIt: Before->getIterator(), FromF: F, FromIt: NewPH->getIterator());
1086 F->splice(ToIt: Before->getIterator(), FromF: F, FromBeginIt: NewLoop->getHeader()->getIterator(),
1087 FromEndIt: F->end());
1088
1089 return NewLoop;
1090}
1091
1092/// Duplicate non-Phi instructions from the beginning of block up to
1093/// StopAt instruction into a split block between BB and its predecessor.
1094BasicBlock *llvm::DuplicateInstructionsInSplitBetween(
1095 BasicBlock *BB, BasicBlock *PredBB, Instruction *StopAt,
1096 ValueToValueMapTy &ValueMapping, DomTreeUpdater &DTU) {
1097
1098 assert(count(successors(PredBB), BB) == 1 &&
1099 "There must be a single edge between PredBB and BB!");
1100 // We are going to have to map operands from the original BB block to the new
1101 // copy of the block 'NewBB'. If there are PHI nodes in BB, evaluate them to
1102 // account for entry from PredBB.
1103 BasicBlock::iterator BI = BB->begin();
1104 for (; PHINode *PN = dyn_cast<PHINode>(Val&: BI); ++BI)
1105 ValueMapping[PN] = PN->getIncomingValueForBlock(BB: PredBB);
1106
1107 BasicBlock *NewBB = SplitEdge(From: PredBB, To: BB);
1108 NewBB->setName(PredBB->getName() + ".split");
1109 Instruction *NewTerm = NewBB->getTerminator();
1110
1111 // FIXME: SplitEdge does not yet take a DTU, so we include the split edge
1112 // in the update set here.
1113 DTU.applyUpdates(Updates: {{DominatorTree::Delete, PredBB, BB},
1114 {DominatorTree::Insert, PredBB, NewBB},
1115 {DominatorTree::Insert, NewBB, BB}});
1116
1117 // Clone the non-phi instructions of BB into NewBB, keeping track of the
1118 // mapping and using it to remap operands in the cloned instructions.
1119 // Stop once we see the terminator too. This covers the case where BB's
1120 // terminator gets replaced and StopAt == BB's terminator.
1121 for (; StopAt != &*BI && BB->getTerminator() != &*BI; ++BI) {
1122 Instruction *New = BI->clone();
1123 New->setName(BI->getName());
1124 New->insertBefore(InsertPos: NewTerm->getIterator());
1125 New->cloneDebugInfoFrom(From: &*BI);
1126 ValueMapping[&*BI] = New;
1127
1128 // Remap operands to patch up intra-block references.
1129 for (unsigned i = 0, e = New->getNumOperands(); i != e; ++i)
1130 if (Instruction *Inst = dyn_cast<Instruction>(Val: New->getOperand(i))) {
1131 auto I = ValueMapping.find(Val: Inst);
1132 if (I != ValueMapping.end())
1133 New->setOperand(i, Val: I->second);
1134 }
1135
1136 // Remap debug variable operands.
1137 remapDebugVariable(Mapping&: ValueMapping, Inst: New);
1138 }
1139
1140 return NewBB;
1141}
1142
1143void llvm::cloneNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
1144 DenseMap<MDNode *, MDNode *> &ClonedScopes,
1145 StringRef Ext, LLVMContext &Context) {
1146 MDBuilder MDB(Context);
1147
1148 for (MDNode *ScopeList : NoAliasDeclScopes) {
1149 for (const MDOperand &MDOp : ScopeList->operands()) {
1150 if (MDNode *MD = dyn_cast<MDNode>(Val: MDOp)) {
1151 AliasScopeNode SNANode(MD);
1152
1153 std::string Name;
1154 auto ScopeName = SNANode.getName();
1155 if (!ScopeName.empty())
1156 Name = (Twine(ScopeName) + ":" + Ext).str();
1157 else
1158 Name = std::string(Ext);
1159
1160 MDNode *NewScope = MDB.createAnonymousAliasScope(
1161 Domain: const_cast<MDNode *>(SNANode.getDomain()), Name);
1162 ClonedScopes.insert(KV: std::make_pair(x&: MD, y&: NewScope));
1163 }
1164 }
1165 }
1166}
1167
1168void llvm::adaptNoAliasScopes(Instruction *I,
1169 const DenseMap<MDNode *, MDNode *> &ClonedScopes,
1170 LLVMContext &Context) {
1171 auto CloneScopeList = [&](const MDNode *ScopeList) -> MDNode * {
1172 bool NeedsReplacement = false;
1173 SmallVector<Metadata *, 8> NewScopeList;
1174 for (const MDOperand &MDOp : ScopeList->operands()) {
1175 if (MDNode *MD = dyn_cast<MDNode>(Val: MDOp)) {
1176 if (auto *NewMD = ClonedScopes.lookup(Val: MD)) {
1177 NewScopeList.push_back(Elt: NewMD);
1178 NeedsReplacement = true;
1179 continue;
1180 }
1181 NewScopeList.push_back(Elt: MD);
1182 }
1183 }
1184 if (NeedsReplacement)
1185 return MDNode::get(Context, MDs: NewScopeList);
1186 return nullptr;
1187 };
1188
1189 if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(Val: I))
1190 if (MDNode *NewScopeList = CloneScopeList(Decl->getScopeList()))
1191 Decl->setScopeList(NewScopeList);
1192
1193 auto replaceWhenNeeded = [&](unsigned MD_ID) {
1194 if (const MDNode *CSNoAlias = I->getMetadata(KindID: MD_ID))
1195 if (MDNode *NewScopeList = CloneScopeList(CSNoAlias))
1196 I->setMetadata(KindID: MD_ID, Node: NewScopeList);
1197 };
1198 replaceWhenNeeded(LLVMContext::MD_noalias);
1199 replaceWhenNeeded(LLVMContext::MD_alias_scope);
1200}
1201
1202void llvm::cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
1203 ArrayRef<BasicBlock *> NewBlocks,
1204 LLVMContext &Context, StringRef Ext) {
1205 if (NoAliasDeclScopes.empty())
1206 return;
1207
1208 DenseMap<MDNode *, MDNode *> ClonedScopes;
1209 LLVM_DEBUG(dbgs() << "cloneAndAdaptNoAliasScopes: cloning "
1210 << NoAliasDeclScopes.size() << " node(s)\n");
1211
1212 cloneNoAliasScopes(NoAliasDeclScopes, ClonedScopes, Ext, Context);
1213 // Identify instructions using metadata that needs adaptation
1214 for (BasicBlock *NewBlock : NewBlocks)
1215 for (Instruction &I : *NewBlock)
1216 adaptNoAliasScopes(I: &I, ClonedScopes, Context);
1217}
1218
1219void llvm::cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
1220 Instruction *IStart, Instruction *IEnd,
1221 LLVMContext &Context, StringRef Ext) {
1222 if (NoAliasDeclScopes.empty())
1223 return;
1224
1225 DenseMap<MDNode *, MDNode *> ClonedScopes;
1226 LLVM_DEBUG(dbgs() << "cloneAndAdaptNoAliasScopes: cloning "
1227 << NoAliasDeclScopes.size() << " node(s)\n");
1228
1229 cloneNoAliasScopes(NoAliasDeclScopes, ClonedScopes, Ext, Context);
1230 // Identify instructions using metadata that needs adaptation
1231 assert(IStart->getParent() == IEnd->getParent() && "different basic block ?");
1232 auto ItStart = IStart->getIterator();
1233 auto ItEnd = IEnd->getIterator();
1234 ++ItEnd; // IEnd is included, increment ItEnd to get the end of the range
1235 for (auto &I : llvm::make_range(x: ItStart, y: ItEnd))
1236 adaptNoAliasScopes(I: &I, ClonedScopes, Context);
1237}
1238
1239void llvm::identifyNoAliasScopesToClone(
1240 ArrayRef<BasicBlock *> BBs, SmallVectorImpl<MDNode *> &NoAliasDeclScopes) {
1241 for (BasicBlock *BB : BBs)
1242 for (Instruction &I : *BB)
1243 if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(Val: &I))
1244 NoAliasDeclScopes.push_back(Elt: Decl->getScopeList());
1245}
1246
1247void llvm::identifyNoAliasScopesToClone(
1248 BasicBlock::iterator Start, BasicBlock::iterator End,
1249 SmallVectorImpl<MDNode *> &NoAliasDeclScopes) {
1250 for (Instruction &I : make_range(x: Start, y: End))
1251 if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(Val: &I))
1252 NoAliasDeclScopes.push_back(Elt: Decl->getScopeList());
1253}
1254