1//===- CloneFunction.cpp - Clone a function into another function ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the CloneFunctionInto interface, which is used as the
10// low-level function cloner. This is used by the CloneFunction and function
11// inliner to do the dirty work of copying the body of a function around.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/ADT/SmallVector.h"
16#include "llvm/ADT/SmallVectorExtras.h"
17#include "llvm/ADT/Statistic.h"
18#include "llvm/Analysis/ConstantFolding.h"
19#include "llvm/Analysis/DomTreeUpdater.h"
20#include "llvm/Analysis/InstructionSimplify.h"
21#include "llvm/Analysis/LoopInfo.h"
22#include "llvm/IR/AttributeMask.h"
23#include "llvm/IR/CFG.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DebugInfo.h"
26#include "llvm/IR/DerivedTypes.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/InstIterator.h"
29#include "llvm/IR/Instructions.h"
30#include "llvm/IR/IntrinsicInst.h"
31#include "llvm/IR/LLVMContext.h"
32#include "llvm/IR/MDBuilder.h"
33#include "llvm/IR/Metadata.h"
34#include "llvm/IR/Module.h"
35#include "llvm/Transforms/Utils/BasicBlockUtils.h"
36#include "llvm/Transforms/Utils/Cloning.h"
37#include "llvm/Transforms/Utils/Local.h"
38#include "llvm/Transforms/Utils/ValueMapper.h"
39#include <cstdint>
40#include <optional>
41using namespace llvm;
42
43#define DEBUG_TYPE "clone-function"
44
45STATISTIC(RemappedAtomMax, "Highest global NextAtomGroup (after mapping)");
46
47void llvm::mapAtomInstance(const DebugLoc &DL, ValueToValueMapTy &VMap) {
48 uint64_t CurGroup = DL->getAtomGroup();
49 if (!CurGroup)
50 return;
51
52 // Try inserting a new entry. If there's already a mapping for this atom
53 // then there's nothing to do.
54 auto [It, Inserted] = VMap.AtomMap.insert(KV: {{DL.getInlinedAt(), CurGroup}, 0});
55 if (!Inserted)
56 return;
57
58 // Map entry to a new atom group.
59 uint64_t NewGroup = DL->getContext().incNextDILocationAtomGroup();
60 assert(NewGroup > CurGroup && "Next should always be greater than current");
61 It->second = NewGroup;
62
63 RemappedAtomMax = std::max<uint64_t>(a: NewGroup, b: RemappedAtomMax);
64}
65
66static void collectDebugInfoFromInstructions(const Function &F,
67 DebugInfoFinder &DIFinder) {
68 const Module *M = F.getParent();
69 if (!M)
70 return;
71 // Inspect instructions to process e.g. DILexicalBlocks of inlined functions
72 for (const Instruction &I : instructions(F))
73 DIFinder.processInstruction(M: *M, I);
74}
75
76// Create a predicate that matches the metadata that should be identity mapped
77// during function cloning.
78static MetadataPredicate
79createIdentityMDPredicate(const Function &F, CloneFunctionChangeType Changes) {
80 if (Changes >= CloneFunctionChangeType::DifferentModule)
81 return [](const Metadata *MD) { return false; };
82
83 DISubprogram *SPClonedWithinModule = F.getSubprogram();
84
85 // Don't clone inlined subprograms.
86 auto ShouldKeep = [SPClonedWithinModule](const DISubprogram *SP) -> bool {
87 return SP != SPClonedWithinModule;
88 };
89
90 return [=](const Metadata *MD) {
91 // Avoid cloning compile units.
92 if (isa<DICompileUnit>(Val: MD))
93 return true;
94
95 if (auto *SP = dyn_cast<DISubprogram>(Val: MD))
96 return ShouldKeep(SP);
97
98 // If a subprogram isn't going to be cloned skip its lexical blocks as well.
99 if (auto *LScope = dyn_cast<DILocalScope>(Val: MD))
100 return ShouldKeep(LScope->getSubprogram());
101
102 // Avoid cloning local variables of subprograms that won't be cloned.
103 if (auto *DV = dyn_cast<DILocalVariable>(Val: MD))
104 if (auto *S = dyn_cast_or_null<DILocalScope>(Val: DV->getScope()))
105 return ShouldKeep(S->getSubprogram());
106
107 // DIGlobalVariableExpression representing static local variable may be
108 // encountered in DISubprogram's retainedNodes list. Do not remap it, and
109 // remove it from retainedNodes after mapping.
110 if (isa<DIGlobalVariableExpression>(Val: MD))
111 return true;
112
113 // Clone types that are local to subprograms being cloned.
114 // Avoid cloning other types.
115 auto *Type = dyn_cast<DIType>(Val: MD);
116 if (!Type)
117 return false;
118
119 // No need to clone types if subprograms are not cloned.
120 if (SPClonedWithinModule == nullptr)
121 return true;
122
123 // Scopeless types may be derived from local types (e.g. pointers to local
124 // types). They may need cloning.
125 if (const DIDerivedType *DTy = dyn_cast_or_null<DIDerivedType>(Val: Type);
126 DTy && !DTy->getScope())
127 return false;
128
129 auto *LScope = dyn_cast_or_null<DILocalScope>(Val: Type->getScope());
130 if (!LScope)
131 return true;
132
133 if (ShouldKeep(LScope->getSubprogram()))
134 return true;
135
136 return false;
137 };
138}
139
140/// See comments in Cloning.h.
141BasicBlock *llvm::CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap,
142 const Twine &NameSuffix, Function *F,
143 ClonedCodeInfo *CodeInfo, bool MapAtoms) {
144 BasicBlock *NewBB = BasicBlock::Create(Context&: BB->getContext(), Name: "", Parent: F);
145 if (BB->hasName())
146 NewBB->setName(BB->getName() + NameSuffix);
147
148 bool hasCalls = false, hasDynamicAllocas = false, hasMemProfMetadata = false;
149
150 // Loop over all instructions, and copy them over.
151 for (const Instruction &I : *BB) {
152 Instruction *NewInst = I.clone();
153 if (I.hasName())
154 NewInst->setName(I.getName() + NameSuffix);
155
156 NewInst->insertBefore(BB&: *NewBB, InsertPos: NewBB->end());
157 NewInst->cloneDebugInfoFrom(From: &I);
158
159 VMap[&I] = NewInst; // Add instruction map to value.
160
161 if (MapAtoms) {
162 if (const DebugLoc &DL = NewInst->getDebugLoc())
163 mapAtomInstance(DL: DL.get(), VMap);
164 }
165
166 if (isa<CallInst>(Val: I)) {
167 hasCalls = true;
168 hasMemProfMetadata |= I.hasMetadata(KindID: LLVMContext::MD_memprof);
169 hasMemProfMetadata |= I.hasMetadata(KindID: LLVMContext::MD_callsite);
170 }
171 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Val: &I)) {
172 if (!AI->isStaticAlloca()) {
173 hasDynamicAllocas = true;
174 }
175 }
176 }
177
178 if (CodeInfo) {
179 CodeInfo->ContainsCalls |= hasCalls;
180 CodeInfo->ContainsMemProfMetadata |= hasMemProfMetadata;
181 CodeInfo->ContainsDynamicAllocas |= hasDynamicAllocas;
182 }
183 return NewBB;
184}
185
186void llvm::CloneFunctionAttributesInto(Function *NewFunc,
187 const Function *OldFunc,
188 ValueToValueMapTy &VMap,
189 bool ModuleLevelChanges,
190 ValueMapTypeRemapper *TypeMapper,
191 ValueMaterializer *Materializer) {
192 // Copy all attributes other than those stored in Function's AttributeList
193 // which holds e.g. parameters and return value attributes.
194 AttributeList NewAttrs = NewFunc->getAttributes();
195 NewFunc->copyAttributesFrom(Src: OldFunc);
196 NewFunc->setAttributes(NewAttrs);
197
198 const RemapFlags FuncGlobalRefFlags =
199 ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges;
200
201 // Fix up the personality function that got copied over.
202 if (OldFunc->hasPersonalityFn())
203 NewFunc->setPersonalityFn(MapValue(V: OldFunc->getPersonalityFn(), VM&: VMap,
204 Flags: FuncGlobalRefFlags, TypeMapper,
205 Materializer));
206
207 if (OldFunc->hasPrefixData()) {
208 NewFunc->setPrefixData(MapValue(V: OldFunc->getPrefixData(), VM&: VMap,
209 Flags: FuncGlobalRefFlags, TypeMapper,
210 Materializer));
211 }
212
213 if (OldFunc->hasPrologueData()) {
214 NewFunc->setPrologueData(MapValue(V: OldFunc->getPrologueData(), VM&: VMap,
215 Flags: FuncGlobalRefFlags, TypeMapper,
216 Materializer));
217 }
218
219 SmallVector<AttributeSet, 4> NewArgAttrs(NewFunc->arg_size());
220 AttributeList OldAttrs = OldFunc->getAttributes();
221
222 // Clone any argument attributes that are present in the VMap.
223 for (const Argument &OldArg : OldFunc->args()) {
224 if (Argument *NewArg = dyn_cast<Argument>(Val&: VMap[&OldArg])) {
225 // Remap the parameter indices.
226 NewArgAttrs[NewArg->getArgNo()] =
227 OldAttrs.getParamAttrs(ArgNo: OldArg.getArgNo());
228 }
229 }
230
231 NewFunc->setAttributes(
232 AttributeList::get(C&: NewFunc->getContext(), FnAttrs: OldAttrs.getFnAttrs(),
233 RetAttrs: OldAttrs.getRetAttrs(), ArgAttrs: NewArgAttrs));
234}
235
236void llvm::CloneFunctionMetadataInto(Function &NewFunc, const Function &OldFunc,
237 ValueToValueMapTy &VMap,
238 RemapFlags RemapFlag,
239 ValueMapTypeRemapper *TypeMapper,
240 ValueMaterializer *Materializer,
241 const MetadataPredicate *IdentityMD) {
242 SmallVector<std::pair<unsigned, MDNode *>, 1> MDs;
243 OldFunc.getAllMetadata(MDs);
244 for (const auto &[Kind, MD] : MDs) {
245 NewFunc.addMetadata(KindID: Kind, MD&: *MapMetadata(MD, VM&: VMap, Flags: RemapFlag, TypeMapper,
246 Materializer, IdentityMD));
247 }
248}
249
250void llvm::CloneFunctionBodyInto(Function &NewFunc, const Function &OldFunc,
251 ValueToValueMapTy &VMap, RemapFlags RemapFlag,
252 SmallVectorImpl<ReturnInst *> &Returns,
253 const char *NameSuffix,
254 ClonedCodeInfo *CodeInfo,
255 ValueMapTypeRemapper *TypeMapper,
256 ValueMaterializer *Materializer,
257 const MetadataPredicate *IdentityMD) {
258 if (OldFunc.isDeclaration())
259 return;
260
261 // Loop over all of the basic blocks in the function, cloning them as
262 // appropriate. Note that we save BE this way in order to handle cloning of
263 // recursive functions into themselves.
264 for (const BasicBlock &BB : OldFunc) {
265 // Create a new basic block and copy instructions into it!
266 BasicBlock *CBB =
267 CloneBasicBlock(BB: &BB, VMap, NameSuffix, F: &NewFunc, CodeInfo);
268
269 // Add basic block mapping.
270 VMap[&BB] = CBB;
271
272 // It is only legal to clone a function if a block address within that
273 // function is never referenced outside of the function. Given that, we
274 // want to map block addresses from the old function to block addresses in
275 // the clone. (This is different from the generic ValueMapper
276 // implementation, which generates an invalid blockaddress when
277 // cloning a function.)
278 if (BB.hasAddressTaken()) {
279 Constant *OldBBAddr = BlockAddress::get(F: const_cast<Function *>(&OldFunc),
280 BB: const_cast<BasicBlock *>(&BB));
281 VMap[OldBBAddr] = BlockAddress::get(F: &NewFunc, BB: CBB);
282 }
283
284 // Note return instructions for the caller.
285 if (ReturnInst *RI = dyn_cast<ReturnInst>(Val: CBB->getTerminator()))
286 Returns.push_back(Elt: RI);
287 }
288
289 // Loop over all of the instructions in the new function, fixing up operand
290 // references as we go. This uses VMap to do all the hard work.
291 ValueMapper Mapper(VMap, RemapFlag, TypeMapper, Materializer, IdentityMD);
292 for (Function::iterator
293 BB = cast<BasicBlock>(Val&: VMap[&OldFunc.front()])->getIterator(),
294 BE = NewFunc.end();
295 BB != BE; ++BB)
296 // Loop over all instructions, fixing each one as we find it, and any
297 // attached debug-info records.
298 for (Instruction &II : *BB) {
299 Mapper.remapInstruction(I&: II);
300 Mapper.remapDbgRecordRange(M: II.getModule(), Range: II.getDbgRecordRange());
301 }
302}
303
304// Clone OldFunc into NewFunc, transforming the old arguments into references to
305// VMap values.
306void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
307 ValueToValueMapTy &VMap,
308 CloneFunctionChangeType Changes,
309 SmallVectorImpl<ReturnInst *> &Returns,
310 const char *NameSuffix, ClonedCodeInfo *CodeInfo,
311 ValueMapTypeRemapper *TypeMapper,
312 ValueMaterializer *Materializer) {
313 assert(NameSuffix && "NameSuffix cannot be null!");
314
315#ifndef NDEBUG
316 for (const Argument &I : OldFunc->args())
317 assert(VMap.count(&I) && "No mapping from source argument specified!");
318#endif
319
320 bool ModuleLevelChanges = Changes > CloneFunctionChangeType::LocalChangesOnly;
321
322 CloneFunctionAttributesInto(NewFunc, OldFunc, VMap, ModuleLevelChanges,
323 TypeMapper, Materializer);
324
325 // Everything else beyond this point deals with function instructions,
326 // so if we are dealing with a function declaration, we're done.
327 if (OldFunc->isDeclaration())
328 return;
329
330 if (Changes < CloneFunctionChangeType::DifferentModule) {
331 assert((NewFunc->getParent() == nullptr ||
332 NewFunc->getParent() == OldFunc->getParent()) &&
333 "Expected NewFunc to have the same parent, or no parent");
334 } else {
335 assert((NewFunc->getParent() == nullptr ||
336 NewFunc->getParent() != OldFunc->getParent()) &&
337 "Expected NewFunc to have different parents, or no parent");
338
339 if (Changes == CloneFunctionChangeType::DifferentModule) {
340 assert(NewFunc->getParent() &&
341 "Need parent of new function to maintain debug info invariants");
342 }
343 }
344
345 MetadataPredicate IdentityMD = createIdentityMDPredicate(F: *OldFunc, Changes);
346
347 // Cloning is always a Module level operation, since Metadata needs to be
348 // cloned.
349 const RemapFlags RemapFlag = RF_None;
350
351 CloneFunctionMetadataInto(NewFunc&: *NewFunc, OldFunc: *OldFunc, VMap, RemapFlag, TypeMapper,
352 Materializer, IdentityMD: &IdentityMD);
353
354 CloneFunctionBodyInto(NewFunc&: *NewFunc, OldFunc: *OldFunc, VMap, RemapFlag, Returns,
355 NameSuffix, CodeInfo, TypeMapper, Materializer,
356 IdentityMD: &IdentityMD);
357
358 // DIGlobalVariableExpressions representing static locals stay in the scope of
359 // OldSP after function cloning. Remove them from retainedNodes of NewSP.
360 if (DISubprogram *NewSP = NewFunc->getSubprogram())
361 NewSP->cleanupRetainedNodesIf(Pred: [NewSP](Metadata *N) {
362 auto *GVE = dyn_cast_or_null<DIGlobalVariableExpression>(Val: N);
363 return !GVE ||
364 DISubprogram::getRetainedNodeScope(N: GVE)->getSubprogram() == NewSP;
365 });
366
367 // Only update !llvm.dbg.cu for DifferentModule (not CloneModule). In the
368 // same module, the compile unit will already be listed (or not). When
369 // cloning a module, CloneModule() will handle creating the named metadata.
370 if (Changes != CloneFunctionChangeType::DifferentModule)
371 return;
372
373 // Update !llvm.dbg.cu with compile units added to the new module if this
374 // function is being cloned in isolation.
375 //
376 // FIXME: This is making global / module-level changes, which doesn't seem
377 // like the right encapsulation Consider dropping the requirement to update
378 // !llvm.dbg.cu (either obsoleting the node, or restricting it to
379 // non-discardable compile units) instead of discovering compile units by
380 // visiting the metadata attached to global values, which would allow this
381 // code to be deleted. Alternatively, perhaps give responsibility for this
382 // update to CloneFunctionInto's callers.
383 Module *NewModule = NewFunc->getParent();
384 NamedMDNode *NMD = NewModule->getOrInsertNamedMetadata(Name: "llvm.dbg.cu");
385 // Avoid multiple insertions of the same DICompileUnit to NMD.
386 SmallPtrSet<const void *, 8> Visited(llvm::from_range, NMD->operands());
387
388 // Collect and clone all the compile units referenced from the instructions in
389 // the function (e.g. as instructions' scope).
390 DebugInfoFinder DIFinder;
391 collectDebugInfoFromInstructions(F: *OldFunc, DIFinder);
392 for (DICompileUnit *Unit : DIFinder.compile_units()) {
393 MDNode *MappedUnit =
394 MapMetadata(MD: Unit, VM&: VMap, Flags: RF_None, TypeMapper, Materializer);
395 if (Visited.insert(Ptr: MappedUnit).second)
396 NMD->addOperand(M: MappedUnit);
397 }
398}
399
400/// Return a copy of the specified function and add it to that function's
401/// module. Also, any references specified in the VMap are changed to refer to
402/// their mapped value instead of the original one. If any of the arguments to
403/// the function are in the VMap, the arguments are deleted from the resultant
404/// function. The VMap is updated to include mappings from all of the
405/// instructions and basicblocks in the function from their old to new values.
406///
407Function *llvm::CloneFunction(Function *F, ValueToValueMapTy &VMap,
408 ClonedCodeInfo *CodeInfo) {
409 std::vector<Type *> ArgTypes;
410
411 // The user might be deleting arguments to the function by specifying them in
412 // the VMap. If so, we need to not add the arguments to the arg ty vector
413 //
414 for (const Argument &I : F->args())
415 if (VMap.count(Val: &I) == 0) // Haven't mapped the argument to anything yet?
416 ArgTypes.push_back(x: I.getType());
417
418 // Create a new function type...
419 FunctionType *FTy =
420 FunctionType::get(Result: F->getFunctionType()->getReturnType(), Params: ArgTypes,
421 isVarArg: F->getFunctionType()->isVarArg());
422
423 // Create the new function...
424 Function *NewF = Function::Create(Ty: FTy, Linkage: F->getLinkage(), AddrSpace: F->getAddressSpace(),
425 N: F->getName(), M: F->getParent());
426
427 // Loop over the arguments, copying the names of the mapped arguments over...
428 Function::arg_iterator DestI = NewF->arg_begin();
429 for (const Argument &I : F->args())
430 if (VMap.count(Val: &I) == 0) { // Is this argument preserved?
431 DestI->setName(I.getName()); // Copy the name over...
432 VMap[&I] = &*DestI++; // Add mapping to VMap
433 }
434
435 SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned.
436 CloneFunctionInto(NewFunc: NewF, OldFunc: F, VMap, Changes: CloneFunctionChangeType::LocalChangesOnly,
437 Returns, NameSuffix: "", CodeInfo);
438
439 return NewF;
440}
441
442namespace {
443/// This is a private class used to implement CloneAndPruneFunctionInto.
444struct PruningFunctionCloner {
445 Function *NewFunc;
446 const Function *OldFunc;
447 ValueToValueMapTy &VMap;
448 ValueMapper &Remapper;
449 bool ModuleLevelChanges;
450 const char *NameSuffix;
451 ClonedCodeInfo &CodeInfo;
452 bool HostFuncIsStrictFP;
453
454 Instruction *cloneInstruction(BasicBlock::const_iterator II);
455
456public:
457 PruningFunctionCloner(Function *newFunc, const Function *oldFunc,
458 ValueToValueMapTy &valueMap, ValueMapper &remapper,
459 bool moduleLevelChanges, const char *nameSuffix,
460 ClonedCodeInfo &codeInfo)
461 : NewFunc(newFunc), OldFunc(oldFunc), VMap(valueMap), Remapper(remapper),
462 ModuleLevelChanges(moduleLevelChanges), NameSuffix(nameSuffix),
463 CodeInfo(codeInfo) {
464 HostFuncIsStrictFP =
465 newFunc->getAttributes().hasFnAttr(Kind: Attribute::StrictFP);
466 }
467
468 /// The specified block is found to be reachable, clone it and
469 /// anything that it can reach.
470 void CloneBlock(const BasicBlock *BB, BasicBlock::const_iterator StartingInst,
471 std::vector<const BasicBlock *> &ToClone);
472};
473} // namespace
474
475Instruction *
476PruningFunctionCloner::cloneInstruction(BasicBlock::const_iterator II) {
477 if (!HostFuncIsStrictFP)
478 return II->clone();
479
480 const Instruction &OldInst = *II;
481 Intrinsic::ID CIID = getConstrainedIntrinsicID(Instr: OldInst);
482 if (CIID == Intrinsic::not_intrinsic)
483 return II->clone();
484
485 // Instead of cloning the instruction, a call to constrained intrinsic should
486 // be created. Assume the first arguments of constrained intrinsics are the
487 // same as the operands of original instruction.
488
489 // Create intrinsic call.
490 LLVMContext &Ctx = NewFunc->getContext();
491 SmallVector<Value *, 4> Args;
492 unsigned NumOperands = OldInst.getNumOperands();
493 if (isa<CallInst>(Val: OldInst))
494 --NumOperands;
495 for (unsigned I = 0; I < NumOperands; ++I)
496 Args.push_back(Elt: OldInst.getOperand(i: I));
497
498 if (const auto *CmpI = dyn_cast<FCmpInst>(Val: &OldInst)) {
499 FCmpInst::Predicate Pred = CmpI->getPredicate();
500 StringRef PredName = FCmpInst::getPredicateName(P: Pred);
501 Args.push_back(Elt: MetadataAsValue::get(Context&: Ctx, MD: MDString::get(Context&: Ctx, Str: PredName)));
502 }
503
504 // The last arguments of a constrained intrinsic are metadata that represent
505 // rounding mode (absent in some intrinsics) and exception behavior. The
506 // inlined function uses default settings.
507 if (Intrinsic::hasConstrainedFPRoundingModeOperand(QID: CIID))
508 Args.push_back(
509 Elt: MetadataAsValue::get(Context&: Ctx, MD: MDString::get(Context&: Ctx, Str: "round.tonearest")));
510 Args.push_back(
511 Elt: MetadataAsValue::get(Context&: Ctx, MD: MDString::get(Context&: Ctx, Str: "fpexcept.ignore")));
512
513 SmallVector<Type *> ArgTys = llvm::map_to_vector(C&: Args, F: &Value::getType);
514 Function *IFn = Intrinsic::getOrInsertDeclaration(M: NewFunc->getParent(), IID: CIID,
515 RetTy: OldInst.getType(), ArgTys);
516 return CallInst::Create(Func: IFn, Args, NameStr: OldInst.getName() + ".strict");
517}
518
519/// The specified block is found to be reachable, clone it and
520/// anything that it can reach.
521void PruningFunctionCloner::CloneBlock(
522 const BasicBlock *BB, BasicBlock::const_iterator StartingInst,
523 std::vector<const BasicBlock *> &ToClone) {
524 WeakTrackingVH &BBEntry = VMap[BB];
525
526 // Have we already cloned this block?
527 if (BBEntry)
528 return;
529
530 // Nope, clone it now.
531 BasicBlock *NewBB;
532 Twine NewName(BB->hasName() ? Twine(BB->getName()) + NameSuffix : "");
533 BBEntry = NewBB = BasicBlock::Create(Context&: BB->getContext(), Name: NewName, Parent: NewFunc);
534
535 // It is only legal to clone a function if a block address within that
536 // function is never referenced outside of the function. Given that, we
537 // want to map block addresses from the old function to block addresses in
538 // the clone. (This is different from the generic ValueMapper
539 // implementation, which generates an invalid blockaddress when
540 // cloning a function.)
541 //
542 // Note that we don't need to fix the mapping for unreachable blocks;
543 // the default mapping there is safe.
544 if (BB->hasAddressTaken()) {
545 Constant *OldBBAddr = BlockAddress::get(F: const_cast<Function *>(OldFunc),
546 BB: const_cast<BasicBlock *>(BB));
547 VMap[OldBBAddr] = BlockAddress::get(F: NewFunc, BB: NewBB);
548 }
549
550 bool hasCalls = false, hasDynamicAllocas = false, hasStaticAllocas = false;
551 bool hasMemProfMetadata = false;
552
553 // Keep a cursor pointing at the last place we cloned debug-info records from.
554 BasicBlock::const_iterator DbgCursor = StartingInst;
555 auto CloneDbgRecordsToHere =
556 [&DbgCursor](Instruction *NewInst, BasicBlock::const_iterator II) {
557 // Clone debug-info records onto this instruction. Iterate through any
558 // source-instructions we've cloned and then subsequently optimised
559 // away, so that their debug-info doesn't go missing.
560 for (; DbgCursor != II; ++DbgCursor)
561 NewInst->cloneDebugInfoFrom(From: &*DbgCursor, FromHere: std::nullopt, InsertAtHead: false);
562 NewInst->cloneDebugInfoFrom(From: &*II);
563 DbgCursor = std::next(x: II);
564 };
565
566 // Loop over all instructions, and copy them over, DCE'ing as we go. This
567 // loop doesn't include the terminator.
568 for (BasicBlock::const_iterator II = StartingInst, IE = --BB->end(); II != IE;
569 ++II) {
570
571 // Don't clone fake_use as it may suppress many optimizations
572 // due to inlining, especially SROA.
573 if (auto *IntrInst = dyn_cast<IntrinsicInst>(Val&: II))
574 if (IntrInst->getIntrinsicID() == Intrinsic::fake_use)
575 continue;
576
577 Instruction *NewInst = cloneInstruction(II);
578 NewInst->insertInto(ParentBB: NewBB, It: NewBB->end());
579
580 if (HostFuncIsStrictFP) {
581 // All function calls in the inlined function must get 'strictfp'
582 // attribute to prevent undesirable optimizations.
583 if (auto *Call = dyn_cast<CallInst>(Val: NewInst))
584 Call->addFnAttr(Kind: Attribute::StrictFP);
585 }
586
587 // Eagerly remap operands to the newly cloned instruction, except for PHI
588 // nodes for which we defer processing until we update the CFG.
589 if (!isa<PHINode>(Val: NewInst)) {
590 Remapper.remapInstruction(I&: *NewInst);
591
592 // Eagerly constant fold the newly cloned instruction. If successful, add
593 // a mapping to the new value. Non-constant operands may be incomplete at
594 // this stage, thus instruction simplification is performed after
595 // processing phi-nodes.
596 if (Value *V = ConstantFoldInstruction(
597 I: NewInst, DL: BB->getDataLayout())) {
598 if (isInstructionTriviallyDead(I: NewInst)) {
599 VMap[&*II] = V;
600 NewInst->eraseFromParent();
601 continue;
602 }
603 }
604 }
605
606 if (auto *CB = dyn_cast<CallBase>(Val&: II); CB && CB->isIndirectCall())
607 CodeInfo.OriginallyIndirectCalls.insert(X: NewInst);
608
609 if (II->hasName())
610 NewInst->setName(II->getName() + NameSuffix);
611 VMap[&*II] = NewInst; // Add instruction map to value.
612 if (isa<CallInst>(Val: II)) {
613 hasCalls = true;
614 hasMemProfMetadata |= II->hasMetadata(KindID: LLVMContext::MD_memprof);
615 hasMemProfMetadata |= II->hasMetadata(KindID: LLVMContext::MD_callsite);
616 }
617
618 CloneDbgRecordsToHere(NewInst, II);
619
620 CodeInfo.OrigVMap[&*II] = NewInst;
621 if (auto *CB = dyn_cast<CallBase>(Val: &*II))
622 if (CB->hasOperandBundles())
623 CodeInfo.OperandBundleCallSites.push_back(x: NewInst);
624
625 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Val&: II)) {
626 if (isa<ConstantInt>(Val: AI->getArraySize()))
627 hasStaticAllocas = true;
628 else
629 hasDynamicAllocas = true;
630 }
631 }
632
633 // Finally, clone over the terminator.
634 const Instruction *OldTI = BB->getTerminator();
635 bool TerminatorDone = false;
636 if (const CondBrInst *BI = dyn_cast<CondBrInst>(Val: OldTI)) {
637 // If the condition was a known constant in the callee...
638 ConstantInt *Cond = dyn_cast<ConstantInt>(Val: BI->getCondition());
639 // Or is a known constant in the caller...
640 if (!Cond) {
641 Value *V = VMap.lookup(Val: BI->getCondition());
642 Cond = dyn_cast_or_null<ConstantInt>(Val: V);
643 }
644
645 // Constant fold to uncond branch!
646 if (Cond) {
647 BasicBlock *Dest = BI->getSuccessor(i: !Cond->getZExtValue());
648 auto *NewBI = UncondBrInst::Create(Target: Dest, InsertBefore: NewBB);
649 NewBI->setDebugLoc(BI->getDebugLoc());
650 VMap[OldTI] = NewBI;
651 ToClone.push_back(x: Dest);
652 TerminatorDone = true;
653 }
654 } else if (const SwitchInst *SI = dyn_cast<SwitchInst>(Val: OldTI)) {
655 // If switching on a value known constant in the caller.
656 ConstantInt *Cond = dyn_cast<ConstantInt>(Val: SI->getCondition());
657 if (!Cond) { // Or known constant after constant prop in the callee...
658 Value *V = VMap.lookup(Val: SI->getCondition());
659 Cond = dyn_cast_or_null<ConstantInt>(Val: V);
660 }
661 if (Cond) { // Constant fold to uncond branch!
662 SwitchInst::ConstCaseHandle Case = *SI->findCaseValue(C: Cond);
663 BasicBlock *Dest = const_cast<BasicBlock *>(Case.getCaseSuccessor());
664 auto *NewBI = UncondBrInst::Create(Target: Dest, InsertBefore: NewBB);
665 NewBI->setDebugLoc(SI->getDebugLoc());
666 VMap[OldTI] = NewBI;
667 ToClone.push_back(x: Dest);
668 TerminatorDone = true;
669 }
670 }
671
672 if (!TerminatorDone) {
673 Instruction *NewInst = OldTI->clone();
674 if (OldTI->hasName())
675 NewInst->setName(OldTI->getName() + NameSuffix);
676 NewInst->insertInto(ParentBB: NewBB, It: NewBB->end());
677
678 CloneDbgRecordsToHere(NewInst, OldTI->getIterator());
679
680 VMap[OldTI] = NewInst; // Add instruction map to value.
681
682 CodeInfo.OrigVMap[OldTI] = NewInst;
683 if (auto *CB = dyn_cast<CallBase>(Val: OldTI))
684 if (CB->hasOperandBundles())
685 CodeInfo.OperandBundleCallSites.push_back(x: NewInst);
686
687 // Recursively clone any reachable successor blocks.
688 append_range(C&: ToClone, R: successors(I: BB->getTerminator()));
689 } else {
690 // If we didn't create a new terminator, clone DbgVariableRecords from the
691 // old terminator onto the new terminator.
692 Instruction *NewInst = NewBB->getTerminator();
693
694 CloneDbgRecordsToHere(NewInst, OldTI->getIterator());
695 }
696
697 CodeInfo.ContainsCalls |= hasCalls;
698 CodeInfo.ContainsMemProfMetadata |= hasMemProfMetadata;
699 CodeInfo.ContainsDynamicAllocas |= hasDynamicAllocas;
700 CodeInfo.ContainsDynamicAllocas |=
701 hasStaticAllocas && BB != &BB->getParent()->front();
702}
703
704/// This works like CloneAndPruneFunctionInto, except that it does not clone the
705/// entire function. Instead it starts at an instruction provided by the caller
706/// and copies (and prunes) only the code reachable from that instruction.
707void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
708 const Instruction *StartingInst,
709 ValueToValueMapTy &VMap,
710 bool ModuleLevelChanges,
711 SmallVectorImpl<ReturnInst *> &Returns,
712 const char *NameSuffix,
713 ClonedCodeInfo &CodeInfo) {
714 assert(NameSuffix && "NameSuffix cannot be null!");
715
716 ValueMapTypeRemapper *TypeMapper = nullptr;
717 ValueMaterializer *Materializer = nullptr;
718
719#ifndef NDEBUG
720 // If the cloning starts at the beginning of the function, verify that
721 // the function arguments are mapped.
722 if (!StartingInst)
723 for (const Argument &II : OldFunc->args())
724 assert(VMap.count(&II) && "No mapping from source argument specified!");
725#endif
726
727 ValueMapper Mapper(VMap,
728 ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges,
729 TypeMapper, Materializer);
730 PruningFunctionCloner PFC(NewFunc, OldFunc, VMap, Mapper, ModuleLevelChanges,
731 NameSuffix, CodeInfo);
732 const BasicBlock *StartingBB;
733 if (StartingInst)
734 StartingBB = StartingInst->getParent();
735 else {
736 StartingBB = &OldFunc->getEntryBlock();
737 StartingInst = &StartingBB->front();
738 }
739
740 // Clone the entry block, and anything recursively reachable from it.
741 std::vector<const BasicBlock *> CloneWorklist;
742 PFC.CloneBlock(BB: StartingBB, StartingInst: StartingInst->getIterator(), ToClone&: CloneWorklist);
743 while (!CloneWorklist.empty()) {
744 const BasicBlock *BB = CloneWorklist.back();
745 CloneWorklist.pop_back();
746 PFC.CloneBlock(BB, StartingInst: BB->begin(), ToClone&: CloneWorklist);
747 }
748
749 // Loop over all of the basic blocks in the old function. If the block was
750 // reachable, we have cloned it and the old block is now in the value map:
751 // insert it into the new function in the right order. If not, ignore it.
752 //
753 // Defer PHI resolution until rest of function is resolved.
754 SmallVector<const PHINode *, 16> PHIToResolve;
755 for (const BasicBlock &BI : *OldFunc) {
756 Value *V = VMap.lookup(Val: &BI);
757 BasicBlock *NewBB = cast_or_null<BasicBlock>(Val: V);
758 if (!NewBB)
759 continue; // Dead block.
760
761 // Move the new block to preserve the order in the original function.
762 NewBB->moveBefore(MovePos: NewFunc->end());
763
764 // Handle PHI nodes specially, as we have to remove references to dead
765 // blocks.
766 for (const PHINode &PN : BI.phis()) {
767 // PHI nodes may have been remapped to non-PHI nodes by the caller or
768 // during the cloning process.
769 if (isa<PHINode>(Val: VMap[&PN]))
770 PHIToResolve.push_back(Elt: &PN);
771 else
772 break;
773 }
774
775 // Finally, remap the terminator instructions, as those can't be remapped
776 // until all BBs are mapped.
777 Mapper.remapInstruction(I&: *NewBB->getTerminator());
778 }
779
780 // Defer PHI resolution until rest of function is resolved, PHI resolution
781 // requires the CFG to be up-to-date.
782 for (unsigned phino = 0, e = PHIToResolve.size(); phino != e;) {
783 const PHINode *OPN = PHIToResolve[phino];
784 unsigned NumPreds = OPN->getNumIncomingValues();
785 const BasicBlock *OldBB = OPN->getParent();
786 BasicBlock *NewBB = cast<BasicBlock>(Val&: VMap[OldBB]);
787
788 // Map operands for blocks that are live and remove operands for blocks
789 // that are dead.
790 for (; phino != PHIToResolve.size() &&
791 PHIToResolve[phino]->getParent() == OldBB;
792 ++phino) {
793 OPN = PHIToResolve[phino];
794 PHINode *PN = cast<PHINode>(Val&: VMap[OPN]);
795 for (int64_t pred = NumPreds - 1; pred >= 0; --pred) {
796 Value *V = VMap.lookup(Val: PN->getIncomingBlock(i: pred));
797 if (BasicBlock *MappedBlock = cast_or_null<BasicBlock>(Val: V)) {
798 Value *InVal = Mapper.mapValue(V: *PN->getIncomingValue(i: pred));
799 assert(InVal && "Unknown input value?");
800 PN->setIncomingValue(i: pred, V: InVal);
801 PN->setIncomingBlock(i: pred, BB: MappedBlock);
802 continue;
803 }
804 PN->removeIncomingValue(Idx: pred, DeletePHIIfEmpty: false);
805 }
806 }
807
808 // The loop above has removed PHI entries for those blocks that are dead
809 // and has updated others. However, if a block is live (i.e. copied over)
810 // but its terminator has been changed to not go to this block, then our
811 // phi nodes will have invalid entries. Update the PHI nodes in this
812 // case.
813 PHINode *PN = cast<PHINode>(Val: NewBB->begin());
814 NumPreds = pred_size(BB: NewBB);
815 if (NumPreds != PN->getNumIncomingValues()) {
816 assert(NumPreds < PN->getNumIncomingValues());
817 // Count how many times each predecessor comes to this block.
818 DenseMap<BasicBlock *, unsigned> PredCount;
819 for (BasicBlock *Pred : predecessors(BB: NewBB))
820 ++PredCount[Pred];
821
822 BasicBlock::iterator I = NewBB->begin();
823 DenseMap<BasicBlock *, unsigned> SeenPredCount;
824 SeenPredCount.reserve(NumEntries: PredCount.size());
825 for (; (PN = dyn_cast<PHINode>(Val&: I)); ++I) {
826 SeenPredCount.clear();
827 PN->removeIncomingValueIf(
828 Predicate: [&](unsigned Idx) {
829 BasicBlock *IncomingBlock = PN->getIncomingBlock(i: Idx);
830 auto It = PredCount.find(Val: IncomingBlock);
831 if (It == PredCount.end())
832 return true;
833 unsigned &SeenCount = SeenPredCount[IncomingBlock];
834 if (SeenCount < It->second) {
835 SeenCount++;
836 return false;
837 }
838 return true;
839 },
840 DeletePHIIfEmpty: false);
841 }
842 }
843
844 // If the loops above have made these phi nodes have 0 or 1 operand,
845 // replace them with poison or the input value. We must do this for
846 // correctness, because 0-operand phis are not valid.
847 PN = cast<PHINode>(Val: NewBB->begin());
848 if (PN->getNumIncomingValues() == 0) {
849 BasicBlock::iterator I = NewBB->begin();
850 BasicBlock::const_iterator OldI = OldBB->begin();
851 while ((PN = dyn_cast<PHINode>(Val: I++))) {
852 Value *NV = PoisonValue::get(T: PN->getType());
853 PN->replaceAllUsesWith(V: NV);
854 assert(VMap[&*OldI] == PN && "VMap mismatch");
855 VMap[&*OldI] = NV;
856 PN->eraseFromParent();
857 ++OldI;
858 }
859 }
860 }
861
862 // Drop all incompatible return attributes that cannot be applied to NewFunc
863 // during cloning, so as to allow instruction simplification to reason on the
864 // old state of the function. The original attributes are restored later.
865 AttributeList Attrs = NewFunc->getAttributes();
866 AttributeMask IncompatibleAttrs = AttributeFuncs::typeIncompatible(
867 Ty: OldFunc->getReturnType(), AS: Attrs.getRetAttrs());
868 NewFunc->removeRetAttrs(Attrs: IncompatibleAttrs);
869
870 // As phi-nodes have been now remapped, allow incremental simplification of
871 // newly-cloned instructions.
872 const DataLayout &DL = NewFunc->getDataLayout();
873 for (const BasicBlock &BB : *OldFunc) {
874 for (const Instruction &I : BB) {
875 auto *NewI = dyn_cast_or_null<Instruction>(Val: VMap.lookup(Val: &I));
876 if (!NewI)
877 continue;
878
879 if (Value *V = simplifyInstruction(I: NewI, Q: DL)) {
880 NewI->replaceAllUsesWith(V);
881
882 if (isInstructionTriviallyDead(I: NewI)) {
883 NewI->eraseFromParent();
884 } else {
885 // Did not erase it? Restore the new instruction into VMap previously
886 // dropped by `ValueIsRAUWd`.
887 VMap[&I] = NewI;
888 }
889 }
890 }
891 }
892
893 // Restore attributes.
894 NewFunc->setAttributes(Attrs);
895
896 // Remap debug records operands now that all values have been mapped.
897 // Doing this now (late) preserves use-before-defs in debug records. If
898 // we didn't do this, ValueAsMetadata(use-before-def) operands would be
899 // replaced by empty metadata. This would signal later cleanup passes to
900 // remove the debug records, potentially causing incorrect locations.
901 Function::iterator Begin = cast<BasicBlock>(Val&: VMap[StartingBB])->getIterator();
902 for (BasicBlock &BB : make_range(x: Begin, y: NewFunc->end())) {
903 for (Instruction &I : BB) {
904 Mapper.remapDbgRecordRange(M: I.getModule(), Range: I.getDbgRecordRange());
905 }
906 }
907
908 // Simplify conditional branches and switches with a constant operand. We try
909 // to prune these out when cloning, but if the simplification required
910 // looking through PHI nodes, those are only available after forming the full
911 // basic block. That may leave some here, and we still want to prune the dead
912 // code as early as possible.
913 for (BasicBlock &BB : make_range(x: Begin, y: NewFunc->end()))
914 ConstantFoldTerminator(BB: &BB);
915
916 // Some blocks may have become unreachable as a result. Find and delete them.
917 {
918 SmallPtrSet<BasicBlock *, 16> ReachableBlocks;
919 SmallVector<BasicBlock *, 16> Worklist;
920 Worklist.push_back(Elt: &*Begin);
921 while (!Worklist.empty()) {
922 BasicBlock *BB = Worklist.pop_back_val();
923 if (ReachableBlocks.insert(Ptr: BB).second)
924 append_range(C&: Worklist, R: successors(BB));
925 }
926
927 SmallVector<BasicBlock *, 16> UnreachableBlocks;
928 for (BasicBlock &BB : make_range(x: Begin, y: NewFunc->end()))
929 if (!ReachableBlocks.contains(Ptr: &BB))
930 UnreachableBlocks.push_back(Elt: &BB);
931 DeleteDeadBlocks(BBs: UnreachableBlocks);
932 }
933
934 // Now that the inlined function body has been fully constructed, go through
935 // and zap unconditional fall-through branches. This happens all the time when
936 // specializing code: code specialization turns conditional branches into
937 // uncond branches, and this code folds them.
938 Function::iterator I = Begin;
939 while (I != NewFunc->end()) {
940 UncondBrInst *BI = dyn_cast<UncondBrInst>(Val: I->getTerminator());
941 if (!BI) {
942 ++I;
943 continue;
944 }
945
946 BasicBlock *Dest = BI->getSuccessor();
947 if (!Dest->getSinglePredecessor() || Dest->hasAddressTaken()) {
948 ++I;
949 continue;
950 }
951
952 // We shouldn't be able to get single-entry PHI nodes here, as instsimplify
953 // above should have zapped all of them..
954 assert(!isa<PHINode>(Dest->begin()));
955
956 // We know all single-entry PHI nodes in the inlined function have been
957 // removed, so we just need to splice the blocks.
958 BI->eraseFromParent();
959
960 // Make all PHI nodes that referred to Dest now refer to I as their source.
961 Dest->replaceAllUsesWith(V: &*I);
962
963 // Move all the instructions in the succ to the pred.
964 I->splice(ToIt: I->end(), FromBB: Dest);
965
966 // Remove the dest block.
967 Dest->eraseFromParent();
968
969 // Do not increment I, iteratively merge all things this block branches to.
970 }
971
972 // Make a final pass over the basic blocks from the old function to gather
973 // any return instructions which survived folding. We have to do this here
974 // because we can iteratively remove and merge returns above.
975 for (Function::iterator I = cast<BasicBlock>(Val&: VMap[StartingBB])->getIterator(),
976 E = NewFunc->end();
977 I != E; ++I)
978 if (ReturnInst *RI = dyn_cast<ReturnInst>(Val: I->getTerminator()))
979 Returns.push_back(Elt: RI);
980}
981
982/// This works exactly like CloneFunctionInto,
983/// except that it does some simple constant prop and DCE on the fly. The
984/// effect of this is to copy significantly less code in cases where (for
985/// example) a function call with constant arguments is inlined, and those
986/// constant arguments cause a significant amount of code in the callee to be
987/// dead. Since this doesn't produce an exact copy of the input, it can't be
988/// used for things like CloneFunction or CloneModule.
989void llvm::CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc,
990 ValueToValueMapTy &VMap,
991 bool ModuleLevelChanges,
992 SmallVectorImpl<ReturnInst *> &Returns,
993 const char *NameSuffix,
994 ClonedCodeInfo &CodeInfo) {
995 CloneAndPruneIntoFromInst(NewFunc, OldFunc, StartingInst: &OldFunc->front().front(), VMap,
996 ModuleLevelChanges, Returns, NameSuffix, CodeInfo);
997}
998
999/// Remaps instructions in \p Blocks using the mapping in \p VMap.
1000void llvm::remapInstructionsInBlocks(ArrayRef<BasicBlock *> Blocks,
1001 ValueToValueMapTy &VMap) {
1002 // Rewrite the code to refer to itself.
1003 ValueMapper Mapper(VMap, RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
1004 for (BasicBlock *BB : Blocks) {
1005 for (Instruction &Inst : *BB) {
1006 Mapper.remapDbgRecordRange(M: Inst.getModule(), Range: Inst.getDbgRecordRange());
1007 Mapper.remapInstruction(I&: Inst);
1008 }
1009 }
1010}
1011
1012/// Clones a loop \p OrigLoop. Returns the loop and the blocks in \p
1013/// Blocks.
1014///
1015/// Updates LoopInfo and DominatorTree assuming the loop is dominated by block
1016/// \p LoopDomBB. Insert the new blocks before block specified in \p Before.
1017/// The client needs to further update the CFG and DominatorTree after calling
1018/// this function, to ensure the IR remains valid.
1019Loop *llvm::cloneLoopWithPreheader(BasicBlock *Before, BasicBlock *LoopDomBB,
1020 Loop *OrigLoop, ValueToValueMapTy &VMap,
1021 const Twine &NameSuffix, LoopInfo *LI,
1022 DominatorTree *DT,
1023 SmallVectorImpl<BasicBlock *> &Blocks) {
1024 Function *F = OrigLoop->getHeader()->getParent();
1025 Loop *ParentLoop = OrigLoop->getParentLoop();
1026 DenseMap<Loop *, Loop *> LMap;
1027
1028 Loop *NewLoop = LI->AllocateLoop();
1029 LMap[OrigLoop] = NewLoop;
1030 if (ParentLoop)
1031 ParentLoop->addChildLoop(NewChild: NewLoop);
1032 else
1033 LI->addTopLevelLoop(New: NewLoop);
1034
1035 BasicBlock *OrigPH = OrigLoop->getLoopPreheader();
1036 assert(OrigPH && "No preheader");
1037 BasicBlock *NewPH = CloneBasicBlock(BB: OrigPH, VMap, NameSuffix, F);
1038 // To rename the loop PHIs.
1039 VMap[OrigPH] = NewPH;
1040 Blocks.push_back(Elt: NewPH);
1041
1042 // Update LoopInfo.
1043 if (ParentLoop)
1044 ParentLoop->addBasicBlockToLoop(NewBB: NewPH, LI&: *LI);
1045
1046 // Update DominatorTree.
1047 DT->addNewBlock(BB: NewPH, DomBB: LoopDomBB);
1048
1049 for (Loop *CurLoop : OrigLoop->getLoopsInPreorder()) {
1050 Loop *&NewLoop = LMap[CurLoop];
1051 if (!NewLoop) {
1052 NewLoop = LI->AllocateLoop();
1053
1054 // Establish the parent/child relationship.
1055 Loop *OrigParent = CurLoop->getParentLoop();
1056 assert(OrigParent && "Could not find the original parent loop");
1057 Loop *NewParentLoop = LMap[OrigParent];
1058 assert(NewParentLoop && "Could not find the new parent loop");
1059
1060 NewParentLoop->addChildLoop(NewChild: NewLoop);
1061 }
1062 }
1063
1064 for (BasicBlock *BB : OrigLoop->getBlocks()) {
1065 Loop *CurLoop = LI->getLoopFor(BB);
1066 Loop *&NewLoop = LMap[CurLoop];
1067 assert(NewLoop && "Expecting new loop to be allocated");
1068
1069 BasicBlock *NewBB = CloneBasicBlock(BB, VMap, NameSuffix, F);
1070 VMap[BB] = NewBB;
1071
1072 // Update LoopInfo.
1073 NewLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
1074
1075 // Add DominatorTree node. After seeing all blocks, update to correct
1076 // IDom.
1077 DT->addNewBlock(BB: NewBB, DomBB: NewPH);
1078
1079 Blocks.push_back(Elt: NewBB);
1080 }
1081
1082 for (BasicBlock *BB : OrigLoop->getBlocks()) {
1083 // Update loop headers.
1084 Loop *CurLoop = LI->getLoopFor(BB);
1085 if (BB == CurLoop->getHeader())
1086 LMap[CurLoop]->moveToHeader(BB: cast<BasicBlock>(Val&: VMap[BB]));
1087
1088 // Update DominatorTree.
1089 BasicBlock *IDomBB = DT->getNode(BB)->getIDom()->getBlock();
1090 DT->changeImmediateDominator(BB: cast<BasicBlock>(Val&: VMap[BB]),
1091 NewBB: cast<BasicBlock>(Val&: VMap[IDomBB]));
1092 }
1093
1094 // Move them physically from the end of the block list.
1095 F->splice(ToIt: Before->getIterator(), FromF: F, FromIt: NewPH->getIterator());
1096 F->splice(ToIt: Before->getIterator(), FromF: F, FromBeginIt: NewLoop->getHeader()->getIterator(),
1097 FromEndIt: F->end());
1098
1099 return NewLoop;
1100}
1101
1102/// Duplicate non-Phi instructions from the beginning of block up to
1103/// StopAt instruction into a split block between BB and its predecessor.
1104BasicBlock *llvm::DuplicateInstructionsInSplitBetween(
1105 BasicBlock *BB, BasicBlock *PredBB, Instruction *StopAt,
1106 ValueToValueMapTy &ValueMapping, DomTreeUpdater &DTU) {
1107
1108 assert(count(successors(PredBB), BB) == 1 &&
1109 "There must be a single edge between PredBB and BB!");
1110 // We are going to have to map operands from the original BB block to the new
1111 // copy of the block 'NewBB'. If there are PHI nodes in BB, evaluate them to
1112 // account for entry from PredBB.
1113 BasicBlock::iterator BI = BB->begin();
1114 for (; PHINode *PN = dyn_cast<PHINode>(Val&: BI); ++BI)
1115 ValueMapping[PN] = PN->getIncomingValueForBlock(BB: PredBB);
1116
1117 BasicBlock *NewBB = SplitEdge(From: PredBB, To: BB);
1118 NewBB->setName(PredBB->getName() + ".split");
1119 Instruction *NewTerm = NewBB->getTerminator();
1120
1121 // FIXME: SplitEdge does not yet take a DTU, so we include the split edge
1122 // in the update set here.
1123 DTU.applyUpdates(Updates: {{DominatorTree::Delete, PredBB, BB},
1124 {DominatorTree::Insert, PredBB, NewBB},
1125 {DominatorTree::Insert, NewBB, BB}});
1126
1127 // Clone the non-phi instructions of BB into NewBB, keeping track of the
1128 // mapping and using it to remap operands in the cloned instructions.
1129 // Stop once we see the terminator too. This covers the case where BB's
1130 // terminator gets replaced and StopAt == BB's terminator.
1131 for (; StopAt != &*BI && BB->getTerminator() != &*BI; ++BI) {
1132 Instruction *New = BI->clone();
1133 New->setName(BI->getName());
1134 New->insertBefore(InsertPos: NewTerm->getIterator());
1135 New->cloneDebugInfoFrom(From: &*BI);
1136 ValueMapping[&*BI] = New;
1137
1138 // Remap operands to patch up intra-block references.
1139 for (unsigned i = 0, e = New->getNumOperands(); i != e; ++i)
1140 if (Instruction *Inst = dyn_cast<Instruction>(Val: New->getOperand(i))) {
1141 auto I = ValueMapping.find(Val: Inst);
1142 if (I != ValueMapping.end())
1143 New->setOperand(i, Val: I->second);
1144 }
1145
1146 // Remap debug variable operands.
1147 remapDebugVariable(Mapping&: ValueMapping, Inst: New);
1148 }
1149
1150 return NewBB;
1151}
1152
1153void llvm::cloneNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
1154 DenseMap<MDNode *, MDNode *> &ClonedScopes,
1155 StringRef Ext, LLVMContext &Context) {
1156 MDBuilder MDB(Context);
1157
1158 DenseMap<const MDNode *, MDNode *> ClonedDomains;
1159
1160 for (MDNode *ScopeList : NoAliasDeclScopes) {
1161 for (const MDOperand &MDOp : ScopeList->operands()) {
1162 if (MDNode *MD = dyn_cast<MDNode>(Val: MDOp)) {
1163 AliasScopeNode SNANode(MD);
1164
1165 std::string Name;
1166 auto ScopeName = SNANode.getName();
1167 if (!ScopeName.empty())
1168 Name = (Twine(ScopeName) + ":" + Ext).str();
1169 else
1170 Name = std::string(Ext);
1171
1172 // A cloned scope has to go into a clone of its domain if that domain
1173 // has disjoint scopes: the copies of a duplicated access are in the
1174 // same memory region, so they must not become implicitly noalias with
1175 // each other.
1176 const MDNode *Domain = SNANode.getDomain();
1177 MDNode *NewDomain = const_cast<MDNode *>(Domain);
1178 if (AliasScopeDomainNode(Domain).hasDisjointScopes()) {
1179 MDNode *&ClonedDomain = ClonedDomains[Domain];
1180 if (!ClonedDomain)
1181 ClonedDomain = MDB.createAnonymousAliasScopeDomain(
1182 Description: AliasScopeDomainNode(Domain).getDescription(),
1183 /*DisjointScopes=*/true);
1184 NewDomain = ClonedDomain;
1185 }
1186
1187 MDNode *NewScope = MDB.createAnonymousAliasScope(Domain: NewDomain, Name);
1188 ClonedScopes.insert(KV: std::make_pair(x&: MD, y&: NewScope));
1189 }
1190 }
1191 }
1192}
1193
1194void llvm::adaptNoAliasScopes(Instruction *I,
1195 const DenseMap<MDNode *, MDNode *> &ClonedScopes,
1196 LLVMContext &Context) {
1197 auto CloneScopeList = [&](const MDNode *ScopeList) -> MDNode * {
1198 bool NeedsReplacement = false;
1199 SmallVector<Metadata *, 8> NewScopeList;
1200 for (const MDOperand &MDOp : ScopeList->operands()) {
1201 if (MDNode *MD = dyn_cast<MDNode>(Val: MDOp)) {
1202 if (auto *NewMD = ClonedScopes.lookup(Val: MD)) {
1203 NewScopeList.push_back(Elt: NewMD);
1204 NeedsReplacement = true;
1205 continue;
1206 }
1207 NewScopeList.push_back(Elt: MD);
1208 }
1209 }
1210 if (NeedsReplacement)
1211 return MDNode::get(Context, MDs: NewScopeList);
1212 return nullptr;
1213 };
1214
1215 if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(Val: I))
1216 if (MDNode *NewScopeList = CloneScopeList(Decl->getScopeList()))
1217 Decl->setScopeList(NewScopeList);
1218
1219 auto replaceWhenNeeded = [&](unsigned MD_ID) {
1220 if (const MDNode *CSNoAlias = I->getMetadata(KindID: MD_ID))
1221 if (MDNode *NewScopeList = CloneScopeList(CSNoAlias))
1222 I->setMetadata(KindID: MD_ID, Node: NewScopeList);
1223 };
1224 replaceWhenNeeded(LLVMContext::MD_noalias);
1225 replaceWhenNeeded(LLVMContext::MD_alias_scope);
1226}
1227
1228void llvm::cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
1229 ArrayRef<BasicBlock *> NewBlocks,
1230 LLVMContext &Context, StringRef Ext) {
1231 if (NoAliasDeclScopes.empty())
1232 return;
1233
1234 DenseMap<MDNode *, MDNode *> ClonedScopes;
1235 LLVM_DEBUG(dbgs() << "cloneAndAdaptNoAliasScopes: cloning "
1236 << NoAliasDeclScopes.size() << " node(s)\n");
1237
1238 cloneNoAliasScopes(NoAliasDeclScopes, ClonedScopes, Ext, Context);
1239 // Identify instructions using metadata that needs adaptation
1240 for (BasicBlock *NewBlock : NewBlocks)
1241 for (Instruction &I : *NewBlock)
1242 adaptNoAliasScopes(I: &I, ClonedScopes, Context);
1243}
1244
1245void llvm::cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
1246 Instruction *IStart, Instruction *IEnd,
1247 LLVMContext &Context, StringRef Ext) {
1248 if (NoAliasDeclScopes.empty())
1249 return;
1250
1251 DenseMap<MDNode *, MDNode *> ClonedScopes;
1252 LLVM_DEBUG(dbgs() << "cloneAndAdaptNoAliasScopes: cloning "
1253 << NoAliasDeclScopes.size() << " node(s)\n");
1254
1255 cloneNoAliasScopes(NoAliasDeclScopes, ClonedScopes, Ext, Context);
1256 // Identify instructions using metadata that needs adaptation
1257 assert(IStart->getParent() == IEnd->getParent() && "different basic block ?");
1258 auto ItStart = IStart->getIterator();
1259 auto ItEnd = IEnd->getIterator();
1260 ++ItEnd; // IEnd is included, increment ItEnd to get the end of the range
1261 for (auto &I : llvm::make_range(x: ItStart, y: ItEnd))
1262 adaptNoAliasScopes(I: &I, ClonedScopes, Context);
1263}
1264
1265void llvm::identifyNoAliasScopesToClone(
1266 ArrayRef<BasicBlock *> BBs, SmallVectorImpl<MDNode *> &NoAliasDeclScopes) {
1267 for (BasicBlock *BB : BBs)
1268 for (Instruction &I : *BB)
1269 if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(Val: &I))
1270 NoAliasDeclScopes.push_back(Elt: Decl->getScopeList());
1271}
1272
1273void llvm::identifyNoAliasScopesToClone(
1274 BasicBlock::iterator Start, BasicBlock::iterator End,
1275 SmallVectorImpl<MDNode *> &NoAliasDeclScopes) {
1276 for (Instruction &I : make_range(x: Start, y: End))
1277 if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(Val: &I))
1278 NoAliasDeclScopes.push_back(Elt: Decl->getScopeList());
1279}
1280