1//===- CloneFunction.cpp - Clone a function into another function ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the CloneFunctionInto interface, which is used as the
10// low-level function cloner. This is used by the CloneFunction and function
11// inliner to do the dirty work of copying the body of a function around.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/ADT/SmallVector.h"
16#include "llvm/ADT/SmallVectorExtras.h"
17#include "llvm/ADT/Statistic.h"
18#include "llvm/Analysis/ConstantFolding.h"
19#include "llvm/Analysis/DomTreeUpdater.h"
20#include "llvm/Analysis/InstructionSimplify.h"
21#include "llvm/Analysis/LoopInfo.h"
22#include "llvm/IR/AttributeMask.h"
23#include "llvm/IR/CFG.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DebugInfo.h"
26#include "llvm/IR/DerivedTypes.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/InstIterator.h"
29#include "llvm/IR/Instructions.h"
30#include "llvm/IR/IntrinsicInst.h"
31#include "llvm/IR/LLVMContext.h"
32#include "llvm/IR/MDBuilder.h"
33#include "llvm/IR/Metadata.h"
34#include "llvm/IR/Module.h"
35#include "llvm/IR/Operator.h"
36#include "llvm/Transforms/Utils/BasicBlockUtils.h"
37#include "llvm/Transforms/Utils/Cloning.h"
38#include "llvm/Transforms/Utils/Local.h"
39#include "llvm/Transforms/Utils/ValueMapper.h"
40#include <cstdint>
41#include <optional>
42using namespace llvm;
43
44#define DEBUG_TYPE "clone-function"
45
46STATISTIC(RemappedAtomMax, "Highest global NextAtomGroup (after mapping)");
47
48void llvm::mapAtomInstance(const DebugLoc &DL, ValueToValueMapTy &VMap) {
49 uint64_t CurGroup = DL->getAtomGroup();
50 if (!CurGroup)
51 return;
52
53 // Try inserting a new entry. If there's already a mapping for this atom
54 // then there's nothing to do.
55 auto [It, Inserted] = VMap.AtomMap.insert(KV: {{DL.getInlinedAt(), CurGroup}, 0});
56 if (!Inserted)
57 return;
58
59 // Map entry to a new atom group.
60 uint64_t NewGroup = DL->getContext().incNextDILocationAtomGroup();
61 assert(NewGroup > CurGroup && "Next should always be greater than current");
62 It->second = NewGroup;
63
64 RemappedAtomMax = std::max<uint64_t>(a: NewGroup, b: RemappedAtomMax);
65}
66
67static void collectDebugInfoFromInstructions(const Function &F,
68 DebugInfoFinder &DIFinder) {
69 const Module *M = F.getParent();
70 if (!M)
71 return;
72 // Inspect instructions to process e.g. DILexicalBlocks of inlined functions
73 for (const Instruction &I : instructions(F))
74 DIFinder.processInstruction(M: *M, I);
75}
76
77// Create a predicate that matches the metadata that should be identity mapped
78// during function cloning.
79static MetadataPredicate
80createIdentityMDPredicate(const Function &F, CloneFunctionChangeType Changes) {
81 if (Changes >= CloneFunctionChangeType::DifferentModule)
82 return [](const Metadata *MD) { return false; };
83
84 DISubprogram *SPClonedWithinModule = F.getSubprogram();
85
86 // Don't clone inlined subprograms.
87 auto ShouldKeep = [SPClonedWithinModule](const DISubprogram *SP) -> bool {
88 return SP != SPClonedWithinModule;
89 };
90
91 return [=](const Metadata *MD) {
92 // Avoid cloning compile units.
93 if (isa<DICompileUnit>(Val: MD))
94 return true;
95
96 if (auto *SP = dyn_cast<DISubprogram>(Val: MD))
97 return ShouldKeep(SP);
98
99 // If a subprogram isn't going to be cloned skip its lexical blocks as well.
100 if (auto *LScope = dyn_cast<DILocalScope>(Val: MD))
101 return ShouldKeep(LScope->getSubprogram());
102
103 // Avoid cloning local variables of subprograms that won't be cloned.
104 if (auto *DV = dyn_cast<DILocalVariable>(Val: MD))
105 if (auto *S = dyn_cast_or_null<DILocalScope>(Val: DV->getScope()))
106 return ShouldKeep(S->getSubprogram());
107
108 // DIGlobalVariableExpression representing static local variable may be
109 // encountered in DISubprogram's retainedNodes list. Do not remap it, and
110 // remove it from retainedNodes after mapping.
111 if (isa<DIGlobalVariableExpression>(Val: MD))
112 return true;
113
114 // Clone types that are local to subprograms being cloned.
115 // Avoid cloning other types.
116 auto *Type = dyn_cast<DIType>(Val: MD);
117 if (!Type)
118 return false;
119
120 // No need to clone types if subprograms are not cloned.
121 if (SPClonedWithinModule == nullptr)
122 return true;
123
124 // Scopeless types may be derived from local types (e.g. pointers to local
125 // types). They may need cloning.
126 if (const DIDerivedType *DTy = dyn_cast_or_null<DIDerivedType>(Val: Type);
127 DTy && !DTy->getScope())
128 return false;
129
130 auto *LScope = dyn_cast_or_null<DILocalScope>(Val: Type->getScope());
131 if (!LScope)
132 return true;
133
134 if (ShouldKeep(LScope->getSubprogram()))
135 return true;
136
137 return false;
138 };
139}
140
141/// See comments in Cloning.h.
142BasicBlock *llvm::CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap,
143 const Twine &NameSuffix, Function *F,
144 ClonedCodeInfo *CodeInfo, bool MapAtoms) {
145 BasicBlock *NewBB = BasicBlock::Create(Context&: BB->getContext(), Name: "", Parent: F);
146 if (BB->hasName())
147 NewBB->setName(BB->getName() + NameSuffix);
148
149 bool hasCalls = false, hasDynamicAllocas = false, hasMemProfMetadata = false;
150
151 // Loop over all instructions, and copy them over.
152 for (const Instruction &I : *BB) {
153 Instruction *NewInst = I.clone();
154 if (I.hasName())
155 NewInst->setName(I.getName() + NameSuffix);
156
157 NewInst->insertBefore(BB&: *NewBB, InsertPos: NewBB->end());
158 NewInst->cloneDebugInfoFrom(From: &I);
159
160 VMap[&I] = NewInst; // Add instruction map to value.
161
162 if (MapAtoms) {
163 if (const DebugLoc &DL = NewInst->getDebugLoc())
164 mapAtomInstance(DL: DL.get(), VMap);
165 }
166
167 if (isa<CallInst>(Val: I)) {
168 hasCalls = true;
169 hasMemProfMetadata |= I.hasMetadata(KindID: LLVMContext::MD_memprof);
170 hasMemProfMetadata |= I.hasMetadata(KindID: LLVMContext::MD_callsite);
171 }
172 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Val: &I)) {
173 if (!AI->isStaticAlloca()) {
174 hasDynamicAllocas = true;
175 }
176 }
177 }
178
179 if (CodeInfo) {
180 CodeInfo->ContainsCalls |= hasCalls;
181 CodeInfo->ContainsMemProfMetadata |= hasMemProfMetadata;
182 CodeInfo->ContainsDynamicAllocas |= hasDynamicAllocas;
183 }
184 return NewBB;
185}
186
187void llvm::CloneFunctionAttributesInto(Function *NewFunc,
188 const Function *OldFunc,
189 ValueToValueMapTy &VMap,
190 bool ModuleLevelChanges,
191 ValueMapTypeRemapper *TypeMapper,
192 ValueMaterializer *Materializer) {
193 // Copy all attributes other than those stored in Function's AttributeList
194 // which holds e.g. parameters and return value attributes.
195 AttributeList NewAttrs = NewFunc->getAttributes();
196 NewFunc->copyAttributesFrom(Src: OldFunc);
197 NewFunc->setAttributes(NewAttrs);
198
199 const RemapFlags FuncGlobalRefFlags =
200 ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges;
201
202 // Fix up the personality function that got copied over.
203 if (OldFunc->hasPersonalityFn())
204 NewFunc->setPersonalityFn(MapValue(V: OldFunc->getPersonalityFn(), VM&: VMap,
205 Flags: FuncGlobalRefFlags, TypeMapper,
206 Materializer));
207
208 if (OldFunc->hasPrefixData()) {
209 NewFunc->setPrefixData(MapValue(V: OldFunc->getPrefixData(), VM&: VMap,
210 Flags: FuncGlobalRefFlags, TypeMapper,
211 Materializer));
212 }
213
214 if (OldFunc->hasPrologueData()) {
215 NewFunc->setPrologueData(MapValue(V: OldFunc->getPrologueData(), VM&: VMap,
216 Flags: FuncGlobalRefFlags, TypeMapper,
217 Materializer));
218 }
219
220 SmallVector<AttributeSet, 4> NewArgAttrs(NewFunc->arg_size());
221 AttributeList OldAttrs = OldFunc->getAttributes();
222
223 // Clone any argument attributes that are present in the VMap.
224 for (const Argument &OldArg : OldFunc->args()) {
225 if (Argument *NewArg = dyn_cast<Argument>(Val&: VMap[&OldArg])) {
226 // Remap the parameter indices.
227 NewArgAttrs[NewArg->getArgNo()] =
228 OldAttrs.getParamAttrs(ArgNo: OldArg.getArgNo());
229 }
230 }
231
232 NewFunc->setAttributes(
233 AttributeList::get(C&: NewFunc->getContext(), FnAttrs: OldAttrs.getFnAttrs(),
234 RetAttrs: OldAttrs.getRetAttrs(), ArgAttrs: NewArgAttrs));
235}
236
237void llvm::CloneFunctionMetadataInto(Function &NewFunc, const Function &OldFunc,
238 ValueToValueMapTy &VMap,
239 RemapFlags RemapFlag,
240 ValueMapTypeRemapper *TypeMapper,
241 ValueMaterializer *Materializer,
242 const MetadataPredicate *IdentityMD) {
243 SmallVector<std::pair<unsigned, MDNode *>, 1> MDs;
244 OldFunc.getAllMetadata(MDs);
245 for (const auto &[Kind, MD] : MDs) {
246 NewFunc.addMetadata(KindID: Kind, MD&: *MapMetadata(MD, VM&: VMap, Flags: RemapFlag, TypeMapper,
247 Materializer, IdentityMD));
248 }
249}
250
251void llvm::CloneFunctionBodyInto(Function &NewFunc, const Function &OldFunc,
252 ValueToValueMapTy &VMap, RemapFlags RemapFlag,
253 SmallVectorImpl<ReturnInst *> &Returns,
254 const char *NameSuffix,
255 ClonedCodeInfo *CodeInfo,
256 ValueMapTypeRemapper *TypeMapper,
257 ValueMaterializer *Materializer,
258 const MetadataPredicate *IdentityMD) {
259 if (OldFunc.isDeclaration())
260 return;
261
262 // Loop over all of the basic blocks in the function, cloning them as
263 // appropriate. Note that we save BE this way in order to handle cloning of
264 // recursive functions into themselves.
265 for (const BasicBlock &BB : OldFunc) {
266 // Create a new basic block and copy instructions into it!
267 BasicBlock *CBB =
268 CloneBasicBlock(BB: &BB, VMap, NameSuffix, F: &NewFunc, CodeInfo);
269
270 // Add basic block mapping.
271 VMap[&BB] = CBB;
272
273 // It is only legal to clone a function if a block address within that
274 // function is never referenced outside of the function. Given that, we
275 // want to map block addresses from the old function to block addresses in
276 // the clone. (This is different from the generic ValueMapper
277 // implementation, which generates an invalid blockaddress when
278 // cloning a function.)
279 if (BB.hasAddressTaken()) {
280 Constant *OldBBAddr = BlockAddress::get(F: const_cast<Function *>(&OldFunc),
281 BB: const_cast<BasicBlock *>(&BB));
282 VMap[OldBBAddr] = BlockAddress::get(F: &NewFunc, BB: CBB);
283 }
284
285 // Note return instructions for the caller.
286 if (ReturnInst *RI = dyn_cast<ReturnInst>(Val: CBB->getTerminator()))
287 Returns.push_back(Elt: RI);
288 }
289
290 // Loop over all of the instructions in the new function, fixing up operand
291 // references as we go. This uses VMap to do all the hard work.
292 ValueMapper Mapper(VMap, RemapFlag, TypeMapper, Materializer, IdentityMD);
293 for (Function::iterator
294 BB = cast<BasicBlock>(Val&: VMap[&OldFunc.front()])->getIterator(),
295 BE = NewFunc.end();
296 BB != BE; ++BB)
297 // Loop over all instructions, fixing each one as we find it, and any
298 // attached debug-info records.
299 for (Instruction &II : *BB) {
300 Mapper.remapInstruction(I&: II);
301 Mapper.remapDbgRecordRange(M: II.getModule(), Range: II.getDbgRecordRange());
302 }
303}
304
305// Clone OldFunc into NewFunc, transforming the old arguments into references to
306// VMap values.
307void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
308 ValueToValueMapTy &VMap,
309 CloneFunctionChangeType Changes,
310 SmallVectorImpl<ReturnInst *> &Returns,
311 const char *NameSuffix, ClonedCodeInfo *CodeInfo,
312 ValueMapTypeRemapper *TypeMapper,
313 ValueMaterializer *Materializer) {
314 assert(NameSuffix && "NameSuffix cannot be null!");
315
316#ifndef NDEBUG
317 for (const Argument &I : OldFunc->args())
318 assert(VMap.count(&I) && "No mapping from source argument specified!");
319#endif
320
321 bool ModuleLevelChanges = Changes > CloneFunctionChangeType::LocalChangesOnly;
322
323 CloneFunctionAttributesInto(NewFunc, OldFunc, VMap, ModuleLevelChanges,
324 TypeMapper, Materializer);
325
326 // Everything else beyond this point deals with function instructions,
327 // so if we are dealing with a function declaration, we're done.
328 if (OldFunc->isDeclaration())
329 return;
330
331 if (Changes < CloneFunctionChangeType::DifferentModule) {
332 assert((NewFunc->getParent() == nullptr ||
333 NewFunc->getParent() == OldFunc->getParent()) &&
334 "Expected NewFunc to have the same parent, or no parent");
335 } else {
336 assert((NewFunc->getParent() == nullptr ||
337 NewFunc->getParent() != OldFunc->getParent()) &&
338 "Expected NewFunc to have different parents, or no parent");
339
340 if (Changes == CloneFunctionChangeType::DifferentModule) {
341 assert(NewFunc->getParent() &&
342 "Need parent of new function to maintain debug info invariants");
343 }
344 }
345
346 MetadataPredicate IdentityMD = createIdentityMDPredicate(F: *OldFunc, Changes);
347
348 // Cloning is always a Module level operation, since Metadata needs to be
349 // cloned.
350 const RemapFlags RemapFlag = RF_None;
351
352 CloneFunctionMetadataInto(NewFunc&: *NewFunc, OldFunc: *OldFunc, VMap, RemapFlag, TypeMapper,
353 Materializer, IdentityMD: &IdentityMD);
354
355 CloneFunctionBodyInto(NewFunc&: *NewFunc, OldFunc: *OldFunc, VMap, RemapFlag, Returns,
356 NameSuffix, CodeInfo, TypeMapper, Materializer,
357 IdentityMD: &IdentityMD);
358
359 // DIGlobalVariableExpressions representing static locals stay in the scope of
360 // OldSP after function cloning. Remove them from retainedNodes of NewSP.
361 if (DISubprogram *NewSP = NewFunc->getSubprogram())
362 NewSP->cleanupRetainedNodesIf(Pred: [NewSP](Metadata *N) {
363 auto *GVE = dyn_cast_or_null<DIGlobalVariableExpression>(Val: N);
364 return !GVE ||
365 DISubprogram::getRetainedNodeScope(N: GVE)->getSubprogram() == NewSP;
366 });
367
368 // Only update !llvm.dbg.cu for DifferentModule (not CloneModule). In the
369 // same module, the compile unit will already be listed (or not). When
370 // cloning a module, CloneModule() will handle creating the named metadata.
371 if (Changes != CloneFunctionChangeType::DifferentModule)
372 return;
373
374 // Update !llvm.dbg.cu with compile units added to the new module if this
375 // function is being cloned in isolation.
376 //
377 // FIXME: This is making global / module-level changes, which doesn't seem
378 // like the right encapsulation Consider dropping the requirement to update
379 // !llvm.dbg.cu (either obsoleting the node, or restricting it to
380 // non-discardable compile units) instead of discovering compile units by
381 // visiting the metadata attached to global values, which would allow this
382 // code to be deleted. Alternatively, perhaps give responsibility for this
383 // update to CloneFunctionInto's callers.
384 Module *NewModule = NewFunc->getParent();
385 NamedMDNode *NMD = NewModule->getOrInsertNamedMetadata(Name: "llvm.dbg.cu");
386 // Avoid multiple insertions of the same DICompileUnit to NMD.
387 SmallPtrSet<const void *, 8> Visited(llvm::from_range, NMD->operands());
388
389 // Collect and clone all the compile units referenced from the instructions in
390 // the function (e.g. as instructions' scope).
391 DebugInfoFinder DIFinder;
392 collectDebugInfoFromInstructions(F: *OldFunc, DIFinder);
393 for (DICompileUnit *Unit : DIFinder.compile_units()) {
394 MDNode *MappedUnit =
395 MapMetadata(MD: Unit, VM&: VMap, Flags: RF_None, TypeMapper, Materializer);
396 if (Visited.insert(Ptr: MappedUnit).second)
397 NMD->addOperand(M: MappedUnit);
398 }
399}
400
401/// Return a copy of the specified function and add it to that function's
402/// module. Also, any references specified in the VMap are changed to refer to
403/// their mapped value instead of the original one. If any of the arguments to
404/// the function are in the VMap, the arguments are deleted from the resultant
405/// function. The VMap is updated to include mappings from all of the
406/// instructions and basicblocks in the function from their old to new values.
407///
408Function *llvm::CloneFunction(Function *F, ValueToValueMapTy &VMap,
409 ClonedCodeInfo *CodeInfo) {
410 std::vector<Type *> ArgTypes;
411
412 // The user might be deleting arguments to the function by specifying them in
413 // the VMap. If so, we need to not add the arguments to the arg ty vector
414 //
415 for (const Argument &I : F->args())
416 if (VMap.count(Val: &I) == 0) // Haven't mapped the argument to anything yet?
417 ArgTypes.push_back(x: I.getType());
418
419 // Create a new function type...
420 FunctionType *FTy =
421 FunctionType::get(Result: F->getFunctionType()->getReturnType(), Params: ArgTypes,
422 isVarArg: F->getFunctionType()->isVarArg());
423
424 // Create the new function...
425 Function *NewF = Function::Create(Ty: FTy, Linkage: F->getLinkage(), AddrSpace: F->getAddressSpace(),
426 N: F->getName(), M: F->getParent());
427
428 // Loop over the arguments, copying the names of the mapped arguments over...
429 Function::arg_iterator DestI = NewF->arg_begin();
430 for (const Argument &I : F->args())
431 if (VMap.count(Val: &I) == 0) { // Is this argument preserved?
432 DestI->setName(I.getName()); // Copy the name over...
433 VMap[&I] = &*DestI++; // Add mapping to VMap
434 }
435
436 SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned.
437 CloneFunctionInto(NewFunc: NewF, OldFunc: F, VMap, Changes: CloneFunctionChangeType::LocalChangesOnly,
438 Returns, NameSuffix: "", CodeInfo);
439
440 return NewF;
441}
442
443namespace {
444/// This is a private class used to implement CloneAndPruneFunctionInto.
445struct PruningFunctionCloner {
446 Function *NewFunc;
447 const Function *OldFunc;
448 ValueToValueMapTy &VMap;
449 ValueMapper &Remapper;
450 bool ModuleLevelChanges;
451 const char *NameSuffix;
452 ClonedCodeInfo &CodeInfo;
453 bool HostFuncIsStrictFP;
454
455 Instruction *cloneInstruction(BasicBlock::const_iterator II);
456
457public:
458 PruningFunctionCloner(Function *newFunc, const Function *oldFunc,
459 ValueToValueMapTy &valueMap, ValueMapper &remapper,
460 bool moduleLevelChanges, const char *nameSuffix,
461 ClonedCodeInfo &codeInfo)
462 : NewFunc(newFunc), OldFunc(oldFunc), VMap(valueMap), Remapper(remapper),
463 ModuleLevelChanges(moduleLevelChanges), NameSuffix(nameSuffix),
464 CodeInfo(codeInfo) {
465 HostFuncIsStrictFP =
466 newFunc->getAttributes().hasFnAttr(Kind: Attribute::StrictFP);
467 }
468
469 /// The specified block is found to be reachable, clone it and
470 /// anything that it can reach.
471 void CloneBlock(const BasicBlock *BB, BasicBlock::const_iterator StartingInst,
472 std::vector<const BasicBlock *> &ToClone);
473};
474} // namespace
475
476Instruction *
477PruningFunctionCloner::cloneInstruction(BasicBlock::const_iterator II) {
478 if (!HostFuncIsStrictFP)
479 return II->clone();
480
481 const Instruction &OldInst = *II;
482 Intrinsic::ID CIID = getConstrainedIntrinsicID(Instr: OldInst);
483 if (CIID == Intrinsic::not_intrinsic)
484 return II->clone();
485
486 // Instead of cloning the instruction, a call to constrained intrinsic should
487 // be created. Assume the first arguments of constrained intrinsics are the
488 // same as the operands of original instruction.
489
490 // Create intrinsic call.
491 LLVMContext &Ctx = NewFunc->getContext();
492 SmallVector<Value *, 4> Args;
493 unsigned NumOperands = OldInst.getNumOperands();
494 if (isa<CallInst>(Val: OldInst))
495 --NumOperands;
496 for (unsigned I = 0; I < NumOperands; ++I)
497 Args.push_back(Elt: OldInst.getOperand(i: I));
498
499 if (const auto *CmpI = dyn_cast<FCmpInst>(Val: &OldInst)) {
500 FCmpInst::Predicate Pred = CmpI->getPredicate();
501 StringRef PredName = FCmpInst::getPredicateName(P: Pred);
502 Args.push_back(Elt: MetadataAsValue::get(Context&: Ctx, MD: MDString::get(Context&: Ctx, Str: PredName)));
503 }
504
505 // The last arguments of a constrained intrinsic are metadata that represent
506 // rounding mode (absent in some intrinsics) and exception behavior. The
507 // inlined function uses default settings.
508 if (Intrinsic::hasConstrainedFPRoundingModeOperand(QID: CIID))
509 Args.push_back(
510 Elt: MetadataAsValue::get(Context&: Ctx, MD: MDString::get(Context&: Ctx, Str: "round.tonearest")));
511 Args.push_back(
512 Elt: MetadataAsValue::get(Context&: Ctx, MD: MDString::get(Context&: Ctx, Str: "fpexcept.ignore")));
513
514 SmallVector<Type *> ArgTys = llvm::map_to_vector(C&: Args, F: &Value::getType);
515 Function *IFn = Intrinsic::getOrInsertDeclaration(M: NewFunc->getParent(), IID: CIID,
516 RetTy: OldInst.getType(), ArgTys);
517 CallInst *NewCall =
518 CallInst::Create(Func: IFn, Args, NameStr: OldInst.getName() + ".strict");
519 if (isa<FPMathOperator>(Val: NewCall))
520 NewCall->copyFastMathFlags(I: &OldInst);
521 return NewCall;
522}
523
524/// The specified block is found to be reachable, clone it and
525/// anything that it can reach.
526void PruningFunctionCloner::CloneBlock(
527 const BasicBlock *BB, BasicBlock::const_iterator StartingInst,
528 std::vector<const BasicBlock *> &ToClone) {
529 WeakTrackingVH &BBEntry = VMap[BB];
530
531 // Have we already cloned this block?
532 if (BBEntry)
533 return;
534
535 // Nope, clone it now.
536 BasicBlock *NewBB;
537 Twine NewName(BB->hasName() ? Twine(BB->getName()) + NameSuffix : "");
538 BBEntry = NewBB = BasicBlock::Create(Context&: BB->getContext(), Name: NewName, Parent: NewFunc);
539
540 // It is only legal to clone a function if a block address within that
541 // function is never referenced outside of the function. Given that, we
542 // want to map block addresses from the old function to block addresses in
543 // the clone. (This is different from the generic ValueMapper
544 // implementation, which generates an invalid blockaddress when
545 // cloning a function.)
546 //
547 // Note that we don't need to fix the mapping for unreachable blocks;
548 // the default mapping there is safe.
549 if (BB->hasAddressTaken()) {
550 Constant *OldBBAddr = BlockAddress::get(F: const_cast<Function *>(OldFunc),
551 BB: const_cast<BasicBlock *>(BB));
552 VMap[OldBBAddr] = BlockAddress::get(F: NewFunc, BB: NewBB);
553 }
554
555 bool hasCalls = false, hasDynamicAllocas = false, hasStaticAllocas = false;
556 bool hasMemProfMetadata = false;
557
558 // Keep a cursor pointing at the last place we cloned debug-info records from.
559 BasicBlock::const_iterator DbgCursor = StartingInst;
560 auto CloneDbgRecordsToHere =
561 [&DbgCursor](Instruction *NewInst, BasicBlock::const_iterator II) {
562 // Clone debug-info records onto this instruction. Iterate through any
563 // source-instructions we've cloned and then subsequently optimised
564 // away, so that their debug-info doesn't go missing.
565 for (; DbgCursor != II; ++DbgCursor)
566 NewInst->cloneDebugInfoFrom(From: &*DbgCursor, FromHere: std::nullopt, InsertAtHead: false);
567 NewInst->cloneDebugInfoFrom(From: &*II);
568 DbgCursor = std::next(x: II);
569 };
570
571 // Loop over all instructions, and copy them over, DCE'ing as we go. This
572 // loop doesn't include the terminator.
573 for (BasicBlock::const_iterator II = StartingInst, IE = --BB->end(); II != IE;
574 ++II) {
575
576 // Don't clone fake_use as it may suppress many optimizations
577 // due to inlining, especially SROA.
578 if (auto *IntrInst = dyn_cast<IntrinsicInst>(Val&: II))
579 if (IntrInst->getIntrinsicID() == Intrinsic::fake_use)
580 continue;
581
582 Instruction *NewInst = cloneInstruction(II);
583 NewInst->insertInto(ParentBB: NewBB, It: NewBB->end());
584
585 if (HostFuncIsStrictFP) {
586 // All function calls in the inlined function must get 'strictfp'
587 // attribute to prevent undesirable optimizations.
588 if (auto *Call = dyn_cast<CallInst>(Val: NewInst))
589 Call->addFnAttr(Kind: Attribute::StrictFP);
590 }
591
592 // Eagerly remap operands to the newly cloned instruction, except for PHI
593 // nodes for which we defer processing until we update the CFG.
594 if (!isa<PHINode>(Val: NewInst)) {
595 Remapper.remapInstruction(I&: *NewInst);
596
597 // Eagerly constant fold the newly cloned instruction. If successful, add
598 // a mapping to the new value. Non-constant operands may be incomplete at
599 // this stage, thus instruction simplification is performed after
600 // processing phi-nodes.
601 if (Value *V = ConstantFoldInstruction(
602 I: NewInst, DL: BB->getDataLayout())) {
603 if (isInstructionTriviallyDead(I: NewInst)) {
604 VMap[&*II] = V;
605 NewInst->eraseFromParent();
606 continue;
607 }
608 }
609 }
610
611 if (auto *CB = dyn_cast<CallBase>(Val&: II); CB && CB->isIndirectCall())
612 CodeInfo.OriginallyIndirectCalls.insert(X: NewInst);
613
614 if (II->hasName())
615 NewInst->setName(II->getName() + NameSuffix);
616 VMap[&*II] = NewInst; // Add instruction map to value.
617 if (isa<CallInst>(Val: II)) {
618 hasCalls = true;
619 hasMemProfMetadata |= II->hasMetadata(KindID: LLVMContext::MD_memprof);
620 hasMemProfMetadata |= II->hasMetadata(KindID: LLVMContext::MD_callsite);
621 }
622
623 CloneDbgRecordsToHere(NewInst, II);
624
625 CodeInfo.OrigVMap[&*II] = NewInst;
626 if (auto *CB = dyn_cast<CallBase>(Val: &*II))
627 if (CB->hasOperandBundles())
628 CodeInfo.OperandBundleCallSites.push_back(x: NewInst);
629
630 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Val&: II)) {
631 if (isa<ConstantInt>(Val: AI->getArraySize()))
632 hasStaticAllocas = true;
633 else
634 hasDynamicAllocas = true;
635 }
636 }
637
638 // Finally, clone over the terminator.
639 const Instruction *OldTI = BB->getTerminator();
640 bool TerminatorDone = false;
641 if (const CondBrInst *BI = dyn_cast<CondBrInst>(Val: OldTI)) {
642 // If the condition was a known constant in the callee...
643 ConstantInt *Cond = dyn_cast<ConstantInt>(Val: BI->getCondition());
644 // Or is a known constant in the caller...
645 if (!Cond) {
646 Value *V = VMap.lookup(Val: BI->getCondition());
647 Cond = dyn_cast_or_null<ConstantInt>(Val: V);
648 }
649
650 // Constant fold to uncond branch!
651 if (Cond) {
652 BasicBlock *Dest = BI->getSuccessor(i: !Cond->getZExtValue());
653 auto *NewBI = UncondBrInst::Create(Target: Dest, InsertBefore: NewBB);
654 NewBI->setDebugLoc(BI->getDebugLoc());
655 VMap[OldTI] = NewBI;
656 ToClone.push_back(x: Dest);
657 TerminatorDone = true;
658 }
659 } else if (const SwitchInst *SI = dyn_cast<SwitchInst>(Val: OldTI)) {
660 // If switching on a value known constant in the caller.
661 ConstantInt *Cond = dyn_cast<ConstantInt>(Val: SI->getCondition());
662 if (!Cond) { // Or known constant after constant prop in the callee...
663 Value *V = VMap.lookup(Val: SI->getCondition());
664 Cond = dyn_cast_or_null<ConstantInt>(Val: V);
665 }
666 if (Cond) { // Constant fold to uncond branch!
667 SwitchInst::ConstCaseHandle Case = *SI->findCaseValue(C: Cond);
668 BasicBlock *Dest = const_cast<BasicBlock *>(Case.getCaseSuccessor());
669 auto *NewBI = UncondBrInst::Create(Target: Dest, InsertBefore: NewBB);
670 NewBI->setDebugLoc(SI->getDebugLoc());
671 VMap[OldTI] = NewBI;
672 ToClone.push_back(x: Dest);
673 TerminatorDone = true;
674 }
675 }
676
677 if (!TerminatorDone) {
678 Instruction *NewInst = OldTI->clone();
679 if (OldTI->hasName())
680 NewInst->setName(OldTI->getName() + NameSuffix);
681 NewInst->insertInto(ParentBB: NewBB, It: NewBB->end());
682
683 CloneDbgRecordsToHere(NewInst, OldTI->getIterator());
684
685 VMap[OldTI] = NewInst; // Add instruction map to value.
686
687 CodeInfo.OrigVMap[OldTI] = NewInst;
688 if (auto *CB = dyn_cast<CallBase>(Val: OldTI))
689 if (CB->hasOperandBundles())
690 CodeInfo.OperandBundleCallSites.push_back(x: NewInst);
691
692 // Recursively clone any reachable successor blocks.
693 append_range(C&: ToClone, R: successors(I: BB->getTerminator()));
694 } else {
695 // If we didn't create a new terminator, clone DbgVariableRecords from the
696 // old terminator onto the new terminator.
697 Instruction *NewInst = NewBB->getTerminator();
698
699 CloneDbgRecordsToHere(NewInst, OldTI->getIterator());
700 }
701
702 CodeInfo.ContainsCalls |= hasCalls;
703 CodeInfo.ContainsMemProfMetadata |= hasMemProfMetadata;
704 CodeInfo.ContainsDynamicAllocas |= hasDynamicAllocas;
705 CodeInfo.ContainsDynamicAllocas |=
706 hasStaticAllocas && BB != &BB->getParent()->front();
707}
708
709/// This works like CloneAndPruneFunctionInto, except that it does not clone the
710/// entire function. Instead it starts at an instruction provided by the caller
711/// and copies (and prunes) only the code reachable from that instruction.
712void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
713 const Instruction *StartingInst,
714 ValueToValueMapTy &VMap,
715 bool ModuleLevelChanges,
716 SmallVectorImpl<ReturnInst *> &Returns,
717 const char *NameSuffix,
718 ClonedCodeInfo &CodeInfo) {
719 assert(NameSuffix && "NameSuffix cannot be null!");
720
721 ValueMapTypeRemapper *TypeMapper = nullptr;
722 ValueMaterializer *Materializer = nullptr;
723
724#ifndef NDEBUG
725 // If the cloning starts at the beginning of the function, verify that
726 // the function arguments are mapped.
727 if (!StartingInst)
728 for (const Argument &II : OldFunc->args())
729 assert(VMap.count(&II) && "No mapping from source argument specified!");
730#endif
731
732 ValueMapper Mapper(VMap,
733 ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges,
734 TypeMapper, Materializer);
735 PruningFunctionCloner PFC(NewFunc, OldFunc, VMap, Mapper, ModuleLevelChanges,
736 NameSuffix, CodeInfo);
737 const BasicBlock *StartingBB;
738 if (StartingInst)
739 StartingBB = StartingInst->getParent();
740 else {
741 StartingBB = &OldFunc->getEntryBlock();
742 StartingInst = &StartingBB->front();
743 }
744
745 // Clone the entry block, and anything recursively reachable from it.
746 std::vector<const BasicBlock *> CloneWorklist;
747 PFC.CloneBlock(BB: StartingBB, StartingInst: StartingInst->getIterator(), ToClone&: CloneWorklist);
748 while (!CloneWorklist.empty()) {
749 const BasicBlock *BB = CloneWorklist.back();
750 CloneWorklist.pop_back();
751 PFC.CloneBlock(BB, StartingInst: BB->begin(), ToClone&: CloneWorklist);
752 }
753
754 // Loop over all of the basic blocks in the old function. If the block was
755 // reachable, we have cloned it and the old block is now in the value map:
756 // insert it into the new function in the right order. If not, ignore it.
757 //
758 // Defer PHI resolution until rest of function is resolved.
759 SmallVector<const PHINode *, 16> PHIToResolve;
760 for (const BasicBlock &BI : *OldFunc) {
761 Value *V = VMap.lookup(Val: &BI);
762 BasicBlock *NewBB = cast_or_null<BasicBlock>(Val: V);
763 if (!NewBB)
764 continue; // Dead block.
765
766 // Move the new block to preserve the order in the original function.
767 NewBB->moveBefore(MovePos: NewFunc->end());
768
769 // Handle PHI nodes specially, as we have to remove references to dead
770 // blocks.
771 for (const PHINode &PN : BI.phis()) {
772 // PHI nodes may have been remapped to non-PHI nodes by the caller or
773 // during the cloning process.
774 if (isa<PHINode>(Val: VMap[&PN]))
775 PHIToResolve.push_back(Elt: &PN);
776 else
777 break;
778 }
779
780 // Finally, remap the terminator instructions, as those can't be remapped
781 // until all BBs are mapped.
782 Mapper.remapInstruction(I&: *NewBB->getTerminator());
783 }
784
785 // Defer PHI resolution until rest of function is resolved, PHI resolution
786 // requires the CFG to be up-to-date.
787 for (unsigned phino = 0, e = PHIToResolve.size(); phino != e;) {
788 const PHINode *OPN = PHIToResolve[phino];
789 unsigned NumPreds = OPN->getNumIncomingValues();
790 const BasicBlock *OldBB = OPN->getParent();
791 BasicBlock *NewBB = cast<BasicBlock>(Val&: VMap[OldBB]);
792
793 // Map operands for blocks that are live and remove operands for blocks
794 // that are dead.
795 for (; phino != PHIToResolve.size() &&
796 PHIToResolve[phino]->getParent() == OldBB;
797 ++phino) {
798 OPN = PHIToResolve[phino];
799 PHINode *PN = cast<PHINode>(Val&: VMap[OPN]);
800 for (int64_t pred = NumPreds - 1; pred >= 0; --pred) {
801 Value *V = VMap.lookup(Val: PN->getIncomingBlock(i: pred));
802 if (BasicBlock *MappedBlock = cast_or_null<BasicBlock>(Val: V)) {
803 Value *InVal = Mapper.mapValue(V: *PN->getIncomingValue(i: pred));
804 assert(InVal && "Unknown input value?");
805 PN->setIncomingValue(i: pred, V: InVal);
806 PN->setIncomingBlock(i: pred, BB: MappedBlock);
807 continue;
808 }
809 PN->removeIncomingValue(Idx: pred, DeletePHIIfEmpty: false);
810 }
811 }
812
813 // The loop above has removed PHI entries for those blocks that are dead
814 // and has updated others. However, if a block is live (i.e. copied over)
815 // but its terminator has been changed to not go to this block, then our
816 // phi nodes will have invalid entries. Update the PHI nodes in this
817 // case.
818 PHINode *PN = cast<PHINode>(Val: NewBB->begin());
819 NumPreds = pred_size(BB: NewBB);
820 if (NumPreds != PN->getNumIncomingValues()) {
821 assert(NumPreds < PN->getNumIncomingValues());
822 // Count how many times each predecessor comes to this block.
823 DenseMap<BasicBlock *, unsigned> PredCount;
824 for (BasicBlock *Pred : predecessors(BB: NewBB))
825 ++PredCount[Pred];
826
827 BasicBlock::iterator I = NewBB->begin();
828 DenseMap<BasicBlock *, unsigned> SeenPredCount;
829 SeenPredCount.reserve(NumEntries: PredCount.size());
830 for (; (PN = dyn_cast<PHINode>(Val&: I)); ++I) {
831 SeenPredCount.clear();
832 PN->removeIncomingValueIf(
833 Predicate: [&](unsigned Idx) {
834 BasicBlock *IncomingBlock = PN->getIncomingBlock(i: Idx);
835 auto It = PredCount.find(Val: IncomingBlock);
836 if (It == PredCount.end())
837 return true;
838 unsigned &SeenCount = SeenPredCount[IncomingBlock];
839 if (SeenCount < It->second) {
840 SeenCount++;
841 return false;
842 }
843 return true;
844 },
845 DeletePHIIfEmpty: false);
846 }
847 }
848
849 // If the loops above have made these phi nodes have 0 or 1 operand,
850 // replace them with poison or the input value. We must do this for
851 // correctness, because 0-operand phis are not valid.
852 PN = cast<PHINode>(Val: NewBB->begin());
853 if (PN->getNumIncomingValues() == 0) {
854 BasicBlock::iterator I = NewBB->begin();
855 BasicBlock::const_iterator OldI = OldBB->begin();
856 while ((PN = dyn_cast<PHINode>(Val: I++))) {
857 Value *NV = PoisonValue::get(T: PN->getType());
858 PN->replaceAllUsesWith(V: NV);
859 assert(VMap[&*OldI] == PN && "VMap mismatch");
860 VMap[&*OldI] = NV;
861 PN->eraseFromParent();
862 ++OldI;
863 }
864 }
865 }
866
867 // Drop all incompatible return attributes that cannot be applied to NewFunc
868 // during cloning, so as to allow instruction simplification to reason on the
869 // old state of the function. The original attributes are restored later.
870 AttributeList Attrs = NewFunc->getAttributes();
871 AttributeMask IncompatibleAttrs = AttributeFuncs::typeIncompatible(
872 Ty: OldFunc->getReturnType(), AS: Attrs.getRetAttrs());
873 NewFunc->removeRetAttrs(Attrs: IncompatibleAttrs);
874
875 // As phi-nodes have been now remapped, allow incremental simplification of
876 // newly-cloned instructions.
877 const DataLayout &DL = NewFunc->getDataLayout();
878 for (const BasicBlock &BB : *OldFunc) {
879 for (const Instruction &I : BB) {
880 auto *NewI = dyn_cast_or_null<Instruction>(Val: VMap.lookup(Val: &I));
881 if (!NewI)
882 continue;
883
884 if (Value *V = simplifyInstruction(I: NewI, Q: DL)) {
885 NewI->replaceAllUsesWith(V);
886
887 if (isInstructionTriviallyDead(I: NewI)) {
888 NewI->eraseFromParent();
889 } else {
890 // Did not erase it? Restore the new instruction into VMap previously
891 // dropped by `ValueIsRAUWd`.
892 VMap[&I] = NewI;
893 }
894 }
895 }
896 }
897
898 // Restore attributes.
899 NewFunc->setAttributes(Attrs);
900
901 // Remap debug records operands now that all values have been mapped.
902 // Doing this now (late) preserves use-before-defs in debug records. If
903 // we didn't do this, ValueAsMetadata(use-before-def) operands would be
904 // replaced by empty metadata. This would signal later cleanup passes to
905 // remove the debug records, potentially causing incorrect locations.
906 Function::iterator Begin = cast<BasicBlock>(Val&: VMap[StartingBB])->getIterator();
907 for (BasicBlock &BB : make_range(x: Begin, y: NewFunc->end())) {
908 for (Instruction &I : BB) {
909 Mapper.remapDbgRecordRange(M: I.getModule(), Range: I.getDbgRecordRange());
910 }
911 }
912
913 // Simplify conditional branches and switches with a constant operand. We try
914 // to prune these out when cloning, but if the simplification required
915 // looking through PHI nodes, those are only available after forming the full
916 // basic block. That may leave some here, and we still want to prune the dead
917 // code as early as possible.
918 for (BasicBlock &BB : make_range(x: Begin, y: NewFunc->end()))
919 ConstantFoldTerminator(BB: &BB);
920
921 // Some blocks may have become unreachable as a result. Find and delete them.
922 {
923 SmallPtrSet<BasicBlock *, 16> ReachableBlocks;
924 SmallVector<BasicBlock *, 16> Worklist;
925 Worklist.push_back(Elt: &*Begin);
926 while (!Worklist.empty()) {
927 BasicBlock *BB = Worklist.pop_back_val();
928 if (ReachableBlocks.insert(Ptr: BB).second)
929 append_range(C&: Worklist, R: successors(BB));
930 }
931
932 SmallVector<BasicBlock *, 16> UnreachableBlocks;
933 for (BasicBlock &BB : make_range(x: Begin, y: NewFunc->end()))
934 if (!ReachableBlocks.contains(Ptr: &BB))
935 UnreachableBlocks.push_back(Elt: &BB);
936 DeleteDeadBlocks(BBs: UnreachableBlocks);
937 }
938
939 // Now that the inlined function body has been fully constructed, go through
940 // and zap unconditional fall-through branches. This happens all the time when
941 // specializing code: code specialization turns conditional branches into
942 // uncond branches, and this code folds them.
943 Function::iterator I = Begin;
944 while (I != NewFunc->end()) {
945 UncondBrInst *BI = dyn_cast<UncondBrInst>(Val: I->getTerminator());
946 if (!BI) {
947 ++I;
948 continue;
949 }
950
951 BasicBlock *Dest = BI->getSuccessor();
952 if (!Dest->getSinglePredecessor() || Dest->hasAddressTaken()) {
953 ++I;
954 continue;
955 }
956
957 // We shouldn't be able to get single-entry PHI nodes here, as instsimplify
958 // above should have zapped all of them..
959 assert(!isa<PHINode>(Dest->begin()));
960
961 // We know all single-entry PHI nodes in the inlined function have been
962 // removed, so we just need to splice the blocks.
963 BI->eraseFromParent();
964
965 // Make all PHI nodes that referred to Dest now refer to I as their source.
966 Dest->replaceAllUsesWith(V: &*I);
967
968 // Move all the instructions in the succ to the pred.
969 I->splice(ToIt: I->end(), FromBB: Dest);
970
971 // Remove the dest block.
972 Dest->eraseFromParent();
973
974 // Do not increment I, iteratively merge all things this block branches to.
975 }
976
977 // Make a final pass over the basic blocks from the old function to gather
978 // any return instructions which survived folding. We have to do this here
979 // because we can iteratively remove and merge returns above.
980 for (Function::iterator I = cast<BasicBlock>(Val&: VMap[StartingBB])->getIterator(),
981 E = NewFunc->end();
982 I != E; ++I)
983 if (ReturnInst *RI = dyn_cast<ReturnInst>(Val: I->getTerminator()))
984 Returns.push_back(Elt: RI);
985}
986
987/// This works exactly like CloneFunctionInto,
988/// except that it does some simple constant prop and DCE on the fly. The
989/// effect of this is to copy significantly less code in cases where (for
990/// example) a function call with constant arguments is inlined, and those
991/// constant arguments cause a significant amount of code in the callee to be
992/// dead. Since this doesn't produce an exact copy of the input, it can't be
993/// used for things like CloneFunction or CloneModule.
994void llvm::CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc,
995 ValueToValueMapTy &VMap,
996 bool ModuleLevelChanges,
997 SmallVectorImpl<ReturnInst *> &Returns,
998 const char *NameSuffix,
999 ClonedCodeInfo &CodeInfo) {
1000 CloneAndPruneIntoFromInst(NewFunc, OldFunc, StartingInst: &OldFunc->front().front(), VMap,
1001 ModuleLevelChanges, Returns, NameSuffix, CodeInfo);
1002}
1003
1004/// Remaps instructions in \p Blocks using the mapping in \p VMap.
1005void llvm::remapInstructionsInBlocks(ArrayRef<BasicBlock *> Blocks,
1006 ValueToValueMapTy &VMap) {
1007 // Rewrite the code to refer to itself.
1008 ValueMapper Mapper(VMap, RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
1009 for (BasicBlock *BB : Blocks) {
1010 for (Instruction &Inst : *BB) {
1011 Mapper.remapDbgRecordRange(M: Inst.getModule(), Range: Inst.getDbgRecordRange());
1012 Mapper.remapInstruction(I&: Inst);
1013 }
1014 }
1015}
1016
1017/// Clones a loop \p OrigLoop. Returns the loop and the blocks in \p
1018/// Blocks.
1019///
1020/// Updates LoopInfo and DominatorTree assuming the loop is dominated by block
1021/// \p LoopDomBB. Insert the new blocks before block specified in \p Before.
1022/// The client needs to further update the CFG and DominatorTree after calling
1023/// this function, to ensure the IR remains valid.
1024Loop *llvm::cloneLoopWithPreheader(BasicBlock *Before, BasicBlock *LoopDomBB,
1025 Loop *OrigLoop, ValueToValueMapTy &VMap,
1026 const Twine &NameSuffix, LoopInfo *LI,
1027 DominatorTree *DT,
1028 SmallVectorImpl<BasicBlock *> &Blocks) {
1029 Function *F = OrigLoop->getHeader()->getParent();
1030 Loop *ParentLoop = OrigLoop->getParentLoop();
1031 DenseMap<Loop *, Loop *> LMap;
1032
1033 Loop *NewLoop = LI->AllocateLoop();
1034 LMap[OrigLoop] = NewLoop;
1035 if (ParentLoop)
1036 ParentLoop->addChildLoop(NewChild: NewLoop);
1037 else
1038 LI->addTopLevelLoop(New: NewLoop);
1039
1040 BasicBlock *OrigPH = OrigLoop->getLoopPreheader();
1041 assert(OrigPH && "No preheader");
1042 BasicBlock *NewPH = CloneBasicBlock(BB: OrigPH, VMap, NameSuffix, F);
1043 // To rename the loop PHIs.
1044 VMap[OrigPH] = NewPH;
1045 Blocks.push_back(Elt: NewPH);
1046
1047 // Update LoopInfo.
1048 if (ParentLoop)
1049 ParentLoop->addBasicBlockToLoop(NewBB: NewPH, LI&: *LI);
1050
1051 // Update DominatorTree.
1052 DT->addNewBlock(BB: NewPH, DomBB: LoopDomBB);
1053
1054 for (Loop *CurLoop : OrigLoop->getLoopsInPreorder()) {
1055 Loop *&NewLoop = LMap[CurLoop];
1056 if (!NewLoop) {
1057 NewLoop = LI->AllocateLoop();
1058
1059 // Establish the parent/child relationship.
1060 Loop *OrigParent = CurLoop->getParentLoop();
1061 assert(OrigParent && "Could not find the original parent loop");
1062 Loop *NewParentLoop = LMap[OrigParent];
1063 assert(NewParentLoop && "Could not find the new parent loop");
1064
1065 NewParentLoop->addChildLoop(NewChild: NewLoop);
1066 }
1067 }
1068
1069 for (BasicBlock *BB : OrigLoop->getBlocks()) {
1070 Loop *CurLoop = LI->getLoopFor(BB);
1071 Loop *&NewLoop = LMap[CurLoop];
1072 assert(NewLoop && "Expecting new loop to be allocated");
1073
1074 BasicBlock *NewBB = CloneBasicBlock(BB, VMap, NameSuffix, F);
1075 VMap[BB] = NewBB;
1076
1077 // Update LoopInfo.
1078 NewLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
1079
1080 // Add DominatorTree node. After seeing all blocks, update to correct
1081 // IDom.
1082 DT->addNewBlock(BB: NewBB, DomBB: NewPH);
1083
1084 Blocks.push_back(Elt: NewBB);
1085 }
1086
1087 for (BasicBlock *BB : OrigLoop->getBlocks()) {
1088 // Update loop headers.
1089 Loop *CurLoop = LI->getLoopFor(BB);
1090 if (BB == CurLoop->getHeader())
1091 LMap[CurLoop]->moveToHeader(BB: cast<BasicBlock>(Val&: VMap[BB]));
1092
1093 // Update DominatorTree.
1094 BasicBlock *IDomBB = DT->getNode(BB)->getIDom()->getBlock();
1095 DT->changeImmediateDominator(BB: cast<BasicBlock>(Val&: VMap[BB]),
1096 NewBB: cast<BasicBlock>(Val&: VMap[IDomBB]));
1097 }
1098
1099 // Move them physically from the end of the block list.
1100 F->splice(ToIt: Before->getIterator(), FromF: F, FromIt: NewPH->getIterator());
1101 F->splice(ToIt: Before->getIterator(), FromF: F, FromBeginIt: NewLoop->getHeader()->getIterator(),
1102 FromEndIt: F->end());
1103
1104 return NewLoop;
1105}
1106
1107/// Duplicate non-Phi instructions from the beginning of block up to
1108/// StopAt instruction into a split block between BB and its predecessor.
1109BasicBlock *llvm::DuplicateInstructionsInSplitBetween(
1110 BasicBlock *BB, BasicBlock *PredBB, Instruction *StopAt,
1111 ValueToValueMapTy &ValueMapping, DomTreeUpdater &DTU) {
1112
1113 assert(count(successors(PredBB), BB) == 1 &&
1114 "There must be a single edge between PredBB and BB!");
1115 // We are going to have to map operands from the original BB block to the new
1116 // copy of the block 'NewBB'. If there are PHI nodes in BB, evaluate them to
1117 // account for entry from PredBB.
1118 BasicBlock::iterator BI = BB->begin();
1119 for (; PHINode *PN = dyn_cast<PHINode>(Val&: BI); ++BI)
1120 ValueMapping[PN] = PN->getIncomingValueForBlock(BB: PredBB);
1121
1122 BasicBlock *NewBB = SplitEdge(From: PredBB, To: BB);
1123 NewBB->setName(PredBB->getName() + ".split");
1124 Instruction *NewTerm = NewBB->getTerminator();
1125
1126 // FIXME: SplitEdge does not yet take a DTU, so we include the split edge
1127 // in the update set here.
1128 DTU.applyUpdates(Updates: {{DominatorTree::Delete, PredBB, BB},
1129 {DominatorTree::Insert, PredBB, NewBB},
1130 {DominatorTree::Insert, NewBB, BB}});
1131
1132 // Clone the non-phi instructions of BB into NewBB, keeping track of the
1133 // mapping and using it to remap operands in the cloned instructions.
1134 // Stop once we see the terminator too. This covers the case where BB's
1135 // terminator gets replaced and StopAt == BB's terminator.
1136 for (; StopAt != &*BI && BB->getTerminator() != &*BI; ++BI) {
1137 Instruction *New = BI->clone();
1138 New->setName(BI->getName());
1139 New->insertBefore(InsertPos: NewTerm->getIterator());
1140 New->cloneDebugInfoFrom(From: &*BI);
1141 ValueMapping[&*BI] = New;
1142
1143 // Remap operands to patch up intra-block references.
1144 for (unsigned i = 0, e = New->getNumOperands(); i != e; ++i)
1145 if (Instruction *Inst = dyn_cast<Instruction>(Val: New->getOperand(i))) {
1146 auto I = ValueMapping.find(Val: Inst);
1147 if (I != ValueMapping.end())
1148 New->setOperand(i, Val: I->second);
1149 }
1150
1151 // Remap debug variable operands.
1152 remapDebugVariable(Mapping&: ValueMapping, Inst: New);
1153 }
1154
1155 return NewBB;
1156}
1157
1158/// Create a clone of the scope \p SNANode in \p Domain, whose name is that of
1159/// the original scope extended with \p Ext.
1160static MDNode *createClonedScope(AliasScopeNode SNANode, MDNode *Domain,
1161 StringRef Ext, MDBuilder &MDB) {
1162 std::string Name;
1163 auto ScopeName = SNANode.getName();
1164 if (!ScopeName.empty())
1165 Name = (Twine(ScopeName) + ":" + Ext).str();
1166 else
1167 Name = std::string(Ext);
1168 return MDB.createAnonymousAliasScope(Domain, Name);
1169}
1170
1171void llvm::cloneNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
1172 DenseMap<MDNode *, MDNode *> &ClonedScopes,
1173 DenseMap<const MDNode *, MDNode *> &ClonedDomains,
1174 StringRef Ext, LLVMContext &Context) {
1175 MDBuilder MDB(Context);
1176
1177 for (MDNode *ScopeList : NoAliasDeclScopes) {
1178 for (const MDOperand &MDOp : ScopeList->operands()) {
1179 if (MDNode *MD = dyn_cast<MDNode>(Val: MDOp)) {
1180 AliasScopeNode SNANode(MD);
1181
1182 // A cloned scope has to go into a clone of its domain if that domain
1183 // has disjoint scopes, otherwise the duplicated operations would become
1184 // implicitly noalias with the original accesses (that are still using
1185 // the original scope). We take care to duplicate all scopes of the
1186 // domain in cloned code, see adaptNoAliasScope.
1187 const MDNode *Domain = SNANode.getDomain();
1188 MDNode *NewDomain = const_cast<MDNode *>(Domain);
1189 if (AliasScopeDomainNode(Domain).hasDisjointScopes()) {
1190 MDNode *&ClonedDomain = ClonedDomains[Domain];
1191 if (!ClonedDomain)
1192 ClonedDomain = MDB.createAnonymousAliasScopeDomain(
1193 Description: AliasScopeDomainNode(Domain).getDescription(),
1194 /*DisjointScopes=*/true);
1195 NewDomain = ClonedDomain;
1196 }
1197
1198 ClonedScopes.insert(KV: std::make_pair(
1199 x&: MD, y: createClonedScope(SNANode, Domain: NewDomain, Ext, MDB)));
1200 }
1201 }
1202 }
1203}
1204
1205void llvm::adaptNoAliasScopes(
1206 Instruction *I, DenseMap<MDNode *, MDNode *> &ClonedScopes,
1207 const DenseMap<const MDNode *, MDNode *> &ClonedDomains, StringRef Ext,
1208 LLVMContext &Context) {
1209 MDBuilder MDB(Context);
1210
1211 auto CloneScopeList = [&](const MDNode *ScopeList) -> MDNode * {
1212 bool NeedsReplacement = false;
1213 SmallVector<Metadata *, 8> NewScopeList;
1214 for (const MDOperand &MDOp : ScopeList->operands()) {
1215 if (MDNode *MD = dyn_cast<MDNode>(Val: MDOp)) {
1216 MDNode *NewMD = ClonedScopes.lookup(Val: MD);
1217 // Scopes must follow their domain if that domain is cloned, even if
1218 // there isn't an llvm.experimental.noalias.scope.decl for them.
1219 if (!NewMD && !ClonedDomains.empty()) {
1220 AliasScopeNode SNANode(MD);
1221 if (MDNode *NewDomain = ClonedDomains.lookup(Val: SNANode.getDomain())) {
1222 NewMD = createClonedScope(SNANode, Domain: NewDomain, Ext, MDB);
1223 ClonedScopes[MD] = NewMD;
1224 }
1225 }
1226 if (NewMD) {
1227 NewScopeList.push_back(Elt: NewMD);
1228 NeedsReplacement = true;
1229 continue;
1230 }
1231 NewScopeList.push_back(Elt: MD);
1232 }
1233 }
1234 if (NeedsReplacement)
1235 return MDNode::get(Context, MDs: NewScopeList);
1236 return nullptr;
1237 };
1238
1239 if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(Val: I))
1240 if (MDNode *NewScopeList = CloneScopeList(Decl->getScopeList()))
1241 Decl->setScopeList(NewScopeList);
1242
1243 auto replaceWhenNeeded = [&](unsigned MD_ID) {
1244 if (const MDNode *CSNoAlias = I->getMetadata(KindID: MD_ID))
1245 if (MDNode *NewScopeList = CloneScopeList(CSNoAlias))
1246 I->setMetadata(KindID: MD_ID, Node: NewScopeList);
1247 };
1248 replaceWhenNeeded(LLVMContext::MD_noalias);
1249 replaceWhenNeeded(LLVMContext::MD_alias_scope);
1250}
1251
1252void llvm::cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
1253 ArrayRef<BasicBlock *> NewBlocks,
1254 LLVMContext &Context, StringRef Ext) {
1255 if (NoAliasDeclScopes.empty())
1256 return;
1257
1258 DenseMap<MDNode *, MDNode *> ClonedScopes;
1259 DenseMap<const MDNode *, MDNode *> ClonedDomains;
1260 LLVM_DEBUG(dbgs() << "cloneAndAdaptNoAliasScopes: cloning "
1261 << NoAliasDeclScopes.size() << " node(s)\n");
1262
1263 cloneNoAliasScopes(NoAliasDeclScopes, ClonedScopes, ClonedDomains, Ext,
1264 Context);
1265 // Identify instructions using metadata that needs adaptation
1266 for (BasicBlock *NewBlock : NewBlocks)
1267 for (Instruction &I : *NewBlock)
1268 adaptNoAliasScopes(I: &I, ClonedScopes, ClonedDomains, Ext, Context);
1269}
1270
1271void llvm::cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
1272 Instruction *IStart, Instruction *IEnd,
1273 LLVMContext &Context, StringRef Ext) {
1274 if (NoAliasDeclScopes.empty())
1275 return;
1276
1277 DenseMap<MDNode *, MDNode *> ClonedScopes;
1278 DenseMap<const MDNode *, MDNode *> ClonedDomains;
1279 LLVM_DEBUG(dbgs() << "cloneAndAdaptNoAliasScopes: cloning "
1280 << NoAliasDeclScopes.size() << " node(s)\n");
1281
1282 cloneNoAliasScopes(NoAliasDeclScopes, ClonedScopes, ClonedDomains, Ext,
1283 Context);
1284 // Identify instructions using metadata that needs adaptation
1285 assert(IStart->getParent() == IEnd->getParent() && "different basic block ?");
1286 auto ItStart = IStart->getIterator();
1287 auto ItEnd = IEnd->getIterator();
1288 ++ItEnd; // IEnd is included, increment ItEnd to get the end of the range
1289 for (auto &I : llvm::make_range(x: ItStart, y: ItEnd))
1290 adaptNoAliasScopes(I: &I, ClonedScopes, ClonedDomains, Ext, Context);
1291}
1292
1293void llvm::identifyNoAliasScopesToClone(
1294 ArrayRef<BasicBlock *> BBs, SmallVectorImpl<MDNode *> &NoAliasDeclScopes) {
1295 for (BasicBlock *BB : BBs)
1296 for (Instruction &I : *BB)
1297 if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(Val: &I))
1298 NoAliasDeclScopes.push_back(Elt: Decl->getScopeList());
1299}
1300
1301void llvm::identifyNoAliasScopesToClone(
1302 BasicBlock::iterator Start, BasicBlock::iterator End,
1303 SmallVectorImpl<MDNode *> &NoAliasDeclScopes) {
1304 for (Instruction &I : make_range(x: Start, y: End))
1305 if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(Val: &I))
1306 NoAliasDeclScopes.push_back(Elt: Decl->getScopeList());
1307}
1308