1//===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass implements IR lowering for the llvm.memcpy, llvm.memmove,
10// llvm.memset, llvm.load.relative and llvm.objc.* intrinsics.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
15#include "llvm/Analysis/ObjCARCInstKind.h"
16#include "llvm/Analysis/ObjCARCUtil.h"
17#include "llvm/Analysis/TargetTransformInfo.h"
18#include "llvm/CodeGen/Passes.h"
19#include "llvm/CodeGen/TargetLowering.h"
20#include "llvm/CodeGen/TargetPassConfig.h"
21#include "llvm/IR/Function.h"
22#include "llvm/IR/IRBuilder.h"
23#include "llvm/IR/Instructions.h"
24#include "llvm/IR/IntrinsicInst.h"
25#include "llvm/IR/Module.h"
26#include "llvm/IR/Type.h"
27#include "llvm/InitializePasses.h"
28#include "llvm/Pass.h"
29#include "llvm/Support/Casting.h"
30#include "llvm/Target/TargetMachine.h"
31#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
32
33using namespace llvm;
34
35/// Threshold to leave statically sized memory intrinsic calls. Calls of known
36/// size larger than this will be expanded by the pass. Calls of unknown or
37/// lower size will be left for expansion in codegen.
38static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt(
39 "mem-intrinsic-expand-size",
40 cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(Val: -1),
41 cl::Hidden);
42
43namespace {
44
45struct PreISelIntrinsicLowering {
46 const TargetMachine &TM;
47 const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
48
49 /// If this is true, assume it's preferably to leave memory intrinsic calls
50 /// for replacement with a library call later. Otherwise this depends on
51 /// TargetLoweringInfo availability of the corresponding function.
52 const bool UseMemIntrinsicLibFunc;
53
54 explicit PreISelIntrinsicLowering(
55 const TargetMachine &TM_,
56 function_ref<TargetTransformInfo &(Function &)> LookupTTI_,
57 bool UseMemIntrinsicLibFunc_ = true)
58 : TM(TM_), LookupTTI(LookupTTI_),
59 UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {}
60
61 static bool shouldExpandMemIntrinsicWithSize(Value *Size,
62 const TargetTransformInfo &TTI);
63 bool expandMemIntrinsicUses(Function &F) const;
64 bool lowerIntrinsics(Module &M) const;
65};
66
67} // namespace
68
69static bool lowerLoadRelative(Function &F) {
70 if (F.use_empty())
71 return false;
72
73 bool Changed = false;
74 Type *Int32Ty = Type::getInt32Ty(C&: F.getContext());
75
76 for (Use &U : llvm::make_early_inc_range(Range: F.uses())) {
77 auto CI = dyn_cast<CallInst>(Val: U.getUser());
78 if (!CI || CI->getCalledOperand() != &F)
79 continue;
80
81 IRBuilder<> B(CI);
82 Value *OffsetPtr =
83 B.CreatePtrAdd(Ptr: CI->getArgOperand(i: 0), Offset: CI->getArgOperand(i: 1));
84 Value *OffsetI32 = B.CreateAlignedLoad(Ty: Int32Ty, Ptr: OffsetPtr, Align: Align(4));
85
86 Value *ResultPtr = B.CreatePtrAdd(Ptr: CI->getArgOperand(i: 0), Offset: OffsetI32);
87
88 CI->replaceAllUsesWith(V: ResultPtr);
89 CI->eraseFromParent();
90 Changed = true;
91 }
92
93 return Changed;
94}
95
96// ObjCARC has knowledge about whether an obj-c runtime function needs to be
97// always tail-called or never tail-called.
98static CallInst::TailCallKind getOverridingTailCallKind(const Function &F) {
99 objcarc::ARCInstKind Kind = objcarc::GetFunctionClass(F: &F);
100 if (objcarc::IsAlwaysTail(Class: Kind))
101 return CallInst::TCK_Tail;
102 else if (objcarc::IsNeverTail(Class: Kind))
103 return CallInst::TCK_NoTail;
104 return CallInst::TCK_None;
105}
106
107static bool lowerObjCCall(Function &F, const char *NewFn,
108 bool setNonLazyBind = false) {
109 assert(IntrinsicInst::mayLowerToFunctionCall(F.getIntrinsicID()) &&
110 "Pre-ISel intrinsics do lower into regular function calls");
111 if (F.use_empty())
112 return false;
113
114 // If we haven't already looked up this function, check to see if the
115 // program already contains a function with this name.
116 Module *M = F.getParent();
117 FunctionCallee FCache = M->getOrInsertFunction(Name: NewFn, T: F.getFunctionType());
118
119 if (Function *Fn = dyn_cast<Function>(Val: FCache.getCallee())) {
120 Fn->setLinkage(F.getLinkage());
121 if (setNonLazyBind && !Fn->isWeakForLinker()) {
122 // If we have Native ARC, set nonlazybind attribute for these APIs for
123 // performance.
124 Fn->addFnAttr(Kind: Attribute::NonLazyBind);
125 }
126 }
127
128 CallInst::TailCallKind OverridingTCK = getOverridingTailCallKind(F);
129
130 for (Use &U : llvm::make_early_inc_range(Range: F.uses())) {
131 auto *CB = cast<CallBase>(Val: U.getUser());
132
133 if (CB->getCalledFunction() != &F) {
134 objcarc::ARCInstKind Kind = objcarc::getAttachedARCFunctionKind(CB);
135 (void)Kind;
136 assert((Kind == objcarc::ARCInstKind::RetainRV ||
137 Kind == objcarc::ARCInstKind::UnsafeClaimRV) &&
138 "use expected to be the argument of operand bundle "
139 "\"clang.arc.attachedcall\"");
140 U.set(FCache.getCallee());
141 continue;
142 }
143
144 auto *CI = cast<CallInst>(Val: CB);
145 assert(CI->getCalledFunction() && "Cannot lower an indirect call!");
146
147 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
148 SmallVector<Value *, 8> Args(CI->args());
149 SmallVector<llvm::OperandBundleDef, 1> BundleList;
150 CI->getOperandBundlesAsDefs(Defs&: BundleList);
151 CallInst *NewCI = Builder.CreateCall(Callee: FCache, Args, OpBundles: BundleList);
152 NewCI->setName(CI->getName());
153
154 // Try to set the most appropriate TailCallKind based on both the current
155 // attributes and the ones that we could get from ObjCARC's special
156 // knowledge of the runtime functions.
157 //
158 // std::max respects both requirements of notail and tail here:
159 // * notail on either the call or from ObjCARC becomes notail
160 // * tail on either side is stronger than none, but not notail
161 CallInst::TailCallKind TCK = CI->getTailCallKind();
162 NewCI->setTailCallKind(std::max(a: TCK, b: OverridingTCK));
163
164 // Transfer the 'returned' attribute from the intrinsic to the call site.
165 // By applying this only to intrinsic call sites, we avoid applying it to
166 // non-ARC explicit calls to things like objc_retain which have not been
167 // auto-upgraded to use the intrinsics.
168 unsigned Index;
169 if (F.getAttributes().hasAttrSomewhere(Kind: Attribute::Returned, Index: &Index) &&
170 Index)
171 NewCI->addParamAttr(ArgNo: Index - AttributeList::FirstArgIndex,
172 Kind: Attribute::Returned);
173
174 if (!CI->use_empty())
175 CI->replaceAllUsesWith(V: NewCI);
176 CI->eraseFromParent();
177 }
178
179 return true;
180}
181
182// TODO: Should refine based on estimated number of accesses (e.g. does it
183// require splitting based on alignment)
184bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(
185 Value *Size, const TargetTransformInfo &TTI) {
186 ConstantInt *CI = dyn_cast<ConstantInt>(Val: Size);
187 if (!CI)
188 return true;
189 uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences()
190 ? MemIntrinsicExpandSizeThresholdOpt
191 : TTI.getMaxMemIntrinsicInlineSizeThreshold();
192 uint64_t SizeVal = CI->getZExtValue();
193
194 // Treat a threshold of 0 as a special case to force expansion of all
195 // intrinsics, including size 0.
196 return SizeVal > Threshold || Threshold == 0;
197}
198
199static bool canEmitLibcall(const TargetMachine &TM, Function *F,
200 RTLIB::Libcall LC) {
201 // TODO: Should this consider the address space of the memcpy?
202 const TargetLowering *TLI = TM.getSubtargetImpl(*F)->getTargetLowering();
203 return TLI->getLibcallName(Call: LC) != nullptr;
204}
205
206// TODO: Handle atomic memcpy and memcpy.inline
207// TODO: Pass ScalarEvolution
208bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const {
209 Intrinsic::ID ID = F.getIntrinsicID();
210 bool Changed = false;
211
212 for (User *U : llvm::make_early_inc_range(Range: F.users())) {
213 Instruction *Inst = cast<Instruction>(Val: U);
214
215 switch (ID) {
216 case Intrinsic::memcpy: {
217 auto *Memcpy = cast<MemCpyInst>(Val: Inst);
218 Function *ParentFunc = Memcpy->getFunction();
219 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
220 if (shouldExpandMemIntrinsicWithSize(Size: Memcpy->getLength(), TTI)) {
221 if (UseMemIntrinsicLibFunc &&
222 canEmitLibcall(TM, F: ParentFunc, LC: RTLIB::MEMCPY))
223 break;
224
225 // TODO: For optsize, emit the loop into a separate function
226 expandMemCpyAsLoop(MemCpy: Memcpy, TTI);
227 Changed = true;
228 Memcpy->eraseFromParent();
229 }
230
231 break;
232 }
233 case Intrinsic::memcpy_inline: {
234 // Only expand llvm.memcpy.inline with non-constant length in this
235 // codepath, leaving the current SelectionDAG expansion for constant
236 // length memcpy intrinsics undisturbed.
237 auto *Memcpy = cast<MemCpyInlineInst>(Val: Inst);
238 if (isa<ConstantInt>(Val: Memcpy->getLength()))
239 break;
240
241 Function *ParentFunc = Memcpy->getFunction();
242 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
243 expandMemCpyAsLoop(MemCpy: Memcpy, TTI);
244 Changed = true;
245 Memcpy->eraseFromParent();
246 break;
247 }
248 case Intrinsic::memmove: {
249 auto *Memmove = cast<MemMoveInst>(Val: Inst);
250 Function *ParentFunc = Memmove->getFunction();
251 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
252 if (shouldExpandMemIntrinsicWithSize(Size: Memmove->getLength(), TTI)) {
253 if (UseMemIntrinsicLibFunc &&
254 canEmitLibcall(TM, F: ParentFunc, LC: RTLIB::MEMMOVE))
255 break;
256
257 if (expandMemMoveAsLoop(MemMove: Memmove, TTI)) {
258 Changed = true;
259 Memmove->eraseFromParent();
260 }
261 }
262
263 break;
264 }
265 case Intrinsic::memset: {
266 auto *Memset = cast<MemSetInst>(Val: Inst);
267 Function *ParentFunc = Memset->getFunction();
268 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
269 if (shouldExpandMemIntrinsicWithSize(Size: Memset->getLength(), TTI)) {
270 if (UseMemIntrinsicLibFunc &&
271 canEmitLibcall(TM, F: ParentFunc, LC: RTLIB::MEMSET))
272 break;
273
274 expandMemSetAsLoop(MemSet: Memset);
275 Changed = true;
276 Memset->eraseFromParent();
277 }
278
279 break;
280 }
281 case Intrinsic::memset_inline: {
282 // Only expand llvm.memset.inline with non-constant length in this
283 // codepath, leaving the current SelectionDAG expansion for constant
284 // length memset intrinsics undisturbed.
285 auto *Memset = cast<MemSetInlineInst>(Val: Inst);
286 if (isa<ConstantInt>(Val: Memset->getLength()))
287 break;
288
289 expandMemSetAsLoop(MemSet: Memset);
290 Changed = true;
291 Memset->eraseFromParent();
292 break;
293 }
294 default:
295 llvm_unreachable("unhandled intrinsic");
296 }
297 }
298
299 return Changed;
300}
301
302bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
303 bool Changed = false;
304 for (Function &F : M) {
305 switch (F.getIntrinsicID()) {
306 default:
307 break;
308 case Intrinsic::memcpy:
309 case Intrinsic::memcpy_inline:
310 case Intrinsic::memmove:
311 case Intrinsic::memset:
312 case Intrinsic::memset_inline:
313 Changed |= expandMemIntrinsicUses(F);
314 break;
315 case Intrinsic::load_relative:
316 Changed |= lowerLoadRelative(F);
317 break;
318 case Intrinsic::objc_autorelease:
319 Changed |= lowerObjCCall(F, NewFn: "objc_autorelease");
320 break;
321 case Intrinsic::objc_autoreleasePoolPop:
322 Changed |= lowerObjCCall(F, NewFn: "objc_autoreleasePoolPop");
323 break;
324 case Intrinsic::objc_autoreleasePoolPush:
325 Changed |= lowerObjCCall(F, NewFn: "objc_autoreleasePoolPush");
326 break;
327 case Intrinsic::objc_autoreleaseReturnValue:
328 Changed |= lowerObjCCall(F, NewFn: "objc_autoreleaseReturnValue");
329 break;
330 case Intrinsic::objc_copyWeak:
331 Changed |= lowerObjCCall(F, NewFn: "objc_copyWeak");
332 break;
333 case Intrinsic::objc_destroyWeak:
334 Changed |= lowerObjCCall(F, NewFn: "objc_destroyWeak");
335 break;
336 case Intrinsic::objc_initWeak:
337 Changed |= lowerObjCCall(F, NewFn: "objc_initWeak");
338 break;
339 case Intrinsic::objc_loadWeak:
340 Changed |= lowerObjCCall(F, NewFn: "objc_loadWeak");
341 break;
342 case Intrinsic::objc_loadWeakRetained:
343 Changed |= lowerObjCCall(F, NewFn: "objc_loadWeakRetained");
344 break;
345 case Intrinsic::objc_moveWeak:
346 Changed |= lowerObjCCall(F, NewFn: "objc_moveWeak");
347 break;
348 case Intrinsic::objc_release:
349 Changed |= lowerObjCCall(F, NewFn: "objc_release", setNonLazyBind: true);
350 break;
351 case Intrinsic::objc_retain:
352 Changed |= lowerObjCCall(F, NewFn: "objc_retain", setNonLazyBind: true);
353 break;
354 case Intrinsic::objc_retainAutorelease:
355 Changed |= lowerObjCCall(F, NewFn: "objc_retainAutorelease");
356 break;
357 case Intrinsic::objc_retainAutoreleaseReturnValue:
358 Changed |= lowerObjCCall(F, NewFn: "objc_retainAutoreleaseReturnValue");
359 break;
360 case Intrinsic::objc_retainAutoreleasedReturnValue:
361 Changed |= lowerObjCCall(F, NewFn: "objc_retainAutoreleasedReturnValue");
362 break;
363 case Intrinsic::objc_retainBlock:
364 Changed |= lowerObjCCall(F, NewFn: "objc_retainBlock");
365 break;
366 case Intrinsic::objc_storeStrong:
367 Changed |= lowerObjCCall(F, NewFn: "objc_storeStrong");
368 break;
369 case Intrinsic::objc_storeWeak:
370 Changed |= lowerObjCCall(F, NewFn: "objc_storeWeak");
371 break;
372 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
373 Changed |= lowerObjCCall(F, NewFn: "objc_unsafeClaimAutoreleasedReturnValue");
374 break;
375 case Intrinsic::objc_retainedObject:
376 Changed |= lowerObjCCall(F, NewFn: "objc_retainedObject");
377 break;
378 case Intrinsic::objc_unretainedObject:
379 Changed |= lowerObjCCall(F, NewFn: "objc_unretainedObject");
380 break;
381 case Intrinsic::objc_unretainedPointer:
382 Changed |= lowerObjCCall(F, NewFn: "objc_unretainedPointer");
383 break;
384 case Intrinsic::objc_retain_autorelease:
385 Changed |= lowerObjCCall(F, NewFn: "objc_retain_autorelease");
386 break;
387 case Intrinsic::objc_sync_enter:
388 Changed |= lowerObjCCall(F, NewFn: "objc_sync_enter");
389 break;
390 case Intrinsic::objc_sync_exit:
391 Changed |= lowerObjCCall(F, NewFn: "objc_sync_exit");
392 break;
393 }
394 }
395 return Changed;
396}
397
398namespace {
399
400class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
401public:
402 static char ID;
403
404 PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}
405
406 void getAnalysisUsage(AnalysisUsage &AU) const override {
407 AU.addRequired<TargetTransformInfoWrapperPass>();
408 AU.addRequired<TargetPassConfig>();
409 }
410
411 bool runOnModule(Module &M) override {
412 auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {
413 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
414 };
415
416 const auto &TM = getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
417 PreISelIntrinsicLowering Lowering(TM, LookupTTI);
418 return Lowering.lowerIntrinsics(M);
419 }
420};
421
422} // end anonymous namespace
423
424char PreISelIntrinsicLoweringLegacyPass::ID;
425
426INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass,
427 "pre-isel-intrinsic-lowering",
428 "Pre-ISel Intrinsic Lowering", false, false)
429INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
430INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
431INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
432 "pre-isel-intrinsic-lowering",
433 "Pre-ISel Intrinsic Lowering", false, false)
434
435ModulePass *llvm::createPreISelIntrinsicLoweringPass() {
436 return new PreISelIntrinsicLoweringLegacyPass();
437}
438
439PreservedAnalyses PreISelIntrinsicLoweringPass::run(Module &M,
440 ModuleAnalysisManager &AM) {
441 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
442
443 auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {
444 return FAM.getResult<TargetIRAnalysis>(IR&: F);
445 };
446
447 PreISelIntrinsicLowering Lowering(TM, LookupTTI);
448 if (!Lowering.lowerIntrinsics(M))
449 return PreservedAnalyses::all();
450 else
451 return PreservedAnalyses::none();
452}
453