1//===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass implements IR lowering for the llvm.memcpy, llvm.memmove,
10// llvm.memset, llvm.load.relative and llvm.objc.* intrinsics.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
15#include "CodeGenOptions.h"
16#include "llvm/ADT/STLExtras.h"
17#include "llvm/Analysis/AssumptionCache.h"
18#include "llvm/Analysis/ObjCARCInstKind.h"
19#include "llvm/Analysis/ObjCARCUtil.h"
20#include "llvm/Analysis/TargetLibraryInfo.h"
21#include "llvm/Analysis/TargetTransformInfo.h"
22#include "llvm/Analysis/ValueTracking.h"
23#include "llvm/CodeGen/ExpandVectorPredication.h"
24#include "llvm/CodeGen/LibcallLoweringInfo.h"
25#include "llvm/CodeGen/Passes.h"
26#include "llvm/CodeGen/RuntimeLibcallUtil.h"
27#include "llvm/CodeGen/TargetLowering.h"
28#include "llvm/CodeGen/TargetPassConfig.h"
29#include "llvm/IR/ConstantRange.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/GlobalValue.h"
32#include "llvm/IR/IRBuilder.h"
33#include "llvm/IR/Instructions.h"
34#include "llvm/IR/IntrinsicInst.h"
35#include "llvm/IR/Metadata.h"
36#include "llvm/IR/Module.h"
37#include "llvm/IR/ProfDataUtils.h"
38#include "llvm/IR/RuntimeLibcalls.h"
39#include "llvm/IR/Type.h"
40#include "llvm/IR/Use.h"
41#include "llvm/InitializePasses.h"
42#include "llvm/Pass.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Target/TargetMachine.h"
45#include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
46#include "llvm/Transforms/Utils/BasicBlockUtils.h"
47#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
48#include "llvm/Transforms/Utils/LowerVectorIntrinsics.h"
49
50using namespace llvm;
51
52#define DEBUG_TYPE "pre-isel-intrinsic-lowering"
53
54namespace {
55
56struct PreISelIntrinsicLowering {
57 const CodeGenOptions &Opts;
58 const TargetMachine &TM;
59 const ModuleLibcallLoweringInfo &ModuleLibcalls;
60 const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
61 const function_ref<TargetLibraryInfo &(Function &)> LookupTLI;
62 const function_ref<AssumptionCache &(Function &)> LookupAC;
63
64 /// If this is true, assume it's preferably to leave memory intrinsic calls
65 /// for replacement with a library call later. Otherwise this depends on
66 /// TargetLoweringInfo availability of the corresponding function.
67 const bool UseMemIntrinsicLibFunc;
68
69 explicit PreISelIntrinsicLowering(
70 const CodeGenOptions &Opts_, const TargetMachine &TM_,
71 const ModuleLibcallLoweringInfo &ModuleLibcalls_,
72 function_ref<TargetTransformInfo &(Function &)> LookupTTI_,
73 function_ref<TargetLibraryInfo &(Function &)> LookupTLI_,
74 function_ref<AssumptionCache &(Function &)> LookupAC_,
75 bool UseMemIntrinsicLibFunc_ = true)
76 : Opts(Opts_), TM(TM_), ModuleLibcalls(ModuleLibcalls_),
77 LookupTTI(LookupTTI_), LookupTLI(LookupTLI_), LookupAC(LookupAC_),
78 UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {}
79
80 bool shouldExpandMemIntrinsicWithSize(Value *Size,
81 const TargetTransformInfo &TTI) const;
82 bool tryExpandBoundedMemSet(MemSetInst *Memset,
83 const TargetTransformInfo &TTI) const;
84 bool
85 expandMemIntrinsicUses(Function &F,
86 DenseMap<Constant *, GlobalVariable *> &CMap) const;
87 bool lowerIntrinsics(Module &M) const;
88};
89
90} // namespace
91
92template <class T> static bool forEachCall(Function &Intrin, T Callback) {
93 // Lowering all intrinsics in a function will delete multiple uses, so we
94 // can't use an early-inc-range. In case some remain, we don't want to look
95 // at them again. Unfortunately, Value::UseList is private, so we can't use a
96 // simple Use**. If LastUse is null, the next use to consider is
97 // Intrin.use_begin(), otherwise it's LastUse->getNext().
98 Use *LastUse = nullptr;
99 bool Changed = false;
100 while (!Intrin.use_empty() && (!LastUse || LastUse->getNext())) {
101 Use *U = LastUse ? LastUse->getNext() : &*Intrin.use_begin();
102 bool Removed = false;
103 // An intrinsic cannot have its address taken, so it cannot be an argument
104 // operand. It might be used as operand in debug metadata, though.
105 if (auto CI = dyn_cast<CallInst>(Val: U->getUser()))
106 Changed |= Removed = Callback(CI);
107 if (!Removed)
108 LastUse = U;
109 }
110 return Changed;
111}
112
113static bool lowerLoadRelative(Function &F) {
114 if (F.use_empty())
115 return false;
116
117 bool Changed = false;
118 Type *Int32Ty = Type::getInt32Ty(C&: F.getContext());
119
120 for (Use &U : llvm::make_early_inc_range(Range: F.uses())) {
121 auto CI = dyn_cast<CallInst>(Val: U.getUser());
122 if (!CI || CI->getCalledOperand() != &F)
123 continue;
124
125 IRBuilder<> B(CI);
126 Value *Ptr = CI->getArgOperand(i: 0);
127 Value *OffsetPtr = B.CreatePtrAddWithSExtOrTrunc(Ptr, Offset: CI->getArgOperand(i: 1));
128 Value *OffsetI32 = B.CreateAlignedLoad(Ty: Int32Ty, Ptr: OffsetPtr, Align: Align(4));
129
130 Value *ResultPtr = B.CreatePtrAddWithSExtOrTrunc(Ptr, Offset: OffsetI32);
131
132 CI->replaceAllUsesWith(V: ResultPtr);
133 CI->eraseFromParent();
134 Changed = true;
135 }
136
137 return Changed;
138}
139
140/// Lower @llvm.can.load.speculatively using target-specific expansion.
141/// Targets may provide their own expansion via
142/// TargetLowering::emitCanLoadSpeculatively; the default expansion
143/// conservatively returns false.
144static bool lowerCanLoadSpeculatively(Function &F, const TargetMachine &TM) {
145 return forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
146 const TargetLowering *TLI =
147 TM.getSubtargetImpl(*CI->getFunction())->getTargetLowering();
148
149 IRBuilder<> Builder(CI);
150 // A null result means the target cannot answer; lower to false.
151 Value *Result = TLI->emitCanLoadSpeculatively(Builder, Ptr: CI->getArgOperand(i: 0),
152 Size: CI->getArgOperand(i: 1));
153 if (!Result)
154 Result = Builder.getFalse();
155
156 CI->replaceAllUsesWith(V: Result);
157 CI->eraseFromParent();
158 return true;
159 });
160}
161
162// ObjCARC has knowledge about whether an obj-c runtime function needs to be
163// always tail-called or never tail-called.
164static CallInst::TailCallKind getOverridingTailCallKind(const Function &F) {
165 objcarc::ARCInstKind Kind = objcarc::GetFunctionClass(F: &F);
166 if (objcarc::IsAlwaysTail(Class: Kind))
167 return CallInst::TCK_Tail;
168 else if (objcarc::IsNeverTail(Class: Kind))
169 return CallInst::TCK_NoTail;
170 return CallInst::TCK_None;
171}
172
173static bool lowerObjCCall(Function &F, RTLIB::LibcallImpl NewFn,
174 bool setNonLazyBind = false) {
175 assert(IntrinsicInst::mayLowerToFunctionCall(F.getIntrinsicID()) &&
176 "Pre-ISel intrinsics do lower into regular function calls");
177 if (F.use_empty())
178 return false;
179
180 // FIXME: When RuntimeLibcalls is an analysis, check if the function is really
181 // supported, and go through RTLIB::Libcall.
182 StringRef NewFnName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: NewFn);
183
184 // If we haven't already looked up this function, check to see if the
185 // program already contains a function with this name.
186 Module *M = F.getParent();
187 FunctionCallee FCache =
188 M->getOrInsertFunction(Name: NewFnName, T: F.getFunctionType());
189
190 if (Function *Fn = dyn_cast<Function>(Val: FCache.getCallee())) {
191 Fn->setLinkage(F.getLinkage());
192 if (setNonLazyBind && !Fn->isWeakForLinker()) {
193 // If we have Native ARC, set nonlazybind attribute for these APIs for
194 // performance.
195 Fn->addFnAttr(Kind: Attribute::NonLazyBind);
196 }
197 }
198
199 CallInst::TailCallKind OverridingTCK = getOverridingTailCallKind(F);
200
201 for (Use &U : llvm::make_early_inc_range(Range: F.uses())) {
202 auto *CB = cast<CallBase>(Val: U.getUser());
203
204 if (CB->getCalledFunction() != &F) {
205 assert(objcarc::getAttachedARCFunction(CB) == &F &&
206 "use expected to be the argument of operand bundle "
207 "\"clang.arc.attachedcall\"");
208 U.set(FCache.getCallee());
209 continue;
210 }
211
212 auto *CI = cast<CallInst>(Val: CB);
213 assert(CI->getCalledFunction() && "Cannot lower an indirect call!");
214
215 IRBuilder<> Builder(CI->getIterator());
216 SmallVector<Value *, 8> Args(CI->args());
217 SmallVector<llvm::OperandBundleDef, 1> BundleList;
218 CI->getOperandBundlesAsDefs(Defs&: BundleList);
219 CallInst *NewCI = Builder.CreateCall(Callee: FCache, Args, OpBundles: BundleList);
220 NewCI->setName(CI->getName());
221
222 // Try to set the most appropriate TailCallKind based on both the current
223 // attributes and the ones that we could get from ObjCARC's special
224 // knowledge of the runtime functions.
225 //
226 // std::max respects both requirements of notail and tail here:
227 // * notail on either the call or from ObjCARC becomes notail
228 // * tail on either side is stronger than none, but not notail
229 CallInst::TailCallKind TCK = CI->getTailCallKind();
230 NewCI->setTailCallKind(std::max(a: TCK, b: OverridingTCK));
231
232 // Transfer the 'returned' attribute from the intrinsic to the call site.
233 // By applying this only to intrinsic call sites, we avoid applying it to
234 // non-ARC explicit calls to things like objc_retain which have not been
235 // auto-upgraded to use the intrinsics.
236 unsigned Index;
237 if (F.getAttributes().hasAttrSomewhere(Kind: Attribute::Returned, Index: &Index) &&
238 Index)
239 NewCI->addParamAttr(ArgNo: Index - AttributeList::FirstArgIndex,
240 Kind: Attribute::Returned);
241
242 if (!CI->use_empty())
243 CI->replaceAllUsesWith(V: NewCI);
244 CI->eraseFromParent();
245 }
246
247 return true;
248}
249
250// TODO: Should refine based on estimated number of accesses (e.g. does it
251// require splitting based on alignment)
252bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(
253 Value *Size, const TargetTransformInfo &TTI) const {
254 ConstantInt *CI = dyn_cast<ConstantInt>(Val: Size);
255 if (!CI)
256 return true;
257 uint64_t Threshold = Opts.mem_intrinsic_expand_size.value_or(
258 u: TTI.getMaxMemIntrinsicInlineSizeThreshold());
259 uint64_t SizeVal = CI->getZExtValue();
260
261 // Treat a threshold of 0 as a special case to force expansion of all
262 // intrinsics, including size 0.
263 return SizeVal > Threshold || Threshold == 0;
264}
265
266static bool canEmitLibcall(const ModuleLibcallLoweringInfo &ModuleLowering,
267 const TargetMachine &TM, Function *F,
268 RTLIB::Libcall LC) {
269 // TODO: Should this consider the address space of the memcpy?
270 const LibcallLoweringInfo &Lowering =
271 getLibcallLowering(ModuleInfo: ModuleLowering, Subtarget: *TM.getSubtargetImpl(*F));
272 return Lowering.getLibcallImpl(Call: LC) != RTLIB::Unsupported;
273}
274
275static bool canEmitMemcpy(const ModuleLibcallLoweringInfo &ModuleLowering,
276 const TargetMachine &TM, Function *F) {
277 // TODO: Should this consider the address space of the memcpy?
278 const LibcallLoweringInfo &Lowering =
279 getLibcallLowering(ModuleInfo: ModuleLowering, Subtarget: *TM.getSubtargetImpl(*F));
280 return Lowering.getMemcpyImpl() != RTLIB::Unsupported;
281}
282
283// Expand a memset whose length is not constant but is known to be small, which
284// is faster than calling the library.
285bool PreISelIntrinsicLowering::tryExpandBoundedMemSet(
286 MemSetInst *Memset, const TargetTransformInfo &TTI) const {
287 // This pass runs even at -O0, so skip if e.g. llc -O0 is used.
288 if (TM.getOptLevel() == CodeGenOptLevel::None)
289 return false;
290
291 Value *Len = Memset->getLength();
292 Function *F = Memset->getFunction();
293 if (isa<Constant>(Val: Len) || Memset->isVolatile() || F->hasOptNone() ||
294 F->hasMinSize())
295 return false;
296 uint64_t Threshold = Opts.bounded_memset_inline_size.value_or(
297 u: TTI.getMaxBoundedMemSetInlineSize());
298 if (!Threshold)
299 return false;
300
301 // Bounds from (1) assumptions, e.g. __builtin_assume(len <= 32)
302 // and (2) known bits and ranges on instructions themselves, and...
303 SimplifyQuery SQ(Memset->getDataLayout(), Memset);
304 SQ.AC = &LookupAC(*F);
305 KnownBits Known = computeKnownBits(V: Len, Q: SQ);
306 ConstantRange CR = computeConstantRangeIncludingKnownBits(
307 V: {Len, Known}, /*ForSigned=*/false, SQ);
308
309 // ... (3) range attr on the memset.
310 Attribute RangeAttr = Memset->getParamAttr(
311 ArgNo: Memset->getLengthUse().getOperandNo(), Kind: Attribute::Range);
312 if (RangeAttr.isValid())
313 CR = CR.intersectWith(CR: RangeAttr.getRange());
314
315 if (CR.isEmptySet() || CR.getUnsignedMax().ugt(RHS: Threshold))
316 return false;
317
318 expandBoundedMemSet(MemSet: Memset, MinLen: CR.getUnsignedMin().getZExtValue(),
319 MaxLen: CR.getUnsignedMax().getZExtValue(), LenKnown: Known);
320 return true;
321}
322
323// Return a value appropriate for use with the memset_pattern16 libcall, if
324// possible and if we know how. (Adapted from equivalent helper in
325// LoopIdiomRecognize).
326static Constant *
327getMemSetPattern16Value(MemSetPatternInst *Inst,
328 const RTLIB::RuntimeLibcallsInfo &RTLCI) {
329 // TODO: This could check for UndefValue because it can be merged into any
330 // other valid pattern.
331
332 // Don't emit libcalls if a non-default address space is being used.
333 if (Inst->getRawDest()->getType()->getPointerAddressSpace() != 0)
334 return nullptr;
335
336 Value *V = Inst->getValue();
337 Type *VTy = V->getType();
338 const DataLayout &DL = Inst->getDataLayout();
339
340 if (!RTLCI.isAvailable(Impl: RTLIB::impl_memset_pattern16))
341 return nullptr;
342
343 // If the value isn't a constant, we can't promote it to being in a constant
344 // array. We could theoretically do a store to an alloca or something, but
345 // that doesn't seem worthwhile.
346 Constant *C = dyn_cast<Constant>(Val: V);
347 if (!C || isa<ConstantExpr>(Val: C))
348 return nullptr;
349
350 // Only handle simple values that are a power of two bytes in size.
351 uint64_t Size = DL.getTypeSizeInBits(Ty: VTy);
352 if (!DL.typeSizeEqualsStoreSize(Ty: VTy) || !isPowerOf2_64(Value: Size))
353 return nullptr;
354
355 // Don't care enough about darwin/ppc to implement this.
356 if (DL.isBigEndian())
357 return nullptr;
358
359 // Convert to size in bytes.
360 Size /= 8;
361
362 // TODO: If CI is larger than 16-bytes, we can try slicing it in half to see
363 // if the top and bottom are the same (e.g. for vectors and large integers).
364 if (Size > 16)
365 return nullptr;
366
367 // If the constant is exactly 16 bytes, just use it.
368 if (Size == 16)
369 return C;
370
371 // Otherwise, we'll use an array of the constants.
372 uint64_t ArraySize = 16 / Size;
373 ArrayType *AT = ArrayType::get(ElementType: V->getType(), NumElements: ArraySize);
374 return ConstantArray::get(T: AT, V: std::vector<Constant *>(ArraySize, C));
375}
376
377// TODO: Handle atomic memcpy and memcpy.inline
378// TODO: Pass ScalarEvolution
379bool PreISelIntrinsicLowering::expandMemIntrinsicUses(
380 Function &F, DenseMap<Constant *, GlobalVariable *> &CMap) const {
381 Intrinsic::ID ID = F.getIntrinsicID();
382 bool Changed = false;
383
384 for (User *U : llvm::make_early_inc_range(Range: F.users())) {
385 Instruction *Inst = cast<Instruction>(Val: U);
386
387 switch (ID) {
388 case Intrinsic::memcpy: {
389 auto *Memcpy = cast<MemCpyInst>(Val: Inst);
390 Function *ParentFunc = Memcpy->getFunction();
391 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
392 if (shouldExpandMemIntrinsicWithSize(Size: Memcpy->getLength(), TTI)) {
393 if (UseMemIntrinsicLibFunc &&
394 canEmitMemcpy(ModuleLowering: ModuleLibcalls, TM, F: ParentFunc))
395 break;
396
397 // TODO: For optsize, emit the loop into a separate function
398 expandMemCpyAsLoop(MemCpy: Memcpy, TTI);
399 Changed = true;
400 Memcpy->eraseFromParent();
401 }
402
403 break;
404 }
405 case Intrinsic::memcpy_inline: {
406 // Only expand llvm.memcpy.inline with non-constant length in this
407 // codepath, leaving the current SelectionDAG expansion for constant
408 // length memcpy intrinsics undisturbed.
409 auto *Memcpy = cast<MemCpyInst>(Val: Inst);
410 if (isa<ConstantInt>(Val: Memcpy->getLength()))
411 break;
412
413 Function *ParentFunc = Memcpy->getFunction();
414 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
415 expandMemCpyAsLoop(MemCpy: Memcpy, TTI);
416 Changed = true;
417 Memcpy->eraseFromParent();
418 break;
419 }
420 case Intrinsic::memmove: {
421 auto *Memmove = cast<MemMoveInst>(Val: Inst);
422 Function *ParentFunc = Memmove->getFunction();
423 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
424 if (shouldExpandMemIntrinsicWithSize(Size: Memmove->getLength(), TTI)) {
425 if (UseMemIntrinsicLibFunc &&
426 canEmitLibcall(ModuleLowering: ModuleLibcalls, TM, F: ParentFunc, LC: RTLIB::MEMMOVE))
427 break;
428
429 if (expandMemMoveAsLoop(MemMove: Memmove, TTI)) {
430 Changed = true;
431 Memmove->eraseFromParent();
432 }
433 }
434
435 break;
436 }
437 case Intrinsic::memset: {
438 auto *Memset = cast<MemSetInst>(Val: Inst);
439 Function *ParentFunc = Memset->getFunction();
440 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
441 if (tryExpandBoundedMemSet(Memset, TTI)) {
442 Changed = true;
443 Memset->eraseFromParent();
444 break;
445 }
446 if (shouldExpandMemIntrinsicWithSize(Size: Memset->getLength(), TTI)) {
447 if (UseMemIntrinsicLibFunc &&
448 canEmitLibcall(ModuleLowering: ModuleLibcalls, TM, F: ParentFunc, LC: RTLIB::MEMSET))
449 break;
450
451 expandMemSetAsLoop(MemSet: Memset, TTI);
452 Changed = true;
453 Memset->eraseFromParent();
454 }
455
456 break;
457 }
458 case Intrinsic::memset_inline: {
459 // Only expand llvm.memset.inline with non-constant length in this
460 // codepath, leaving the current SelectionDAG expansion for constant
461 // length memset intrinsics undisturbed.
462 auto *Memset = cast<MemSetInst>(Val: Inst);
463 if (isa<ConstantInt>(Val: Memset->getLength()))
464 break;
465
466 Function *ParentFunc = Memset->getFunction();
467 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
468 expandMemSetAsLoop(MemSet: Memset, TTI);
469 Changed = true;
470 Memset->eraseFromParent();
471 break;
472 }
473 case Intrinsic::experimental_memset_pattern: {
474 auto *Memset = cast<MemSetPatternInst>(Val: Inst);
475 Function *ParentFunc = Memset->getFunction();
476 const RTLIB::RuntimeLibcallsInfo &RTLCI =
477 ModuleLibcalls.getRuntimeLibcallsInfo();
478 Constant *PatternValue = getMemSetPattern16Value(Inst: Memset, RTLCI);
479 if (!PatternValue) {
480 // If it isn't possible to emit a memset_pattern16 libcall, expand to
481 // a loop instead.
482 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
483 expandMemSetPatternAsLoop(MemSet: Memset, TTI);
484 Changed = true;
485 Memset->eraseFromParent();
486 break;
487 }
488 // FIXME: There is currently no profitability calculation for emitting
489 // the libcall vs expanding the memset.pattern directly.
490 IRBuilder<> Builder(Inst);
491 Module *M = Memset->getModule();
492 const DataLayout &DL = Memset->getDataLayout();
493
494 FunctionCallee MSP =
495 RTLCI.getOrInsertLibcallDecl(M&: *M, LibcallImpl: RTLIB::impl_memset_pattern16);
496 Type *SizeTTy = MSP.getFunctionType()->getParamType(i: 2);
497
498 // Otherwise we should form a memset_pattern16. PatternValue is known
499 // to be an constant array of 16-bytes. Put the value into a mergable
500 // global.
501 assert(Memset->getRawDest()->getType()->getPointerAddressSpace() == 0 &&
502 "Should have skipped if non-zero AS");
503 GlobalVariable *GV;
504 auto It = CMap.find(Val: PatternValue);
505 if (It != CMap.end()) {
506 GV = It->second;
507 } else {
508 GV = new GlobalVariable(
509 *M, PatternValue->getType(), /*isConstant=*/true,
510 GlobalValue::PrivateLinkage, PatternValue, ".memset_pattern");
511 GV->setUnnamedAddr(
512 GlobalValue::UnnamedAddr::Global); // Ok to merge these.
513 // TODO: Consider relaxing alignment requirement.
514 GV->setAlignment(Align(16));
515 CMap[PatternValue] = GV;
516 }
517 Value *PatternPtr = GV;
518 Value *NumBytes = Builder.CreateMul(
519 LHS: ConstantInt::get(Ty: SizeTTy,
520 V: DL.getTypeAllocSize(Ty: Memset->getValue()->getType())),
521 RHS: Builder.CreateZExtOrTrunc(V: Memset->getLength(), DestTy: SizeTTy));
522 CallInst *MemsetPattern16Call =
523 Builder.CreateCall(Callee: MSP, Args: {Memset->getRawDest(), PatternPtr, NumBytes});
524 MemsetPattern16Call->setCallingConv(
525 RTLCI.getLibcallImplCallingConv(Call: RTLIB::impl_memset_pattern16));
526 MemsetPattern16Call->setAAMetadata(Memset->getAAMetadata());
527 // Preserve any call site attributes on the destination pointer
528 // argument (e.g. alignment).
529 AttrBuilder ArgAttrs(Memset->getContext(),
530 Memset->getAttributes().getParamAttrs(ArgNo: 0));
531 MemsetPattern16Call->setAttributes(
532 MemsetPattern16Call->getAttributes().addParamAttributes(
533 C&: Memset->getContext(), ArgNo: 0, B: ArgAttrs));
534 Changed = true;
535 Memset->eraseFromParent();
536 break;
537 }
538 default:
539 llvm_unreachable("unhandled intrinsic");
540 }
541 }
542
543 return Changed;
544}
545
546static GlobalValue *getDeactivationSymbol(CallInst *Call) {
547 if (auto Bundle = Call->getOperandBundle(ID: LLVMContext::OB_deactivation_symbol))
548 return cast<GlobalValue>(Val: Bundle->Inputs[0]);
549 return nullptr;
550}
551
552static bool expandPtrauthForEmuPAC(Function &Intr) {
553 Module &M = *Intr.getParent();
554 if (Triple(M.getTargetTriple()).isArm64e())
555 return false;
556
557 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
558
559 assert(Intr.getIntrinsicID() == Intrinsic::ptrauth_sign ||
560 Intr.getIntrinsicID() == Intrinsic::ptrauth_auth);
561 auto *EmuFnTy = FunctionType::get(Result: Int64Ty, Params: {Int64Ty, Int64Ty}, isVarArg: false);
562 FunctionCallee EmuIntr = M.getOrInsertFunction(
563 Name: Intr.getIntrinsicID() == Intrinsic::ptrauth_auth ? "__emupac_autda"
564 : "__emupac_pacda",
565 T: EmuFnTy);
566
567 for (User *U : llvm::make_early_inc_range(Range: Intr.users())) {
568 auto *Call = cast<CallInst>(Val: U);
569 // We only support the DA key for now.
570 if (auto *Key = dyn_cast<ConstantInt>(Val: Call->getArgOperand(i: 1));
571 !Key || Key->getZExtValue() != /*AArch64PACKey::DA*/ 2)
572 continue;
573
574 Function *F = Call->getParent()->getParent();
575 Attribute FSAttr = F->getFnAttribute(Kind: "target-features");
576 if (FSAttr.isValid() && FSAttr.getValueAsString().contains(Other: "+pauth"))
577 continue;
578
579 std::vector<OperandBundleDef> DSBundle;
580 if (auto *DS = getDeactivationSymbol(Call))
581 DSBundle.push_back(x: OperandBundleDef("deactivation-symbol", DS));
582
583 IRBuilder<> B(Call);
584 auto *EmuCall = B.CreateCall(
585 Callee: EmuIntr, Args: {Call->getArgOperand(i: 0), Call->getArgOperand(i: 2)}, OpBundles: DSBundle);
586 Call->replaceAllUsesWith(V: EmuCall);
587 Call->eraseFromParent();
588 }
589 return true;
590}
591
592static bool expandProtectedFieldPtr(Function &Intr) {
593 Module &M = *Intr.getParent();
594
595 SmallPtrSet<GlobalValue *, 2> DSsToDeactivate;
596
597 Type *Int8Ty = Type::getInt8Ty(C&: M.getContext());
598 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
599 PointerType *PtrTy = PointerType::get(C&: M.getContext(), AddressSpace: 0);
600
601 for (User *U : llvm::make_early_inc_range(Range: Intr.users())) {
602 auto *Call = cast<CallInst>(Val: U);
603
604 auto *Pointer = Call->getArgOperand(i: 0);
605 bool UseHWEncoding =
606 cast<ConstantInt>(Val: Call->getArgOperand(i: 2))->getZExtValue();
607 if (!UseHWEncoding)
608 reportFatalUsageError(reason: "software encoding currently unsupported");
609
610 auto *DS = getDeactivationSymbol(Call);
611
612 for (Use &U : llvm::make_early_inc_range(Range: Call->uses())) {
613 // Comparisons against null cannot be used to recover the original
614 // pointer so we replace them with comparisons against the original
615 // pointer.
616 if (auto *CI = dyn_cast<ICmpInst>(Val: U.getUser())) {
617 if (auto *Op = dyn_cast<Constant>(Val: CI->getOperand(i_nocapture: 0))) {
618 if (Op->isNullValue()) {
619 CI->setOperand(i_nocapture: 1, Val_nocapture: Pointer);
620 continue;
621 }
622 }
623 if (auto *Op = dyn_cast<Constant>(Val: CI->getOperand(i_nocapture: 1))) {
624 if (Op->isNullValue()) {
625 CI->setOperand(i_nocapture: 0, Val_nocapture: Pointer);
626 continue;
627 }
628 }
629 }
630
631 // If we are here, this means that we couldn't rewrite away this use of
632 // the intrinsic. Any load or store uses were removed by InstCombine, and
633 // in general, we can't rewrite away non-load/store uses of
634 // llvm.protected.field.ptr because doing so could expose the encoded
635 // pointer value to the program. Replace it with the pointer operand, and
636 // arrange to define a deactivation symbol.
637 U.set(Pointer);
638 if (DS)
639 DSsToDeactivate.insert(Ptr: DS);
640 }
641
642 Call->eraseFromParent();
643 }
644
645 if (!DSsToDeactivate.empty()) {
646 // This is an AArch64 NOP instruction. When the deactivation symbol support
647 // is expanded to more architectures, there will likely need to be an API
648 // for retrieving this constant.
649 Constant *Nop =
650 ConstantExpr::getIntToPtr(C: ConstantInt::get(Ty: Int64Ty, V: 0xd503201f), Ty: PtrTy);
651 for (GlobalValue *OldDS : DSsToDeactivate) {
652 GlobalValue *DS = GlobalAlias::create(
653 Ty: Int8Ty, AddressSpace: 0, Linkage: GlobalValue::ExternalLinkage, Name: OldDS->getName(), Aliasee: Nop, Parent: &M);
654 DS->setVisibility(GlobalValue::HiddenVisibility);
655 DS->takeName(V: OldDS);
656 OldDS->replaceAllUsesWith(V: DS);
657 OldDS->eraseFromParent();
658 }
659 }
660 return true;
661}
662
663static bool expandCondLoop(Function &Intr) {
664 for (User *U : llvm::make_early_inc_range(Range: Intr.users())) {
665 auto *Call = cast<CallInst>(Val: U);
666
667 auto *Br = cast<UncondBrInst>(
668 Val: SplitBlockAndInsertIfThen(Cond: Call->getArgOperand(i: 0), SplitBefore: Call, Unreachable: false,
669 BranchWeights: getExplicitlyUnknownBranchWeightsIfProfiled(
670 F&: *Call->getFunction(), DEBUG_TYPE)));
671 Br->setSuccessor(Br->getParent());
672 Call->eraseFromParent();
673 }
674 return true;
675}
676
677static bool expandLoopTrap(Function &Intr) {
678 for (User *U : make_early_inc_range(Range: Intr.users())) {
679 auto *Call = cast<CallInst>(Val: U);
680 if (!Call->getParent()->isEntryBlock() &&
681 std::all_of(first: Call->getParent()->begin(), last: BasicBlock::iterator(Call),
682 pred: [](Instruction &I) { return !I.mayHaveSideEffects(); })) {
683 for (auto *BB : predecessors(BB: Call->getParent())) {
684 auto *BI = dyn_cast<CondBrInst>(Val: BB->getTerminator());
685 if (!BI)
686 continue;
687 IRBuilder<> B(BI);
688 Value *Cond;
689 // The looptrap can either be on the true branch or the false branch.
690 // We insert the cond loop before the branch, which uses the branch's
691 // original condition for going to the looptrap as its condition, and
692 // force the branch to take whichever path does not lead to the
693 // looptrap, as the original path to the looptrap is now unreachable
694 // thanks to the cond loop. The codegenprepare pass will clean up our
695 // "unconditional conditional branch" by combining the two basic blocks
696 // if possible, or replacing it with an unconditional branch.
697 if (BI->getSuccessor(i: 0) == Call->getParent()) {
698 // The looptrap is on the true branch.
699 Cond = BI->getCondition();
700 BI->setCondition(ConstantInt::getFalse(Context&: BI->getContext()));
701 } else {
702 // The looptrap is on the false branch, which means that we need to
703 // invert the condition.
704 Cond = B.CreateNot(V: BI->getCondition());
705 BI->setCondition(ConstantInt::getTrue(Context&: BI->getContext()));
706 }
707 B.CreateIntrinsic(ID: Intrinsic::cond_loop, Args: Cond);
708 }
709 }
710 IRBuilder<> B(Call);
711 B.CreateIntrinsic(ID: Intrinsic::cond_loop,
712 Args: ConstantInt::getTrue(Context&: Call->getContext()));
713 Call->eraseFromParent();
714 }
715 return true;
716}
717
718bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
719 // Map unique constants to globals.
720 DenseMap<Constant *, GlobalVariable *> CMap;
721 bool Changed = false;
722 for (Function &F : M) {
723 switch (F.getIntrinsicID()) {
724 default:
725 break;
726 case Intrinsic::memcpy:
727 case Intrinsic::memcpy_inline:
728 case Intrinsic::memmove:
729 case Intrinsic::memset:
730 case Intrinsic::memset_inline:
731 case Intrinsic::experimental_memset_pattern:
732 Changed |= expandMemIntrinsicUses(F, CMap);
733 break;
734 case Intrinsic::load_relative:
735 Changed |= lowerLoadRelative(F);
736 break;
737 case Intrinsic::can_load_speculatively:
738 Changed |= lowerCanLoadSpeculatively(F, TM);
739 break;
740 case Intrinsic::is_constant:
741 case Intrinsic::objectsize:
742 Changed |= forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
743 Function *Parent = CI->getParent()->getParent();
744 TargetLibraryInfo &TLI = LookupTLI(*Parent);
745 // Intrinsics in unreachable code are not lowered.
746 bool Changed = lowerConstantIntrinsics(F&: *Parent, TLI, /*DT=*/nullptr);
747 return Changed;
748 });
749 break;
750#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
751 case Intrinsic::VPID:
752#include "llvm/IR/VPIntrinsics.def"
753 forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
754 Function *Parent = CI->getParent()->getParent();
755 const TargetTransformInfo &TTI = LookupTTI(*Parent);
756 auto *VPI = cast<VPIntrinsic>(Val: CI);
757 VPExpansionDetails ED = expandVectorPredicationIntrinsic(VPI&: *VPI, TTI);
758 // Expansion of VP intrinsics may change the IR but not actually
759 // replace the intrinsic, so update Changed for the pass
760 // and compute Removed for forEachCall.
761 Changed |= ED != VPExpansionDetails::IntrinsicUnchanged;
762 bool Removed = ED == VPExpansionDetails::IntrinsicReplaced;
763 return Removed;
764 });
765 break;
766 case Intrinsic::objc_autorelease:
767 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autorelease);
768 break;
769 case Intrinsic::objc_autoreleasePoolPop:
770 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autoreleasePoolPop);
771 break;
772 case Intrinsic::objc_autoreleasePoolPush:
773 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autoreleasePoolPush);
774 break;
775 case Intrinsic::objc_autoreleaseReturnValue:
776 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autoreleaseReturnValue);
777 break;
778 case Intrinsic::objc_copyWeak:
779 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_copyWeak);
780 break;
781 case Intrinsic::objc_destroyWeak:
782 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_destroyWeak);
783 break;
784 case Intrinsic::objc_initWeak:
785 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_initWeak);
786 break;
787 case Intrinsic::objc_loadWeak:
788 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_loadWeak);
789 break;
790 case Intrinsic::objc_loadWeakRetained:
791 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_loadWeakRetained);
792 break;
793 case Intrinsic::objc_moveWeak:
794 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_moveWeak);
795 break;
796 case Intrinsic::objc_release:
797 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_release, setNonLazyBind: true);
798 break;
799 case Intrinsic::objc_retain:
800 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retain, setNonLazyBind: true);
801 break;
802 case Intrinsic::objc_retainAutorelease:
803 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainAutorelease);
804 break;
805 case Intrinsic::objc_retainAutoreleaseReturnValue:
806 Changed |=
807 lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainAutoreleaseReturnValue);
808 break;
809 case Intrinsic::objc_retainAutoreleasedReturnValue:
810 Changed |=
811 lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainAutoreleasedReturnValue);
812 break;
813 case Intrinsic::objc_claimAutoreleasedReturnValue:
814 Changed |=
815 lowerObjCCall(F, NewFn: RTLIB::impl_objc_claimAutoreleasedReturnValue);
816 break;
817 case Intrinsic::objc_retainBlock:
818 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainBlock);
819 break;
820 case Intrinsic::objc_storeStrong:
821 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_storeStrong);
822 break;
823 case Intrinsic::objc_storeWeak:
824 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_storeWeak);
825 break;
826 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
827 Changed |=
828 lowerObjCCall(F, NewFn: RTLIB::impl_objc_unsafeClaimAutoreleasedReturnValue);
829 break;
830 case Intrinsic::objc_retainedObject:
831 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainedObject);
832 break;
833 case Intrinsic::objc_unretainedObject:
834 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_unretainedObject);
835 break;
836 case Intrinsic::objc_unretainedPointer:
837 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_unretainedPointer);
838 break;
839 case Intrinsic::objc_retain_autorelease:
840 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retain_autorelease);
841 break;
842 case Intrinsic::objc_sync_enter:
843 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_sync_enter);
844 break;
845 case Intrinsic::objc_sync_exit:
846 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_sync_exit);
847 break;
848 case Intrinsic::acos:
849 case Intrinsic::asin:
850 case Intrinsic::atan:
851 case Intrinsic::cos:
852 case Intrinsic::cosh:
853 case Intrinsic::exp:
854 case Intrinsic::exp2:
855 case Intrinsic::exp10:
856 case Intrinsic::log:
857 case Intrinsic::log2:
858 case Intrinsic::log10:
859 case Intrinsic::sin:
860 case Intrinsic::sinh:
861 case Intrinsic::tan:
862 case Intrinsic::tanh:
863 Changed |= forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
864 Type *Ty = CI->getArgOperand(i: 0)->getType();
865 if (!isa<ScalableVectorType>(Val: Ty))
866 return false;
867 const TargetLowering *TL = TM.getSubtargetImpl(F)->getTargetLowering();
868 unsigned Op = TL->IntrinsicIDToISD(ID: F.getIntrinsicID());
869 assert(Op != ISD::DELETED_NODE && "unsupported intrinsic");
870 if (!TL->isOperationExpand(Op, VT: EVT::getEVT(Ty)))
871 return false;
872 return lowerUnaryVectorIntrinsicAsLoop(M, CI);
873 });
874 break;
875 case Intrinsic::modf:
876 case Intrinsic::sincos:
877 case Intrinsic::sincospi:
878 Changed |= forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
879 Type *Ty = CI->getArgOperand(i: 0)->getType();
880 if (!isa<ScalableVectorType>(Val: Ty))
881 return false;
882 const TargetSubtargetInfo *Subtarget = TM.getSubtargetImpl(F);
883 const TargetLowering *TL = Subtarget->getTargetLowering();
884 unsigned Op = TL->IntrinsicIDToISD(ID: F.getIntrinsicID());
885 assert(Op != ISD::DELETED_NODE && "unsupported intrinsic");
886 EVT VT = EVT::getEVT(Ty);
887 if (!TL->isOperationExpand(Op, VT))
888 return false;
889 // The vector legalizer can expand these to a vector math library call.
890 RTLIB::Libcall LC;
891 switch (Op) {
892 case ISD::FMODF:
893 LC = RTLIB::getMODF(VT);
894 break;
895 case ISD::FSINCOS:
896 LC = RTLIB::getSINCOS(VT);
897 break;
898 case ISD::FSINCOSPI:
899 LC = RTLIB::getSINCOSPI(VT);
900 break;
901 default:
902 llvm_unreachable("unexpected intrinsic");
903 }
904
905 const LibcallLoweringInfo &Libcalls =
906 getLibcallLowering(ModuleInfo: ModuleLibcalls, Subtarget: *Subtarget);
907 if (Libcalls.getVectorLibcallImplOrMasked(Call: LC) != RTLIB::Unsupported)
908 return false;
909 return lowerUnaryVectorIntrinsicAsLoop(M, CI);
910 });
911 break;
912 case Intrinsic::ptrauth_sign:
913 case Intrinsic::ptrauth_auth:
914 Changed |= expandPtrauthForEmuPAC(Intr&: F);
915 break;
916 case Intrinsic::protected_field_ptr:
917 Changed |= expandProtectedFieldPtr(Intr&: F);
918 break;
919 case Intrinsic::cond_loop:
920 if (!TM.canLowerCondLoop())
921 Changed |= expandCondLoop(Intr&: F);
922 break;
923 case Intrinsic::looptrap:
924 Changed |= expandLoopTrap(Intr&: F);
925 if (!TM.canLowerCondLoop())
926 if (auto *CondLoop = M.getFunction(Name: "llvm.cond.loop"))
927 Changed |= expandCondLoop(Intr&: *CondLoop);
928 break;
929 }
930 }
931 return Changed;
932}
933
934namespace {
935
936class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
937public:
938 static char ID;
939
940 PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}
941
942 void getAnalysisUsage(AnalysisUsage &AU) const override {
943 AU.addRequired<TargetTransformInfoWrapperPass>();
944 AU.addRequired<TargetLibraryInfoWrapperPass>();
945 AU.addRequired<LibcallLoweringInfoWrapper>();
946 AU.addRequired<TargetPassConfig>();
947 AU.addRequired<AssumptionCacheTracker>();
948 }
949
950 bool runOnModule(Module &M) override {
951 const ModuleLibcallLoweringInfo &ModuleLibcalls =
952 getAnalysis<LibcallLoweringInfoWrapper>().getResult(M);
953
954 auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {
955 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
956 };
957 auto LookupTLI = [this](Function &F) -> TargetLibraryInfo & {
958 return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
959 };
960 auto LookupAC = [this](Function &F) -> AssumptionCache & {
961 return this->getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
962 };
963
964 const auto &TM = getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
965 PreISelIntrinsicLowering Lowering(getCodeGenOptions(M), TM, ModuleLibcalls,
966 LookupTTI, LookupTLI, LookupAC);
967 return Lowering.lowerIntrinsics(M);
968 }
969};
970
971} // end anonymous namespace
972
973char PreISelIntrinsicLoweringLegacyPass::ID;
974
975INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass,
976 "pre-isel-intrinsic-lowering",
977 "Pre-ISel Intrinsic Lowering", false, false)
978INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
979INITIALIZE_PASS_DEPENDENCY(LibcallLoweringInfoWrapper)
980INITIALIZE_PASS_DEPENDENCY(RuntimeLibraryInfoWrapper)
981INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
982INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
983INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
984INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
985 "pre-isel-intrinsic-lowering",
986 "Pre-ISel Intrinsic Lowering", false, false)
987
988ModulePass *llvm::createPreISelIntrinsicLoweringPass() {
989 return new PreISelIntrinsicLoweringLegacyPass();
990}
991
992PreservedAnalyses
993PreISelIntrinsicLoweringPass::run(Module &M, ModuleAnalysisManager &MAM) {
994 const ModuleLibcallLoweringInfo &LibcallLowering =
995 MAM.getResult<LibcallLoweringModuleAnalysis>(IR&: M);
996
997 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
998
999 auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {
1000 return FAM.getResult<TargetIRAnalysis>(IR&: F);
1001 };
1002 auto LookupTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
1003 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
1004 };
1005 auto LookupAC = [&FAM](Function &F) -> AssumptionCache & {
1006 return FAM.getResult<AssumptionAnalysis>(IR&: F);
1007 };
1008
1009 PreISelIntrinsicLowering Lowering(getCodeGenOptions(M), TM, LibcallLowering,
1010 LookupTTI, LookupTLI, LookupAC);
1011 if (!Lowering.lowerIntrinsics(M))
1012 return PreservedAnalyses::all();
1013 else
1014 return PreservedAnalyses::none();
1015}
1016