1//===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass implements IR lowering for the llvm.memcpy, llvm.memmove,
10// llvm.memset, llvm.load.relative and llvm.objc.* intrinsics.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/Analysis/ObjCARCInstKind.h"
17#include "llvm/Analysis/ObjCARCUtil.h"
18#include "llvm/Analysis/TargetLibraryInfo.h"
19#include "llvm/Analysis/TargetTransformInfo.h"
20#include "llvm/CodeGen/ExpandVectorPredication.h"
21#include "llvm/CodeGen/LibcallLoweringInfo.h"
22#include "llvm/CodeGen/Passes.h"
23#include "llvm/CodeGen/RuntimeLibcallUtil.h"
24#include "llvm/CodeGen/TargetLowering.h"
25#include "llvm/CodeGen/TargetPassConfig.h"
26#include "llvm/IR/Function.h"
27#include "llvm/IR/GlobalValue.h"
28#include "llvm/IR/IRBuilder.h"
29#include "llvm/IR/Instructions.h"
30#include "llvm/IR/IntrinsicInst.h"
31#include "llvm/IR/Metadata.h"
32#include "llvm/IR/Module.h"
33#include "llvm/IR/ProfDataUtils.h"
34#include "llvm/IR/RuntimeLibcalls.h"
35#include "llvm/IR/Type.h"
36#include "llvm/IR/Use.h"
37#include "llvm/InitializePasses.h"
38#include "llvm/Pass.h"
39#include "llvm/Support/Casting.h"
40#include "llvm/Target/TargetMachine.h"
41#include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
42#include "llvm/Transforms/Utils/BasicBlockUtils.h"
43#include "llvm/Transforms/Utils/BuildLibCalls.h"
44#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
45#include "llvm/Transforms/Utils/LowerVectorIntrinsics.h"
46
47using namespace llvm;
48
49#define DEBUG_TYPE "pre-isel-intrinsic-lowering"
50
51/// Threshold to leave statically sized memory intrinsic calls. Calls of known
52/// size larger than this will be expanded by the pass. Calls of unknown or
53/// lower size will be left for expansion in codegen.
54static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt(
55 "mem-intrinsic-expand-size",
56 cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(Val: -1),
57 cl::Hidden);
58
59namespace {
60
61struct PreISelIntrinsicLowering {
62 const TargetMachine *TM;
63 const ModuleLibcallLoweringInfo &ModuleLibcalls;
64 const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
65 const function_ref<TargetLibraryInfo &(Function &)> LookupTLI;
66
67 /// If this is true, assume it's preferably to leave memory intrinsic calls
68 /// for replacement with a library call later. Otherwise this depends on
69 /// TargetLoweringInfo availability of the corresponding function.
70 const bool UseMemIntrinsicLibFunc;
71
72 explicit PreISelIntrinsicLowering(
73 const TargetMachine *TM_,
74 const ModuleLibcallLoweringInfo &ModuleLibcalls_,
75 function_ref<TargetTransformInfo &(Function &)> LookupTTI_,
76 function_ref<TargetLibraryInfo &(Function &)> LookupTLI_,
77 bool UseMemIntrinsicLibFunc_ = true)
78 : TM(TM_), ModuleLibcalls(ModuleLibcalls_), LookupTTI(LookupTTI_),
79 LookupTLI(LookupTLI_), UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {
80 }
81
82 static bool shouldExpandMemIntrinsicWithSize(Value *Size,
83 const TargetTransformInfo &TTI);
84 bool
85 expandMemIntrinsicUses(Function &F,
86 DenseMap<Constant *, GlobalVariable *> &CMap) const;
87 bool lowerIntrinsics(Module &M) const;
88};
89
90} // namespace
91
92template <class T> static bool forEachCall(Function &Intrin, T Callback) {
93 // Lowering all intrinsics in a function will delete multiple uses, so we
94 // can't use an early-inc-range. In case some remain, we don't want to look
95 // at them again. Unfortunately, Value::UseList is private, so we can't use a
96 // simple Use**. If LastUse is null, the next use to consider is
97 // Intrin.use_begin(), otherwise it's LastUse->getNext().
98 Use *LastUse = nullptr;
99 bool Changed = false;
100 while (!Intrin.use_empty() && (!LastUse || LastUse->getNext())) {
101 Use *U = LastUse ? LastUse->getNext() : &*Intrin.use_begin();
102 bool Removed = false;
103 // An intrinsic cannot have its address taken, so it cannot be an argument
104 // operand. It might be used as operand in debug metadata, though.
105 if (auto CI = dyn_cast<CallInst>(Val: U->getUser()))
106 Changed |= Removed = Callback(CI);
107 if (!Removed)
108 LastUse = U;
109 }
110 return Changed;
111}
112
113static bool lowerLoadRelative(Function &F) {
114 if (F.use_empty())
115 return false;
116
117 bool Changed = false;
118 Type *Int32Ty = Type::getInt32Ty(C&: F.getContext());
119
120 for (Use &U : llvm::make_early_inc_range(Range: F.uses())) {
121 auto CI = dyn_cast<CallInst>(Val: U.getUser());
122 if (!CI || CI->getCalledOperand() != &F)
123 continue;
124
125 IRBuilder<> B(CI);
126 Value *OffsetPtr =
127 B.CreatePtrAdd(Ptr: CI->getArgOperand(i: 0), Offset: CI->getArgOperand(i: 1));
128 Value *OffsetI32 = B.CreateAlignedLoad(Ty: Int32Ty, Ptr: OffsetPtr, Align: Align(4));
129
130 Value *ResultPtr = B.CreatePtrAdd(Ptr: CI->getArgOperand(i: 0), Offset: OffsetI32);
131
132 CI->replaceAllUsesWith(V: ResultPtr);
133 CI->eraseFromParent();
134 Changed = true;
135 }
136
137 return Changed;
138}
139
140/// Lower @llvm.can.load.speculatively using target-specific expansion.
141/// Targets may provide their own expansion via
142/// TargetLowering::emitCanLoadSpeculatively; the default expansion
143/// conservatively returns false.
144static bool lowerCanLoadSpeculatively(Function &F, const TargetMachine *TM) {
145 if (!TM)
146 return false;
147
148 return forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
149 const TargetLowering *TLI =
150 TM->getSubtargetImpl(*CI->getFunction())->getTargetLowering();
151
152 IRBuilder<> Builder(CI);
153 // A null result means the target cannot answer; lower to false.
154 Value *Result = TLI->emitCanLoadSpeculatively(Builder, Ptr: CI->getArgOperand(i: 0),
155 Size: CI->getArgOperand(i: 1));
156 if (!Result)
157 Result = Builder.getFalse();
158
159 CI->replaceAllUsesWith(V: Result);
160 CI->eraseFromParent();
161 return true;
162 });
163}
164
165// ObjCARC has knowledge about whether an obj-c runtime function needs to be
166// always tail-called or never tail-called.
167static CallInst::TailCallKind getOverridingTailCallKind(const Function &F) {
168 objcarc::ARCInstKind Kind = objcarc::GetFunctionClass(F: &F);
169 if (objcarc::IsAlwaysTail(Class: Kind))
170 return CallInst::TCK_Tail;
171 else if (objcarc::IsNeverTail(Class: Kind))
172 return CallInst::TCK_NoTail;
173 return CallInst::TCK_None;
174}
175
176static bool lowerObjCCall(Function &F, RTLIB::LibcallImpl NewFn,
177 bool setNonLazyBind = false) {
178 assert(IntrinsicInst::mayLowerToFunctionCall(F.getIntrinsicID()) &&
179 "Pre-ISel intrinsics do lower into regular function calls");
180 if (F.use_empty())
181 return false;
182
183 // FIXME: When RuntimeLibcalls is an analysis, check if the function is really
184 // supported, and go through RTLIB::Libcall.
185 StringRef NewFnName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: NewFn);
186
187 // If we haven't already looked up this function, check to see if the
188 // program already contains a function with this name.
189 Module *M = F.getParent();
190 FunctionCallee FCache =
191 M->getOrInsertFunction(Name: NewFnName, T: F.getFunctionType());
192
193 if (Function *Fn = dyn_cast<Function>(Val: FCache.getCallee())) {
194 Fn->setLinkage(F.getLinkage());
195 if (setNonLazyBind && !Fn->isWeakForLinker()) {
196 // If we have Native ARC, set nonlazybind attribute for these APIs for
197 // performance.
198 Fn->addFnAttr(Kind: Attribute::NonLazyBind);
199 }
200 }
201
202 CallInst::TailCallKind OverridingTCK = getOverridingTailCallKind(F);
203
204 for (Use &U : llvm::make_early_inc_range(Range: F.uses())) {
205 auto *CB = cast<CallBase>(Val: U.getUser());
206
207 if (CB->getCalledFunction() != &F) {
208 assert(objcarc::getAttachedARCFunction(CB) == &F &&
209 "use expected to be the argument of operand bundle "
210 "\"clang.arc.attachedcall\"");
211 U.set(FCache.getCallee());
212 continue;
213 }
214
215 auto *CI = cast<CallInst>(Val: CB);
216 assert(CI->getCalledFunction() && "Cannot lower an indirect call!");
217
218 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
219 SmallVector<Value *, 8> Args(CI->args());
220 SmallVector<llvm::OperandBundleDef, 1> BundleList;
221 CI->getOperandBundlesAsDefs(Defs&: BundleList);
222 CallInst *NewCI = Builder.CreateCall(Callee: FCache, Args, OpBundles: BundleList);
223 NewCI->setName(CI->getName());
224
225 // Try to set the most appropriate TailCallKind based on both the current
226 // attributes and the ones that we could get from ObjCARC's special
227 // knowledge of the runtime functions.
228 //
229 // std::max respects both requirements of notail and tail here:
230 // * notail on either the call or from ObjCARC becomes notail
231 // * tail on either side is stronger than none, but not notail
232 CallInst::TailCallKind TCK = CI->getTailCallKind();
233 NewCI->setTailCallKind(std::max(a: TCK, b: OverridingTCK));
234
235 // Transfer the 'returned' attribute from the intrinsic to the call site.
236 // By applying this only to intrinsic call sites, we avoid applying it to
237 // non-ARC explicit calls to things like objc_retain which have not been
238 // auto-upgraded to use the intrinsics.
239 unsigned Index;
240 if (F.getAttributes().hasAttrSomewhere(Kind: Attribute::Returned, Index: &Index) &&
241 Index)
242 NewCI->addParamAttr(ArgNo: Index - AttributeList::FirstArgIndex,
243 Kind: Attribute::Returned);
244
245 if (!CI->use_empty())
246 CI->replaceAllUsesWith(V: NewCI);
247 CI->eraseFromParent();
248 }
249
250 return true;
251}
252
253// TODO: Should refine based on estimated number of accesses (e.g. does it
254// require splitting based on alignment)
255bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(
256 Value *Size, const TargetTransformInfo &TTI) {
257 ConstantInt *CI = dyn_cast<ConstantInt>(Val: Size);
258 if (!CI)
259 return true;
260 uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences()
261 ? MemIntrinsicExpandSizeThresholdOpt
262 : TTI.getMaxMemIntrinsicInlineSizeThreshold();
263 uint64_t SizeVal = CI->getZExtValue();
264
265 // Treat a threshold of 0 as a special case to force expansion of all
266 // intrinsics, including size 0.
267 return SizeVal > Threshold || Threshold == 0;
268}
269
270static bool canEmitLibcall(const ModuleLibcallLoweringInfo &ModuleLowering,
271 const TargetMachine *TM, Function *F,
272 RTLIB::Libcall LC) {
273 // TODO: Should this consider the address space of the memcpy?
274 if (!TM)
275 return true;
276 const LibcallLoweringInfo &Lowering =
277 getLibcallLowering(ModuleInfo: ModuleLowering, Subtarget: *TM->getSubtargetImpl(*F));
278 return Lowering.getLibcallImpl(Call: LC) != RTLIB::Unsupported;
279}
280
281static bool canEmitMemcpy(const ModuleLibcallLoweringInfo &ModuleLowering,
282 const TargetMachine *TM, Function *F) {
283 // TODO: Should this consider the address space of the memcpy?
284 if (!TM)
285 return true;
286 const LibcallLoweringInfo &Lowering =
287 getLibcallLowering(ModuleInfo: ModuleLowering, Subtarget: *TM->getSubtargetImpl(*F));
288 return Lowering.getMemcpyImpl() != RTLIB::Unsupported;
289}
290
291// Return a value appropriate for use with the memset_pattern16 libcall, if
292// possible and if we know how. (Adapted from equivalent helper in
293// LoopIdiomRecognize).
294static Constant *getMemSetPattern16Value(MemSetPatternInst *Inst,
295 const TargetLibraryInfo &TLI) {
296 // TODO: This could check for UndefValue because it can be merged into any
297 // other valid pattern.
298
299 // Don't emit libcalls if a non-default address space is being used.
300 if (Inst->getRawDest()->getType()->getPointerAddressSpace() != 0)
301 return nullptr;
302
303 Value *V = Inst->getValue();
304 Type *VTy = V->getType();
305 const DataLayout &DL = Inst->getDataLayout();
306 Module *M = Inst->getModule();
307
308 if (!isLibFuncEmittable(M, TLI: &TLI, TheLibFunc: LibFunc_memset_pattern16))
309 return nullptr;
310
311 // If the value isn't a constant, we can't promote it to being in a constant
312 // array. We could theoretically do a store to an alloca or something, but
313 // that doesn't seem worthwhile.
314 Constant *C = dyn_cast<Constant>(Val: V);
315 if (!C || isa<ConstantExpr>(Val: C))
316 return nullptr;
317
318 // Only handle simple values that are a power of two bytes in size.
319 uint64_t Size = DL.getTypeSizeInBits(Ty: VTy);
320 if (!DL.typeSizeEqualsStoreSize(Ty: VTy) || !isPowerOf2_64(Value: Size))
321 return nullptr;
322
323 // Don't care enough about darwin/ppc to implement this.
324 if (DL.isBigEndian())
325 return nullptr;
326
327 // Convert to size in bytes.
328 Size /= 8;
329
330 // TODO: If CI is larger than 16-bytes, we can try slicing it in half to see
331 // if the top and bottom are the same (e.g. for vectors and large integers).
332 if (Size > 16)
333 return nullptr;
334
335 // If the constant is exactly 16 bytes, just use it.
336 if (Size == 16)
337 return C;
338
339 // Otherwise, we'll use an array of the constants.
340 uint64_t ArraySize = 16 / Size;
341 ArrayType *AT = ArrayType::get(ElementType: V->getType(), NumElements: ArraySize);
342 return ConstantArray::get(T: AT, V: std::vector<Constant *>(ArraySize, C));
343}
344
345// TODO: Handle atomic memcpy and memcpy.inline
346// TODO: Pass ScalarEvolution
347bool PreISelIntrinsicLowering::expandMemIntrinsicUses(
348 Function &F, DenseMap<Constant *, GlobalVariable *> &CMap) const {
349 Intrinsic::ID ID = F.getIntrinsicID();
350 bool Changed = false;
351
352 for (User *U : llvm::make_early_inc_range(Range: F.users())) {
353 Instruction *Inst = cast<Instruction>(Val: U);
354
355 switch (ID) {
356 case Intrinsic::memcpy: {
357 auto *Memcpy = cast<MemCpyInst>(Val: Inst);
358 Function *ParentFunc = Memcpy->getFunction();
359 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
360 if (shouldExpandMemIntrinsicWithSize(Size: Memcpy->getLength(), TTI)) {
361 if (UseMemIntrinsicLibFunc &&
362 canEmitMemcpy(ModuleLowering: ModuleLibcalls, TM, F: ParentFunc))
363 break;
364
365 // TODO: For optsize, emit the loop into a separate function
366 expandMemCpyAsLoop(MemCpy: Memcpy, TTI);
367 Changed = true;
368 Memcpy->eraseFromParent();
369 }
370
371 break;
372 }
373 case Intrinsic::memcpy_inline: {
374 // Only expand llvm.memcpy.inline with non-constant length in this
375 // codepath, leaving the current SelectionDAG expansion for constant
376 // length memcpy intrinsics undisturbed.
377 auto *Memcpy = cast<MemCpyInst>(Val: Inst);
378 if (isa<ConstantInt>(Val: Memcpy->getLength()))
379 break;
380
381 Function *ParentFunc = Memcpy->getFunction();
382 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
383 expandMemCpyAsLoop(MemCpy: Memcpy, TTI);
384 Changed = true;
385 Memcpy->eraseFromParent();
386 break;
387 }
388 case Intrinsic::memmove: {
389 auto *Memmove = cast<MemMoveInst>(Val: Inst);
390 Function *ParentFunc = Memmove->getFunction();
391 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
392 if (shouldExpandMemIntrinsicWithSize(Size: Memmove->getLength(), TTI)) {
393 if (UseMemIntrinsicLibFunc &&
394 canEmitLibcall(ModuleLowering: ModuleLibcalls, TM, F: ParentFunc, LC: RTLIB::MEMMOVE))
395 break;
396
397 if (expandMemMoveAsLoop(MemMove: Memmove, TTI)) {
398 Changed = true;
399 Memmove->eraseFromParent();
400 }
401 }
402
403 break;
404 }
405 case Intrinsic::memset: {
406 auto *Memset = cast<MemSetInst>(Val: Inst);
407 Function *ParentFunc = Memset->getFunction();
408 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
409 if (shouldExpandMemIntrinsicWithSize(Size: Memset->getLength(), TTI)) {
410 if (UseMemIntrinsicLibFunc &&
411 canEmitLibcall(ModuleLowering: ModuleLibcalls, TM, F: ParentFunc, LC: RTLIB::MEMSET))
412 break;
413
414 expandMemSetAsLoop(MemSet: Memset, TTI);
415 Changed = true;
416 Memset->eraseFromParent();
417 }
418
419 break;
420 }
421 case Intrinsic::memset_inline: {
422 // Only expand llvm.memset.inline with non-constant length in this
423 // codepath, leaving the current SelectionDAG expansion for constant
424 // length memset intrinsics undisturbed.
425 auto *Memset = cast<MemSetInst>(Val: Inst);
426 if (isa<ConstantInt>(Val: Memset->getLength()))
427 break;
428
429 Function *ParentFunc = Memset->getFunction();
430 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
431 expandMemSetAsLoop(MemSet: Memset, TTI);
432 Changed = true;
433 Memset->eraseFromParent();
434 break;
435 }
436 case Intrinsic::experimental_memset_pattern: {
437 auto *Memset = cast<MemSetPatternInst>(Val: Inst);
438 Function *ParentFunc = Memset->getFunction();
439 const TargetLibraryInfo &TLI = LookupTLI(*ParentFunc);
440 Constant *PatternValue = getMemSetPattern16Value(Inst: Memset, TLI);
441 if (!PatternValue) {
442 // If it isn't possible to emit a memset_pattern16 libcall, expand to
443 // a loop instead.
444 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
445 expandMemSetPatternAsLoop(MemSet: Memset, TTI);
446 Changed = true;
447 Memset->eraseFromParent();
448 break;
449 }
450 // FIXME: There is currently no profitability calculation for emitting
451 // the libcall vs expanding the memset.pattern directly.
452 IRBuilder<> Builder(Inst);
453 Module *M = Memset->getModule();
454 const DataLayout &DL = Memset->getDataLayout();
455
456 Type *DestPtrTy = Memset->getRawDest()->getType();
457 Type *SizeTTy = TLI.getSizeTType(M: *M);
458 StringRef FuncName = "memset_pattern16";
459 FunctionCallee MSP = getOrInsertLibFunc(M, TLI, TheLibFunc: LibFunc_memset_pattern16,
460 RetTy: Builder.getVoidTy(), Args: DestPtrTy,
461 Args: Builder.getPtrTy(), Args: SizeTTy);
462 inferNonMandatoryLibFuncAttrs(M, Name: FuncName, TLI);
463
464 // Otherwise we should form a memset_pattern16. PatternValue is known
465 // to be an constant array of 16-bytes. Put the value into a mergable
466 // global.
467 assert(Memset->getRawDest()->getType()->getPointerAddressSpace() == 0 &&
468 "Should have skipped if non-zero AS");
469 GlobalVariable *GV;
470 auto It = CMap.find(Val: PatternValue);
471 if (It != CMap.end()) {
472 GV = It->second;
473 } else {
474 GV = new GlobalVariable(
475 *M, PatternValue->getType(), /*isConstant=*/true,
476 GlobalValue::PrivateLinkage, PatternValue, ".memset_pattern");
477 GV->setUnnamedAddr(
478 GlobalValue::UnnamedAddr::Global); // Ok to merge these.
479 // TODO: Consider relaxing alignment requirement.
480 GV->setAlignment(Align(16));
481 CMap[PatternValue] = GV;
482 }
483 Value *PatternPtr = GV;
484 Value *NumBytes = Builder.CreateMul(
485 LHS: TLI.getAsSizeT(V: DL.getTypeAllocSize(Ty: Memset->getValue()->getType()),
486 M: *M),
487 RHS: Builder.CreateZExtOrTrunc(V: Memset->getLength(), DestTy: SizeTTy));
488 CallInst *MemsetPattern16Call =
489 Builder.CreateCall(Callee: MSP, Args: {Memset->getRawDest(), PatternPtr, NumBytes});
490 MemsetPattern16Call->setAAMetadata(Memset->getAAMetadata());
491 // Preserve any call site attributes on the destination pointer
492 // argument (e.g. alignment).
493 AttrBuilder ArgAttrs(Memset->getContext(),
494 Memset->getAttributes().getParamAttrs(ArgNo: 0));
495 MemsetPattern16Call->setAttributes(
496 MemsetPattern16Call->getAttributes().addParamAttributes(
497 C&: Memset->getContext(), ArgNo: 0, B: ArgAttrs));
498 Changed = true;
499 Memset->eraseFromParent();
500 break;
501 }
502 default:
503 llvm_unreachable("unhandled intrinsic");
504 }
505 }
506
507 return Changed;
508}
509
510static GlobalValue *getDeactivationSymbol(CallInst *Call) {
511 if (auto Bundle = Call->getOperandBundle(ID: LLVMContext::OB_deactivation_symbol))
512 return cast<GlobalValue>(Val: Bundle->Inputs[0]);
513 return nullptr;
514}
515
516static bool expandPtrauthForEmuPAC(Function &Intr) {
517 Module &M = *Intr.getParent();
518 if (Triple(M.getTargetTriple()).isArm64e())
519 return false;
520
521 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
522
523 assert(Intr.getIntrinsicID() == Intrinsic::ptrauth_sign ||
524 Intr.getIntrinsicID() == Intrinsic::ptrauth_auth);
525 auto *EmuFnTy = FunctionType::get(Result: Int64Ty, Params: {Int64Ty, Int64Ty}, isVarArg: false);
526 FunctionCallee EmuIntr = M.getOrInsertFunction(
527 Name: Intr.getIntrinsicID() == Intrinsic::ptrauth_auth ? "__emupac_autda"
528 : "__emupac_pacda",
529 T: EmuFnTy);
530
531 for (User *U : llvm::make_early_inc_range(Range: Intr.users())) {
532 auto *Call = cast<CallInst>(Val: U);
533 // We only support the DA key for now.
534 if (auto *Key = dyn_cast<ConstantInt>(Val: Call->getArgOperand(i: 1));
535 !Key || Key->getZExtValue() != /*AArch64PACKey::DA*/ 2)
536 continue;
537
538 Function *F = Call->getParent()->getParent();
539 Attribute FSAttr = F->getFnAttribute(Kind: "target-features");
540 if (FSAttr.isValid() && FSAttr.getValueAsString().contains(Other: "+pauth"))
541 continue;
542
543 std::vector<OperandBundleDef> DSBundle;
544 if (auto *DS = getDeactivationSymbol(Call))
545 DSBundle.push_back(x: OperandBundleDef("deactivation-symbol", DS));
546
547 IRBuilder<> B(Call);
548 auto *EmuCall = B.CreateCall(
549 Callee: EmuIntr, Args: {Call->getArgOperand(i: 0), Call->getArgOperand(i: 2)}, OpBundles: DSBundle);
550 Call->replaceAllUsesWith(V: EmuCall);
551 Call->eraseFromParent();
552 }
553 return true;
554}
555
556static bool expandProtectedFieldPtr(Function &Intr) {
557 Module &M = *Intr.getParent();
558
559 SmallPtrSet<GlobalValue *, 2> DSsToDeactivate;
560
561 Type *Int8Ty = Type::getInt8Ty(C&: M.getContext());
562 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
563 PointerType *PtrTy = PointerType::get(C&: M.getContext(), AddressSpace: 0);
564
565 for (User *U : llvm::make_early_inc_range(Range: Intr.users())) {
566 auto *Call = cast<CallInst>(Val: U);
567
568 auto *Pointer = Call->getArgOperand(i: 0);
569 bool UseHWEncoding =
570 cast<ConstantInt>(Val: Call->getArgOperand(i: 2))->getZExtValue();
571 if (!UseHWEncoding)
572 reportFatalUsageError(reason: "software encoding currently unsupported");
573
574 auto *DS = getDeactivationSymbol(Call);
575 OperandBundleDef DSBundle("deactivation-symbol", DS);
576
577 for (Use &U : llvm::make_early_inc_range(Range: Call->uses())) {
578 // Comparisons against null cannot be used to recover the original
579 // pointer so we replace them with comparisons against the original
580 // pointer.
581 if (auto *CI = dyn_cast<ICmpInst>(Val: U.getUser())) {
582 if (auto *Op = dyn_cast<Constant>(Val: CI->getOperand(i_nocapture: 0))) {
583 if (Op->isNullValue()) {
584 CI->setOperand(i_nocapture: 1, Val_nocapture: Pointer);
585 continue;
586 }
587 }
588 if (auto *Op = dyn_cast<Constant>(Val: CI->getOperand(i_nocapture: 1))) {
589 if (Op->isNullValue()) {
590 CI->setOperand(i_nocapture: 0, Val_nocapture: Pointer);
591 continue;
592 }
593 }
594 }
595
596 // If we are here, this means that we couldn't rewrite away this use of
597 // the intrinsic. Any load or store uses were removed by InstCombine, and
598 // in general, we can't rewrite away non-load/store uses of
599 // llvm.protected.field.ptr because doing so could expose the encoded
600 // pointer value to the program. Replace it with the pointer operand, and
601 // arrange to define a deactivation symbol.
602 U.set(Pointer);
603 if (DS)
604 DSsToDeactivate.insert(Ptr: DS);
605 }
606
607 Call->eraseFromParent();
608 }
609
610 if (!DSsToDeactivate.empty()) {
611 // This is an AArch64 NOP instruction. When the deactivation symbol support
612 // is expanded to more architectures, there will likely need to be an API
613 // for retrieving this constant.
614 Constant *Nop =
615 ConstantExpr::getIntToPtr(C: ConstantInt::get(Ty: Int64Ty, V: 0xd503201f), Ty: PtrTy);
616 for (GlobalValue *OldDS : DSsToDeactivate) {
617 GlobalValue *DS = GlobalAlias::create(
618 Ty: Int8Ty, AddressSpace: 0, Linkage: GlobalValue::ExternalLinkage, Name: OldDS->getName(), Aliasee: Nop, Parent: &M);
619 DS->setVisibility(GlobalValue::HiddenVisibility);
620 DS->takeName(V: OldDS);
621 OldDS->replaceAllUsesWith(V: DS);
622 OldDS->eraseFromParent();
623 }
624 }
625 return true;
626}
627
628static bool expandCondLoop(Function &Intr) {
629 for (User *U : llvm::make_early_inc_range(Range: Intr.users())) {
630 auto *Call = cast<CallInst>(Val: U);
631
632 auto *Br = cast<UncondBrInst>(
633 Val: SplitBlockAndInsertIfThen(Cond: Call->getArgOperand(i: 0), SplitBefore: Call, Unreachable: false,
634 BranchWeights: getExplicitlyUnknownBranchWeightsIfProfiled(
635 F&: *Call->getFunction(), DEBUG_TYPE)));
636 Br->setSuccessor(Br->getParent());
637 Call->eraseFromParent();
638 }
639 return true;
640}
641
642static bool expandLoopTrap(Function &Intr) {
643 for (User *U : make_early_inc_range(Range: Intr.users())) {
644 auto *Call = cast<CallInst>(Val: U);
645 if (!Call->getParent()->isEntryBlock() &&
646 std::all_of(first: Call->getParent()->begin(), last: BasicBlock::iterator(Call),
647 pred: [](Instruction &I) { return !I.mayHaveSideEffects(); })) {
648 for (auto *BB : predecessors(BB: Call->getParent())) {
649 auto *BI = dyn_cast<CondBrInst>(Val: BB->getTerminator());
650 if (!BI)
651 continue;
652 IRBuilder<> B(BI);
653 Value *Cond;
654 // The looptrap can either be on the true branch or the false branch.
655 // We insert the cond loop before the branch, which uses the branch's
656 // original condition for going to the looptrap as its condition, and
657 // force the branch to take whichever path does not lead to the
658 // looptrap, as the original path to the looptrap is now unreachable
659 // thanks to the cond loop. The codegenprepare pass will clean up our
660 // "unconditional conditional branch" by combining the two basic blocks
661 // if possible, or replacing it with an unconditional branch.
662 if (BI->getSuccessor(i: 0) == Call->getParent()) {
663 // The looptrap is on the true branch.
664 Cond = BI->getCondition();
665 BI->setCondition(ConstantInt::getFalse(Context&: BI->getContext()));
666 } else {
667 // The looptrap is on the false branch, which means that we need to
668 // invert the condition.
669 Cond = B.CreateNot(V: BI->getCondition());
670 BI->setCondition(ConstantInt::getTrue(Context&: BI->getContext()));
671 }
672 B.CreateIntrinsic(ID: Intrinsic::cond_loop, Args: Cond);
673 }
674 }
675 IRBuilder<> B(Call);
676 B.CreateIntrinsic(ID: Intrinsic::cond_loop,
677 Args: ConstantInt::getTrue(Context&: Call->getContext()));
678 Call->eraseFromParent();
679 }
680 return true;
681}
682
683bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
684 // Map unique constants to globals.
685 DenseMap<Constant *, GlobalVariable *> CMap;
686 bool Changed = false;
687 for (Function &F : M) {
688 switch (F.getIntrinsicID()) {
689 default:
690 break;
691 case Intrinsic::memcpy:
692 case Intrinsic::memcpy_inline:
693 case Intrinsic::memmove:
694 case Intrinsic::memset:
695 case Intrinsic::memset_inline:
696 case Intrinsic::experimental_memset_pattern:
697 Changed |= expandMemIntrinsicUses(F, CMap);
698 break;
699 case Intrinsic::load_relative:
700 Changed |= lowerLoadRelative(F);
701 break;
702 case Intrinsic::can_load_speculatively:
703 Changed |= lowerCanLoadSpeculatively(F, TM);
704 break;
705 case Intrinsic::is_constant:
706 case Intrinsic::objectsize:
707 Changed |= forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
708 Function *Parent = CI->getParent()->getParent();
709 TargetLibraryInfo &TLI = LookupTLI(*Parent);
710 // Intrinsics in unreachable code are not lowered.
711 bool Changed = lowerConstantIntrinsics(F&: *Parent, TLI, /*DT=*/nullptr);
712 return Changed;
713 });
714 break;
715#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
716 case Intrinsic::VPID:
717#include "llvm/IR/VPIntrinsics.def"
718 forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
719 Function *Parent = CI->getParent()->getParent();
720 const TargetTransformInfo &TTI = LookupTTI(*Parent);
721 auto *VPI = cast<VPIntrinsic>(Val: CI);
722 VPExpansionDetails ED = expandVectorPredicationIntrinsic(VPI&: *VPI, TTI);
723 // Expansion of VP intrinsics may change the IR but not actually
724 // replace the intrinsic, so update Changed for the pass
725 // and compute Removed for forEachCall.
726 Changed |= ED != VPExpansionDetails::IntrinsicUnchanged;
727 bool Removed = ED == VPExpansionDetails::IntrinsicReplaced;
728 return Removed;
729 });
730 break;
731 case Intrinsic::objc_autorelease:
732 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autorelease);
733 break;
734 case Intrinsic::objc_autoreleasePoolPop:
735 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autoreleasePoolPop);
736 break;
737 case Intrinsic::objc_autoreleasePoolPush:
738 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autoreleasePoolPush);
739 break;
740 case Intrinsic::objc_autoreleaseReturnValue:
741 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autoreleaseReturnValue);
742 break;
743 case Intrinsic::objc_copyWeak:
744 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_copyWeak);
745 break;
746 case Intrinsic::objc_destroyWeak:
747 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_destroyWeak);
748 break;
749 case Intrinsic::objc_initWeak:
750 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_initWeak);
751 break;
752 case Intrinsic::objc_loadWeak:
753 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_loadWeak);
754 break;
755 case Intrinsic::objc_loadWeakRetained:
756 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_loadWeakRetained);
757 break;
758 case Intrinsic::objc_moveWeak:
759 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_moveWeak);
760 break;
761 case Intrinsic::objc_release:
762 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_release, setNonLazyBind: true);
763 break;
764 case Intrinsic::objc_retain:
765 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retain, setNonLazyBind: true);
766 break;
767 case Intrinsic::objc_retainAutorelease:
768 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainAutorelease);
769 break;
770 case Intrinsic::objc_retainAutoreleaseReturnValue:
771 Changed |=
772 lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainAutoreleaseReturnValue);
773 break;
774 case Intrinsic::objc_retainAutoreleasedReturnValue:
775 Changed |=
776 lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainAutoreleasedReturnValue);
777 break;
778 case Intrinsic::objc_claimAutoreleasedReturnValue:
779 Changed |=
780 lowerObjCCall(F, NewFn: RTLIB::impl_objc_claimAutoreleasedReturnValue);
781 break;
782 case Intrinsic::objc_retainBlock:
783 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainBlock);
784 break;
785 case Intrinsic::objc_storeStrong:
786 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_storeStrong);
787 break;
788 case Intrinsic::objc_storeWeak:
789 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_storeWeak);
790 break;
791 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
792 Changed |=
793 lowerObjCCall(F, NewFn: RTLIB::impl_objc_unsafeClaimAutoreleasedReturnValue);
794 break;
795 case Intrinsic::objc_retainedObject:
796 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainedObject);
797 break;
798 case Intrinsic::objc_unretainedObject:
799 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_unretainedObject);
800 break;
801 case Intrinsic::objc_unretainedPointer:
802 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_unretainedPointer);
803 break;
804 case Intrinsic::objc_retain_autorelease:
805 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retain_autorelease);
806 break;
807 case Intrinsic::objc_sync_enter:
808 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_sync_enter);
809 break;
810 case Intrinsic::objc_sync_exit:
811 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_sync_exit);
812 break;
813 case Intrinsic::acos:
814 case Intrinsic::asin:
815 case Intrinsic::atan:
816 case Intrinsic::cos:
817 case Intrinsic::cosh:
818 case Intrinsic::exp:
819 case Intrinsic::exp2:
820 case Intrinsic::exp10:
821 case Intrinsic::log:
822 case Intrinsic::log2:
823 case Intrinsic::log10:
824 case Intrinsic::sin:
825 case Intrinsic::sinh:
826 case Intrinsic::tan:
827 case Intrinsic::tanh:
828 Changed |= forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
829 Type *Ty = CI->getArgOperand(i: 0)->getType();
830 if (!isa<ScalableVectorType>(Val: Ty))
831 return false;
832 const TargetLowering *TL = TM->getSubtargetImpl(F)->getTargetLowering();
833 unsigned Op = TL->IntrinsicIDToISD(ID: F.getIntrinsicID());
834 assert(Op != ISD::DELETED_NODE && "unsupported intrinsic");
835 if (!TL->isOperationExpand(Op, VT: EVT::getEVT(Ty)))
836 return false;
837 return lowerUnaryVectorIntrinsicAsLoop(M, CI);
838 });
839 break;
840 case Intrinsic::modf:
841 case Intrinsic::sincos:
842 case Intrinsic::sincospi:
843 Changed |= forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
844 Type *Ty = CI->getArgOperand(i: 0)->getType();
845 if (!isa<ScalableVectorType>(Val: Ty))
846 return false;
847 const TargetLowering *TL = TM->getSubtargetImpl(F)->getTargetLowering();
848 unsigned Op = TL->IntrinsicIDToISD(ID: F.getIntrinsicID());
849 assert(Op != ISD::DELETED_NODE && "unsupported intrinsic");
850 EVT VT = EVT::getEVT(Ty);
851 if (!TL->isOperationExpand(Op, VT))
852 return false;
853 // The vector legalizer can expand these to a vector math library call.
854 RTLIB::Libcall LC;
855 switch (Op) {
856 case ISD::FMODF:
857 LC = RTLIB::getMODF(VT);
858 break;
859 case ISD::FSINCOS:
860 LC = RTLIB::getSINCOS(VT);
861 break;
862 case ISD::FSINCOSPI:
863 LC = RTLIB::getSINCOSPI(VT);
864 break;
865 default:
866 llvm_unreachable("unexpected intrinsic");
867 }
868 if (TL->getLibcallImpl(Call: LC) != RTLIB::Unsupported)
869 return false;
870 return lowerUnaryVectorIntrinsicAsLoop(M, CI);
871 });
872 break;
873 case Intrinsic::ptrauth_sign:
874 case Intrinsic::ptrauth_auth:
875 Changed |= expandPtrauthForEmuPAC(Intr&: F);
876 break;
877 case Intrinsic::protected_field_ptr:
878 Changed |= expandProtectedFieldPtr(Intr&: F);
879 break;
880 case Intrinsic::cond_loop:
881 if (!TM->canLowerCondLoop())
882 Changed |= expandCondLoop(Intr&: F);
883 break;
884 case Intrinsic::looptrap:
885 Changed |= expandLoopTrap(Intr&: F);
886 if (!TM->canLowerCondLoop())
887 if (auto *CondLoop = M.getFunction(Name: "llvm.cond.loop"))
888 Changed |= expandCondLoop(Intr&: *CondLoop);
889 break;
890 }
891 }
892 return Changed;
893}
894
895namespace {
896
897class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
898public:
899 static char ID;
900
901 PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}
902
903 void getAnalysisUsage(AnalysisUsage &AU) const override {
904 AU.addRequired<TargetTransformInfoWrapperPass>();
905 AU.addRequired<TargetLibraryInfoWrapperPass>();
906 AU.addRequired<LibcallLoweringInfoWrapper>();
907 AU.addRequired<TargetPassConfig>();
908 }
909
910 bool runOnModule(Module &M) override {
911 const ModuleLibcallLoweringInfo &ModuleLibcalls =
912 getAnalysis<LibcallLoweringInfoWrapper>().getResult(M);
913
914 auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {
915 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
916 };
917 auto LookupTLI = [this](Function &F) -> TargetLibraryInfo & {
918 return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
919 };
920
921 const auto *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
922 PreISelIntrinsicLowering Lowering(TM, ModuleLibcalls, LookupTTI, LookupTLI);
923 return Lowering.lowerIntrinsics(M);
924 }
925};
926
927} // end anonymous namespace
928
929char PreISelIntrinsicLoweringLegacyPass::ID;
930
931INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass,
932 "pre-isel-intrinsic-lowering",
933 "Pre-ISel Intrinsic Lowering", false, false)
934INITIALIZE_PASS_DEPENDENCY(LibcallLoweringInfoWrapper)
935INITIALIZE_PASS_DEPENDENCY(RuntimeLibraryInfoWrapper)
936INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
937INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
938INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
939INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
940 "pre-isel-intrinsic-lowering",
941 "Pre-ISel Intrinsic Lowering", false, false)
942
943ModulePass *llvm::createPreISelIntrinsicLoweringPass() {
944 return new PreISelIntrinsicLoweringLegacyPass();
945}
946
947PreservedAnalyses
948PreISelIntrinsicLoweringPass::run(Module &M, ModuleAnalysisManager &MAM) {
949 const ModuleLibcallLoweringInfo &LibcallLowering =
950 MAM.getResult<LibcallLoweringModuleAnalysis>(IR&: M);
951
952 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
953
954 auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {
955 return FAM.getResult<TargetIRAnalysis>(IR&: F);
956 };
957 auto LookupTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
958 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
959 };
960
961 PreISelIntrinsicLowering Lowering(TM, LibcallLowering, LookupTTI, LookupTLI);
962 if (!Lowering.lowerIntrinsics(M))
963 return PreservedAnalyses::all();
964 else
965 return PreservedAnalyses::none();
966}
967