1//===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass implements IR lowering for the llvm.memcpy, llvm.memmove,
10// llvm.memset, llvm.load.relative and llvm.objc.* intrinsics.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/Analysis/ObjCARCInstKind.h"
17#include "llvm/Analysis/ObjCARCUtil.h"
18#include "llvm/Analysis/TargetLibraryInfo.h"
19#include "llvm/Analysis/TargetTransformInfo.h"
20#include "llvm/CodeGen/ExpandVectorPredication.h"
21#include "llvm/CodeGen/LibcallLoweringInfo.h"
22#include "llvm/CodeGen/Passes.h"
23#include "llvm/CodeGen/TargetLowering.h"
24#include "llvm/CodeGen/TargetPassConfig.h"
25#include "llvm/IR/Function.h"
26#include "llvm/IR/GlobalValue.h"
27#include "llvm/IR/IRBuilder.h"
28#include "llvm/IR/Instructions.h"
29#include "llvm/IR/IntrinsicInst.h"
30#include "llvm/IR/Metadata.h"
31#include "llvm/IR/Module.h"
32#include "llvm/IR/ProfDataUtils.h"
33#include "llvm/IR/RuntimeLibcalls.h"
34#include "llvm/IR/Type.h"
35#include "llvm/IR/Use.h"
36#include "llvm/InitializePasses.h"
37#include "llvm/Pass.h"
38#include "llvm/Support/Casting.h"
39#include "llvm/Target/TargetMachine.h"
40#include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
41#include "llvm/Transforms/Utils/BasicBlockUtils.h"
42#include "llvm/Transforms/Utils/BuildLibCalls.h"
43#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
44#include "llvm/Transforms/Utils/LowerVectorIntrinsics.h"
45
46using namespace llvm;
47
48#define DEBUG_TYPE "pre-isel-intrinsic-lowering"
49
50/// Threshold to leave statically sized memory intrinsic calls. Calls of known
51/// size larger than this will be expanded by the pass. Calls of unknown or
52/// lower size will be left for expansion in codegen.
53static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt(
54 "mem-intrinsic-expand-size",
55 cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(Val: -1),
56 cl::Hidden);
57
58namespace {
59
60struct PreISelIntrinsicLowering {
61 const TargetMachine *TM;
62 const ModuleLibcallLoweringInfo &ModuleLibcalls;
63 const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
64 const function_ref<TargetLibraryInfo &(Function &)> LookupTLI;
65
66 /// If this is true, assume it's preferably to leave memory intrinsic calls
67 /// for replacement with a library call later. Otherwise this depends on
68 /// TargetLoweringInfo availability of the corresponding function.
69 const bool UseMemIntrinsicLibFunc;
70
71 explicit PreISelIntrinsicLowering(
72 const TargetMachine *TM_,
73 const ModuleLibcallLoweringInfo &ModuleLibcalls_,
74 function_ref<TargetTransformInfo &(Function &)> LookupTTI_,
75 function_ref<TargetLibraryInfo &(Function &)> LookupTLI_,
76 bool UseMemIntrinsicLibFunc_ = true)
77 : TM(TM_), ModuleLibcalls(ModuleLibcalls_), LookupTTI(LookupTTI_),
78 LookupTLI(LookupTLI_), UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {
79 }
80
81 static bool shouldExpandMemIntrinsicWithSize(Value *Size,
82 const TargetTransformInfo &TTI);
83 bool
84 expandMemIntrinsicUses(Function &F,
85 DenseMap<Constant *, GlobalVariable *> &CMap) const;
86 bool lowerIntrinsics(Module &M) const;
87};
88
89} // namespace
90
91template <class T> static bool forEachCall(Function &Intrin, T Callback) {
92 // Lowering all intrinsics in a function will delete multiple uses, so we
93 // can't use an early-inc-range. In case some remain, we don't want to look
94 // at them again. Unfortunately, Value::UseList is private, so we can't use a
95 // simple Use**. If LastUse is null, the next use to consider is
96 // Intrin.use_begin(), otherwise it's LastUse->getNext().
97 Use *LastUse = nullptr;
98 bool Changed = false;
99 while (!Intrin.use_empty() && (!LastUse || LastUse->getNext())) {
100 Use *U = LastUse ? LastUse->getNext() : &*Intrin.use_begin();
101 bool Removed = false;
102 // An intrinsic cannot have its address taken, so it cannot be an argument
103 // operand. It might be used as operand in debug metadata, though.
104 if (auto CI = dyn_cast<CallInst>(Val: U->getUser()))
105 Changed |= Removed = Callback(CI);
106 if (!Removed)
107 LastUse = U;
108 }
109 return Changed;
110}
111
112static bool lowerLoadRelative(Function &F) {
113 if (F.use_empty())
114 return false;
115
116 bool Changed = false;
117 Type *Int32Ty = Type::getInt32Ty(C&: F.getContext());
118
119 for (Use &U : llvm::make_early_inc_range(Range: F.uses())) {
120 auto CI = dyn_cast<CallInst>(Val: U.getUser());
121 if (!CI || CI->getCalledOperand() != &F)
122 continue;
123
124 IRBuilder<> B(CI);
125 Value *OffsetPtr =
126 B.CreatePtrAdd(Ptr: CI->getArgOperand(i: 0), Offset: CI->getArgOperand(i: 1));
127 Value *OffsetI32 = B.CreateAlignedLoad(Ty: Int32Ty, Ptr: OffsetPtr, Align: Align(4));
128
129 Value *ResultPtr = B.CreatePtrAdd(Ptr: CI->getArgOperand(i: 0), Offset: OffsetI32);
130
131 CI->replaceAllUsesWith(V: ResultPtr);
132 CI->eraseFromParent();
133 Changed = true;
134 }
135
136 return Changed;
137}
138
139// ObjCARC has knowledge about whether an obj-c runtime function needs to be
140// always tail-called or never tail-called.
141static CallInst::TailCallKind getOverridingTailCallKind(const Function &F) {
142 objcarc::ARCInstKind Kind = objcarc::GetFunctionClass(F: &F);
143 if (objcarc::IsAlwaysTail(Class: Kind))
144 return CallInst::TCK_Tail;
145 else if (objcarc::IsNeverTail(Class: Kind))
146 return CallInst::TCK_NoTail;
147 return CallInst::TCK_None;
148}
149
150static bool lowerObjCCall(Function &F, RTLIB::LibcallImpl NewFn,
151 bool setNonLazyBind = false) {
152 assert(IntrinsicInst::mayLowerToFunctionCall(F.getIntrinsicID()) &&
153 "Pre-ISel intrinsics do lower into regular function calls");
154 if (F.use_empty())
155 return false;
156
157 // FIXME: When RuntimeLibcalls is an analysis, check if the function is really
158 // supported, and go through RTLIB::Libcall.
159 StringRef NewFnName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: NewFn);
160
161 // If we haven't already looked up this function, check to see if the
162 // program already contains a function with this name.
163 Module *M = F.getParent();
164 FunctionCallee FCache =
165 M->getOrInsertFunction(Name: NewFnName, T: F.getFunctionType());
166
167 if (Function *Fn = dyn_cast<Function>(Val: FCache.getCallee())) {
168 Fn->setLinkage(F.getLinkage());
169 if (setNonLazyBind && !Fn->isWeakForLinker()) {
170 // If we have Native ARC, set nonlazybind attribute for these APIs for
171 // performance.
172 Fn->addFnAttr(Kind: Attribute::NonLazyBind);
173 }
174 }
175
176 CallInst::TailCallKind OverridingTCK = getOverridingTailCallKind(F);
177
178 for (Use &U : llvm::make_early_inc_range(Range: F.uses())) {
179 auto *CB = cast<CallBase>(Val: U.getUser());
180
181 if (CB->getCalledFunction() != &F) {
182 assert(objcarc::getAttachedARCFunction(CB) == &F &&
183 "use expected to be the argument of operand bundle "
184 "\"clang.arc.attachedcall\"");
185 U.set(FCache.getCallee());
186 continue;
187 }
188
189 auto *CI = cast<CallInst>(Val: CB);
190 assert(CI->getCalledFunction() && "Cannot lower an indirect call!");
191
192 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
193 SmallVector<Value *, 8> Args(CI->args());
194 SmallVector<llvm::OperandBundleDef, 1> BundleList;
195 CI->getOperandBundlesAsDefs(Defs&: BundleList);
196 CallInst *NewCI = Builder.CreateCall(Callee: FCache, Args, OpBundles: BundleList);
197 NewCI->setName(CI->getName());
198
199 // Try to set the most appropriate TailCallKind based on both the current
200 // attributes and the ones that we could get from ObjCARC's special
201 // knowledge of the runtime functions.
202 //
203 // std::max respects both requirements of notail and tail here:
204 // * notail on either the call or from ObjCARC becomes notail
205 // * tail on either side is stronger than none, but not notail
206 CallInst::TailCallKind TCK = CI->getTailCallKind();
207 NewCI->setTailCallKind(std::max(a: TCK, b: OverridingTCK));
208
209 // Transfer the 'returned' attribute from the intrinsic to the call site.
210 // By applying this only to intrinsic call sites, we avoid applying it to
211 // non-ARC explicit calls to things like objc_retain which have not been
212 // auto-upgraded to use the intrinsics.
213 unsigned Index;
214 if (F.getAttributes().hasAttrSomewhere(Kind: Attribute::Returned, Index: &Index) &&
215 Index)
216 NewCI->addParamAttr(ArgNo: Index - AttributeList::FirstArgIndex,
217 Kind: Attribute::Returned);
218
219 if (!CI->use_empty())
220 CI->replaceAllUsesWith(V: NewCI);
221 CI->eraseFromParent();
222 }
223
224 return true;
225}
226
227// TODO: Should refine based on estimated number of accesses (e.g. does it
228// require splitting based on alignment)
229bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(
230 Value *Size, const TargetTransformInfo &TTI) {
231 ConstantInt *CI = dyn_cast<ConstantInt>(Val: Size);
232 if (!CI)
233 return true;
234 uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences()
235 ? MemIntrinsicExpandSizeThresholdOpt
236 : TTI.getMaxMemIntrinsicInlineSizeThreshold();
237 uint64_t SizeVal = CI->getZExtValue();
238
239 // Treat a threshold of 0 as a special case to force expansion of all
240 // intrinsics, including size 0.
241 return SizeVal > Threshold || Threshold == 0;
242}
243
244static bool canEmitLibcall(const ModuleLibcallLoweringInfo &ModuleLowering,
245 const TargetMachine *TM, Function *F,
246 RTLIB::Libcall LC) {
247 // TODO: Should this consider the address space of the memcpy?
248 if (!TM)
249 return true;
250 const LibcallLoweringInfo &Lowering =
251 ModuleLowering.getLibcallLowering(Subtarget: *TM->getSubtargetImpl(*F));
252 return Lowering.getLibcallImpl(Call: LC) != RTLIB::Unsupported;
253}
254
255static bool canEmitMemcpy(const ModuleLibcallLoweringInfo &ModuleLowering,
256 const TargetMachine *TM, Function *F) {
257 // TODO: Should this consider the address space of the memcpy?
258 if (!TM)
259 return true;
260 const LibcallLoweringInfo &Lowering =
261 ModuleLowering.getLibcallLowering(Subtarget: *TM->getSubtargetImpl(*F));
262 return Lowering.getMemcpyImpl() != RTLIB::Unsupported;
263}
264
265// Return a value appropriate for use with the memset_pattern16 libcall, if
266// possible and if we know how. (Adapted from equivalent helper in
267// LoopIdiomRecognize).
268static Constant *getMemSetPattern16Value(MemSetPatternInst *Inst,
269 const TargetLibraryInfo &TLI) {
270 // TODO: This could check for UndefValue because it can be merged into any
271 // other valid pattern.
272
273 // Don't emit libcalls if a non-default address space is being used.
274 if (Inst->getRawDest()->getType()->getPointerAddressSpace() != 0)
275 return nullptr;
276
277 Value *V = Inst->getValue();
278 Type *VTy = V->getType();
279 const DataLayout &DL = Inst->getDataLayout();
280 Module *M = Inst->getModule();
281
282 if (!isLibFuncEmittable(M, TLI: &TLI, TheLibFunc: LibFunc_memset_pattern16))
283 return nullptr;
284
285 // If the value isn't a constant, we can't promote it to being in a constant
286 // array. We could theoretically do a store to an alloca or something, but
287 // that doesn't seem worthwhile.
288 Constant *C = dyn_cast<Constant>(Val: V);
289 if (!C || isa<ConstantExpr>(Val: C))
290 return nullptr;
291
292 // Only handle simple values that are a power of two bytes in size.
293 uint64_t Size = DL.getTypeSizeInBits(Ty: VTy);
294 if (!DL.typeSizeEqualsStoreSize(Ty: VTy) || !isPowerOf2_64(Value: Size))
295 return nullptr;
296
297 // Don't care enough about darwin/ppc to implement this.
298 if (DL.isBigEndian())
299 return nullptr;
300
301 // Convert to size in bytes.
302 Size /= 8;
303
304 // TODO: If CI is larger than 16-bytes, we can try slicing it in half to see
305 // if the top and bottom are the same (e.g. for vectors and large integers).
306 if (Size > 16)
307 return nullptr;
308
309 // If the constant is exactly 16 bytes, just use it.
310 if (Size == 16)
311 return C;
312
313 // Otherwise, we'll use an array of the constants.
314 uint64_t ArraySize = 16 / Size;
315 ArrayType *AT = ArrayType::get(ElementType: V->getType(), NumElements: ArraySize);
316 return ConstantArray::get(T: AT, V: std::vector<Constant *>(ArraySize, C));
317}
318
319// TODO: Handle atomic memcpy and memcpy.inline
320// TODO: Pass ScalarEvolution
321bool PreISelIntrinsicLowering::expandMemIntrinsicUses(
322 Function &F, DenseMap<Constant *, GlobalVariable *> &CMap) const {
323 Intrinsic::ID ID = F.getIntrinsicID();
324 bool Changed = false;
325
326 for (User *U : llvm::make_early_inc_range(Range: F.users())) {
327 Instruction *Inst = cast<Instruction>(Val: U);
328
329 switch (ID) {
330 case Intrinsic::memcpy: {
331 auto *Memcpy = cast<MemCpyInst>(Val: Inst);
332 Function *ParentFunc = Memcpy->getFunction();
333 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
334 if (shouldExpandMemIntrinsicWithSize(Size: Memcpy->getLength(), TTI)) {
335 if (UseMemIntrinsicLibFunc &&
336 canEmitMemcpy(ModuleLowering: ModuleLibcalls, TM, F: ParentFunc))
337 break;
338
339 // TODO: For optsize, emit the loop into a separate function
340 expandMemCpyAsLoop(MemCpy: Memcpy, TTI);
341 Changed = true;
342 Memcpy->eraseFromParent();
343 }
344
345 break;
346 }
347 case Intrinsic::memcpy_inline: {
348 // Only expand llvm.memcpy.inline with non-constant length in this
349 // codepath, leaving the current SelectionDAG expansion for constant
350 // length memcpy intrinsics undisturbed.
351 auto *Memcpy = cast<MemCpyInst>(Val: Inst);
352 if (isa<ConstantInt>(Val: Memcpy->getLength()))
353 break;
354
355 Function *ParentFunc = Memcpy->getFunction();
356 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
357 expandMemCpyAsLoop(MemCpy: Memcpy, TTI);
358 Changed = true;
359 Memcpy->eraseFromParent();
360 break;
361 }
362 case Intrinsic::memmove: {
363 auto *Memmove = cast<MemMoveInst>(Val: Inst);
364 Function *ParentFunc = Memmove->getFunction();
365 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
366 if (shouldExpandMemIntrinsicWithSize(Size: Memmove->getLength(), TTI)) {
367 if (UseMemIntrinsicLibFunc &&
368 canEmitLibcall(ModuleLowering: ModuleLibcalls, TM, F: ParentFunc, LC: RTLIB::MEMMOVE))
369 break;
370
371 if (expandMemMoveAsLoop(MemMove: Memmove, TTI)) {
372 Changed = true;
373 Memmove->eraseFromParent();
374 }
375 }
376
377 break;
378 }
379 case Intrinsic::memset: {
380 auto *Memset = cast<MemSetInst>(Val: Inst);
381 Function *ParentFunc = Memset->getFunction();
382 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
383 if (shouldExpandMemIntrinsicWithSize(Size: Memset->getLength(), TTI)) {
384 if (UseMemIntrinsicLibFunc &&
385 canEmitLibcall(ModuleLowering: ModuleLibcalls, TM, F: ParentFunc, LC: RTLIB::MEMSET))
386 break;
387
388 expandMemSetAsLoop(MemSet: Memset, TTI);
389 Changed = true;
390 Memset->eraseFromParent();
391 }
392
393 break;
394 }
395 case Intrinsic::memset_inline: {
396 // Only expand llvm.memset.inline with non-constant length in this
397 // codepath, leaving the current SelectionDAG expansion for constant
398 // length memset intrinsics undisturbed.
399 auto *Memset = cast<MemSetInst>(Val: Inst);
400 if (isa<ConstantInt>(Val: Memset->getLength()))
401 break;
402
403 Function *ParentFunc = Memset->getFunction();
404 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
405 expandMemSetAsLoop(MemSet: Memset, TTI);
406 Changed = true;
407 Memset->eraseFromParent();
408 break;
409 }
410 case Intrinsic::experimental_memset_pattern: {
411 auto *Memset = cast<MemSetPatternInst>(Val: Inst);
412 Function *ParentFunc = Memset->getFunction();
413 const TargetLibraryInfo &TLI = LookupTLI(*ParentFunc);
414 Constant *PatternValue = getMemSetPattern16Value(Inst: Memset, TLI);
415 if (!PatternValue) {
416 // If it isn't possible to emit a memset_pattern16 libcall, expand to
417 // a loop instead.
418 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
419 expandMemSetPatternAsLoop(MemSet: Memset, TTI);
420 Changed = true;
421 Memset->eraseFromParent();
422 break;
423 }
424 // FIXME: There is currently no profitability calculation for emitting
425 // the libcall vs expanding the memset.pattern directly.
426 IRBuilder<> Builder(Inst);
427 Module *M = Memset->getModule();
428 const DataLayout &DL = Memset->getDataLayout();
429
430 Type *DestPtrTy = Memset->getRawDest()->getType();
431 Type *SizeTTy = TLI.getSizeTType(M: *M);
432 StringRef FuncName = "memset_pattern16";
433 FunctionCallee MSP = getOrInsertLibFunc(M, TLI, TheLibFunc: LibFunc_memset_pattern16,
434 RetTy: Builder.getVoidTy(), Args: DestPtrTy,
435 Args: Builder.getPtrTy(), Args: SizeTTy);
436 inferNonMandatoryLibFuncAttrs(M, Name: FuncName, TLI);
437
438 // Otherwise we should form a memset_pattern16. PatternValue is known
439 // to be an constant array of 16-bytes. Put the value into a mergable
440 // global.
441 assert(Memset->getRawDest()->getType()->getPointerAddressSpace() == 0 &&
442 "Should have skipped if non-zero AS");
443 GlobalVariable *GV;
444 auto It = CMap.find(Val: PatternValue);
445 if (It != CMap.end()) {
446 GV = It->second;
447 } else {
448 GV = new GlobalVariable(
449 *M, PatternValue->getType(), /*isConstant=*/true,
450 GlobalValue::PrivateLinkage, PatternValue, ".memset_pattern");
451 GV->setUnnamedAddr(
452 GlobalValue::UnnamedAddr::Global); // Ok to merge these.
453 // TODO: Consider relaxing alignment requirement.
454 GV->setAlignment(Align(16));
455 CMap[PatternValue] = GV;
456 }
457 Value *PatternPtr = GV;
458 Value *NumBytes = Builder.CreateMul(
459 LHS: TLI.getAsSizeT(V: DL.getTypeAllocSize(Ty: Memset->getValue()->getType()),
460 M: *M),
461 RHS: Builder.CreateZExtOrTrunc(V: Memset->getLength(), DestTy: SizeTTy));
462 CallInst *MemsetPattern16Call =
463 Builder.CreateCall(Callee: MSP, Args: {Memset->getRawDest(), PatternPtr, NumBytes});
464 MemsetPattern16Call->setAAMetadata(Memset->getAAMetadata());
465 // Preserve any call site attributes on the destination pointer
466 // argument (e.g. alignment).
467 AttrBuilder ArgAttrs(Memset->getContext(),
468 Memset->getAttributes().getParamAttrs(ArgNo: 0));
469 MemsetPattern16Call->setAttributes(
470 MemsetPattern16Call->getAttributes().addParamAttributes(
471 C&: Memset->getContext(), ArgNo: 0, B: ArgAttrs));
472 Changed = true;
473 Memset->eraseFromParent();
474 break;
475 }
476 default:
477 llvm_unreachable("unhandled intrinsic");
478 }
479 }
480
481 return Changed;
482}
483
484static GlobalValue *getDeactivationSymbol(CallInst *Call) {
485 if (auto Bundle = Call->getOperandBundle(ID: LLVMContext::OB_deactivation_symbol))
486 return cast<GlobalValue>(Val: Bundle->Inputs[0]);
487 return nullptr;
488}
489
490static bool expandPtrauthForEmuPAC(Function &Intr) {
491 Module &M = *Intr.getParent();
492 if (Triple(M.getTargetTriple()).isArm64e())
493 return false;
494
495 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
496
497 assert(Intr.getIntrinsicID() == Intrinsic::ptrauth_sign ||
498 Intr.getIntrinsicID() == Intrinsic::ptrauth_auth);
499 auto *EmuFnTy = FunctionType::get(Result: Int64Ty, Params: {Int64Ty, Int64Ty}, isVarArg: false);
500 FunctionCallee EmuIntr = M.getOrInsertFunction(
501 Name: Intr.getIntrinsicID() == Intrinsic::ptrauth_auth ? "__emupac_autda"
502 : "__emupac_pacda",
503 T: EmuFnTy);
504
505 for (User *U : llvm::make_early_inc_range(Range: Intr.users())) {
506 auto *Call = cast<CallInst>(Val: U);
507 // We only support the DA key for now.
508 if (auto *Key = dyn_cast<ConstantInt>(Val: Call->getArgOperand(i: 1));
509 !Key || Key->getZExtValue() != /*AArch64PACKey::DA*/ 2)
510 continue;
511
512 Function *F = Call->getParent()->getParent();
513 Attribute FSAttr = F->getFnAttribute(Kind: "target-features");
514 if (FSAttr.isValid() && FSAttr.getValueAsString().contains(Other: "+pauth"))
515 continue;
516
517 std::vector<OperandBundleDef> DSBundle;
518 if (auto *DS = getDeactivationSymbol(Call))
519 DSBundle.push_back(x: OperandBundleDef("deactivation-symbol", DS));
520
521 IRBuilder<> B(Call);
522 auto *EmuCall = B.CreateCall(
523 Callee: EmuIntr, Args: {Call->getArgOperand(i: 0), Call->getArgOperand(i: 2)}, OpBundles: DSBundle);
524 Call->replaceAllUsesWith(V: EmuCall);
525 Call->eraseFromParent();
526 }
527 return true;
528}
529
530static bool expandProtectedFieldPtr(Function &Intr) {
531 Module &M = *Intr.getParent();
532
533 SmallPtrSet<GlobalValue *, 2> DSsToDeactivate;
534
535 Type *Int8Ty = Type::getInt8Ty(C&: M.getContext());
536 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
537 PointerType *PtrTy = PointerType::get(C&: M.getContext(), AddressSpace: 0);
538
539 for (User *U : llvm::make_early_inc_range(Range: Intr.users())) {
540 auto *Call = cast<CallInst>(Val: U);
541
542 auto *Pointer = Call->getArgOperand(i: 0);
543 bool UseHWEncoding =
544 cast<ConstantInt>(Val: Call->getArgOperand(i: 2))->getZExtValue();
545 if (!UseHWEncoding)
546 reportFatalUsageError(reason: "software encoding currently unsupported");
547
548 auto *DS = getDeactivationSymbol(Call);
549 OperandBundleDef DSBundle("deactivation-symbol", DS);
550
551 for (Use &U : llvm::make_early_inc_range(Range: Call->uses())) {
552 // Comparisons against null cannot be used to recover the original
553 // pointer so we replace them with comparisons against the original
554 // pointer.
555 if (auto *CI = dyn_cast<ICmpInst>(Val: U.getUser())) {
556 if (auto *Op = dyn_cast<Constant>(Val: CI->getOperand(i_nocapture: 0))) {
557 if (Op->isNullValue()) {
558 CI->setOperand(i_nocapture: 1, Val_nocapture: Pointer);
559 continue;
560 }
561 }
562 if (auto *Op = dyn_cast<Constant>(Val: CI->getOperand(i_nocapture: 1))) {
563 if (Op->isNullValue()) {
564 CI->setOperand(i_nocapture: 0, Val_nocapture: Pointer);
565 continue;
566 }
567 }
568 }
569
570 // If we are here, this means that we couldn't rewrite away this use of
571 // the intrinsic. Any load or store uses were removed by InstCombine, and
572 // in general, we can't rewrite away non-load/store uses of
573 // llvm.protected.field.ptr because doing so could expose the encoded
574 // pointer value to the program. Replace it with the pointer operand, and
575 // arrange to define a deactivation symbol.
576 U.set(Pointer);
577 if (DS)
578 DSsToDeactivate.insert(Ptr: DS);
579 }
580
581 Call->eraseFromParent();
582 }
583
584 if (!DSsToDeactivate.empty()) {
585 // This is an AArch64 NOP instruction. When the deactivation symbol support
586 // is expanded to more architectures, there will likely need to be an API
587 // for retrieving this constant.
588 Constant *Nop =
589 ConstantExpr::getIntToPtr(C: ConstantInt::get(Ty: Int64Ty, V: 0xd503201f), Ty: PtrTy);
590 for (GlobalValue *OldDS : DSsToDeactivate) {
591 GlobalValue *DS = GlobalAlias::create(
592 Ty: Int8Ty, AddressSpace: 0, Linkage: GlobalValue::ExternalLinkage, Name: OldDS->getName(), Aliasee: Nop, Parent: &M);
593 DS->setVisibility(GlobalValue::HiddenVisibility);
594 DS->takeName(V: OldDS);
595 OldDS->replaceAllUsesWith(V: DS);
596 OldDS->eraseFromParent();
597 }
598 }
599 return true;
600}
601
602static bool expandCondLoop(Function &Intr) {
603 for (User *U : llvm::make_early_inc_range(Range: Intr.users())) {
604 auto *Call = cast<CallInst>(Val: U);
605
606 auto *Br = cast<UncondBrInst>(
607 Val: SplitBlockAndInsertIfThen(Cond: Call->getArgOperand(i: 0), SplitBefore: Call, Unreachable: false,
608 BranchWeights: getExplicitlyUnknownBranchWeightsIfProfiled(
609 F&: *Call->getFunction(), DEBUG_TYPE)));
610 Br->setSuccessor(Br->getParent());
611 Call->eraseFromParent();
612 }
613 return true;
614}
615
616static bool expandLoopTrap(Function &Intr) {
617 for (User *U : make_early_inc_range(Range: Intr.users())) {
618 auto *Call = cast<CallInst>(Val: U);
619 if (!Call->getParent()->isEntryBlock() &&
620 std::all_of(first: Call->getParent()->begin(), last: BasicBlock::iterator(Call),
621 pred: [](Instruction &I) { return !I.mayHaveSideEffects(); })) {
622 for (auto *BB : predecessors(BB: Call->getParent())) {
623 auto *BI = dyn_cast<CondBrInst>(Val: BB->getTerminator());
624 if (!BI)
625 continue;
626 IRBuilder<> B(BI);
627 Value *Cond;
628 // The looptrap can either be on the true branch or the false branch.
629 // We insert the cond loop before the branch, which uses the branch's
630 // original condition for going to the looptrap as its condition, and
631 // force the branch to take whichever path does not lead to the
632 // looptrap, as the original path to the looptrap is now unreachable
633 // thanks to the cond loop. The codegenprepare pass will clean up our
634 // "unconditional conditional branch" by combining the two basic blocks
635 // if possible, or replacing it with an unconditional branch.
636 if (BI->getSuccessor(i: 0) == Call->getParent()) {
637 // The looptrap is on the true branch.
638 Cond = BI->getCondition();
639 BI->setCondition(ConstantInt::getFalse(Context&: BI->getContext()));
640 } else {
641 // The looptrap is on the false branch, which means that we need to
642 // invert the condition.
643 Cond = B.CreateNot(V: BI->getCondition());
644 BI->setCondition(ConstantInt::getTrue(Context&: BI->getContext()));
645 }
646 B.CreateIntrinsic(ID: Intrinsic::cond_loop, Args: Cond);
647 }
648 }
649 IRBuilder<> B(Call);
650 B.CreateIntrinsic(ID: Intrinsic::cond_loop,
651 Args: ConstantInt::getTrue(Context&: Call->getContext()));
652 Call->eraseFromParent();
653 }
654 return true;
655}
656
657bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
658 // Map unique constants to globals.
659 DenseMap<Constant *, GlobalVariable *> CMap;
660 bool Changed = false;
661 for (Function &F : M) {
662 switch (F.getIntrinsicID()) {
663 default:
664 break;
665 case Intrinsic::memcpy:
666 case Intrinsic::memcpy_inline:
667 case Intrinsic::memmove:
668 case Intrinsic::memset:
669 case Intrinsic::memset_inline:
670 case Intrinsic::experimental_memset_pattern:
671 Changed |= expandMemIntrinsicUses(F, CMap);
672 break;
673 case Intrinsic::load_relative:
674 Changed |= lowerLoadRelative(F);
675 break;
676 case Intrinsic::is_constant:
677 case Intrinsic::objectsize:
678 Changed |= forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
679 Function *Parent = CI->getParent()->getParent();
680 TargetLibraryInfo &TLI = LookupTLI(*Parent);
681 // Intrinsics in unreachable code are not lowered.
682 bool Changed = lowerConstantIntrinsics(F&: *Parent, TLI, /*DT=*/nullptr);
683 return Changed;
684 });
685 break;
686#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
687 case Intrinsic::VPID:
688#include "llvm/IR/VPIntrinsics.def"
689 forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
690 Function *Parent = CI->getParent()->getParent();
691 const TargetTransformInfo &TTI = LookupTTI(*Parent);
692 auto *VPI = cast<VPIntrinsic>(Val: CI);
693 VPExpansionDetails ED = expandVectorPredicationIntrinsic(VPI&: *VPI, TTI);
694 // Expansion of VP intrinsics may change the IR but not actually
695 // replace the intrinsic, so update Changed for the pass
696 // and compute Removed for forEachCall.
697 Changed |= ED != VPExpansionDetails::IntrinsicUnchanged;
698 bool Removed = ED == VPExpansionDetails::IntrinsicReplaced;
699 return Removed;
700 });
701 break;
702 case Intrinsic::objc_autorelease:
703 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autorelease);
704 break;
705 case Intrinsic::objc_autoreleasePoolPop:
706 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autoreleasePoolPop);
707 break;
708 case Intrinsic::objc_autoreleasePoolPush:
709 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autoreleasePoolPush);
710 break;
711 case Intrinsic::objc_autoreleaseReturnValue:
712 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autoreleaseReturnValue);
713 break;
714 case Intrinsic::objc_copyWeak:
715 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_copyWeak);
716 break;
717 case Intrinsic::objc_destroyWeak:
718 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_destroyWeak);
719 break;
720 case Intrinsic::objc_initWeak:
721 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_initWeak);
722 break;
723 case Intrinsic::objc_loadWeak:
724 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_loadWeak);
725 break;
726 case Intrinsic::objc_loadWeakRetained:
727 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_loadWeakRetained);
728 break;
729 case Intrinsic::objc_moveWeak:
730 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_moveWeak);
731 break;
732 case Intrinsic::objc_release:
733 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_release, setNonLazyBind: true);
734 break;
735 case Intrinsic::objc_retain:
736 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retain, setNonLazyBind: true);
737 break;
738 case Intrinsic::objc_retainAutorelease:
739 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainAutorelease);
740 break;
741 case Intrinsic::objc_retainAutoreleaseReturnValue:
742 Changed |=
743 lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainAutoreleaseReturnValue);
744 break;
745 case Intrinsic::objc_retainAutoreleasedReturnValue:
746 Changed |=
747 lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainAutoreleasedReturnValue);
748 break;
749 case Intrinsic::objc_claimAutoreleasedReturnValue:
750 Changed |=
751 lowerObjCCall(F, NewFn: RTLIB::impl_objc_claimAutoreleasedReturnValue);
752 break;
753 case Intrinsic::objc_retainBlock:
754 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainBlock);
755 break;
756 case Intrinsic::objc_storeStrong:
757 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_storeStrong);
758 break;
759 case Intrinsic::objc_storeWeak:
760 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_storeWeak);
761 break;
762 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
763 Changed |=
764 lowerObjCCall(F, NewFn: RTLIB::impl_objc_unsafeClaimAutoreleasedReturnValue);
765 break;
766 case Intrinsic::objc_retainedObject:
767 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainedObject);
768 break;
769 case Intrinsic::objc_unretainedObject:
770 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_unretainedObject);
771 break;
772 case Intrinsic::objc_unretainedPointer:
773 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_unretainedPointer);
774 break;
775 case Intrinsic::objc_retain_autorelease:
776 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retain_autorelease);
777 break;
778 case Intrinsic::objc_sync_enter:
779 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_sync_enter);
780 break;
781 case Intrinsic::objc_sync_exit:
782 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_sync_exit);
783 break;
784 case Intrinsic::acos:
785 case Intrinsic::asin:
786 case Intrinsic::atan:
787 case Intrinsic::cos:
788 case Intrinsic::cosh:
789 case Intrinsic::exp:
790 case Intrinsic::exp2:
791 case Intrinsic::exp10:
792 case Intrinsic::log:
793 case Intrinsic::log2:
794 case Intrinsic::log10:
795 case Intrinsic::sin:
796 case Intrinsic::sinh:
797 case Intrinsic::tan:
798 case Intrinsic::tanh:
799 Changed |= forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
800 Type *Ty = CI->getArgOperand(i: 0)->getType();
801 if (!TM || !isa<ScalableVectorType>(Val: Ty))
802 return false;
803 const TargetLowering *TL = TM->getSubtargetImpl(F)->getTargetLowering();
804 unsigned Op = TL->IntrinsicIDToISD(ID: F.getIntrinsicID());
805 assert(Op != ISD::DELETED_NODE && "unsupported intrinsic");
806 if (!TL->isOperationExpand(Op, VT: EVT::getEVT(Ty)))
807 return false;
808 return lowerUnaryVectorIntrinsicAsLoop(M, CI);
809 });
810 break;
811 case Intrinsic::ptrauth_sign:
812 case Intrinsic::ptrauth_auth:
813 Changed |= expandPtrauthForEmuPAC(Intr&: F);
814 break;
815 case Intrinsic::protected_field_ptr:
816 Changed |= expandProtectedFieldPtr(Intr&: F);
817 break;
818 case Intrinsic::cond_loop:
819 if (!TM->canLowerCondLoop())
820 Changed |= expandCondLoop(Intr&: F);
821 break;
822 case Intrinsic::looptrap:
823 Changed |= expandLoopTrap(Intr&: F);
824 if (!TM->canLowerCondLoop())
825 if (auto *CondLoop = M.getFunction(Name: "llvm.cond.loop"))
826 Changed |= expandCondLoop(Intr&: *CondLoop);
827 break;
828 }
829 }
830 return Changed;
831}
832
833namespace {
834
835class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
836public:
837 static char ID;
838
839 PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}
840
841 void getAnalysisUsage(AnalysisUsage &AU) const override {
842 AU.addRequired<TargetTransformInfoWrapperPass>();
843 AU.addRequired<TargetLibraryInfoWrapperPass>();
844 AU.addRequired<LibcallLoweringInfoWrapper>();
845 AU.addRequired<TargetPassConfig>();
846 }
847
848 bool runOnModule(Module &M) override {
849 const ModuleLibcallLoweringInfo &ModuleLibcalls =
850 getAnalysis<LibcallLoweringInfoWrapper>().getResult(M);
851
852 auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {
853 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
854 };
855 auto LookupTLI = [this](Function &F) -> TargetLibraryInfo & {
856 return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
857 };
858
859 const auto *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
860 PreISelIntrinsicLowering Lowering(TM, ModuleLibcalls, LookupTTI, LookupTLI);
861 return Lowering.lowerIntrinsics(M);
862 }
863};
864
865} // end anonymous namespace
866
867char PreISelIntrinsicLoweringLegacyPass::ID;
868
869INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass,
870 "pre-isel-intrinsic-lowering",
871 "Pre-ISel Intrinsic Lowering", false, false)
872INITIALIZE_PASS_DEPENDENCY(LibcallLoweringInfoWrapper)
873INITIALIZE_PASS_DEPENDENCY(RuntimeLibraryInfoWrapper)
874INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
875INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
876INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
877INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
878 "pre-isel-intrinsic-lowering",
879 "Pre-ISel Intrinsic Lowering", false, false)
880
881ModulePass *llvm::createPreISelIntrinsicLoweringPass() {
882 return new PreISelIntrinsicLoweringLegacyPass();
883}
884
885PreservedAnalyses
886PreISelIntrinsicLoweringPass::run(Module &M, ModuleAnalysisManager &MAM) {
887 const ModuleLibcallLoweringInfo &LibcallLowering =
888 MAM.getResult<LibcallLoweringModuleAnalysis>(IR&: M);
889
890 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
891
892 auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {
893 return FAM.getResult<TargetIRAnalysis>(IR&: F);
894 };
895 auto LookupTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
896 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
897 };
898
899 PreISelIntrinsicLowering Lowering(TM, LibcallLowering, LookupTTI, LookupTLI);
900 if (!Lowering.lowerIntrinsics(M))
901 return PreservedAnalyses::all();
902 else
903 return PreservedAnalyses::none();
904}
905