1//===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass implements IR lowering for the llvm.memcpy, llvm.memmove,
10// llvm.memset, llvm.load.relative and llvm.objc.* intrinsics.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
15#include "llvm/Analysis/ObjCARCInstKind.h"
16#include "llvm/Analysis/ObjCARCUtil.h"
17#include "llvm/Analysis/TargetLibraryInfo.h"
18#include "llvm/Analysis/TargetTransformInfo.h"
19#include "llvm/CodeGen/ExpandVectorPredication.h"
20#include "llvm/CodeGen/LibcallLoweringInfo.h"
21#include "llvm/CodeGen/Passes.h"
22#include "llvm/CodeGen/TargetLowering.h"
23#include "llvm/CodeGen/TargetPassConfig.h"
24#include "llvm/IR/Function.h"
25#include "llvm/IR/GlobalValue.h"
26#include "llvm/IR/IRBuilder.h"
27#include "llvm/IR/Instructions.h"
28#include "llvm/IR/IntrinsicInst.h"
29#include "llvm/IR/Metadata.h"
30#include "llvm/IR/Module.h"
31#include "llvm/IR/RuntimeLibcalls.h"
32#include "llvm/IR/Type.h"
33#include "llvm/IR/Use.h"
34#include "llvm/InitializePasses.h"
35#include "llvm/Pass.h"
36#include "llvm/Support/Casting.h"
37#include "llvm/Target/TargetMachine.h"
38#include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
39#include "llvm/Transforms/Utils/BuildLibCalls.h"
40#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
41#include "llvm/Transforms/Utils/LowerVectorIntrinsics.h"
42
43using namespace llvm;
44
45/// Threshold to leave statically sized memory intrinsic calls. Calls of known
46/// size larger than this will be expanded by the pass. Calls of unknown or
47/// lower size will be left for expansion in codegen.
48static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt(
49 "mem-intrinsic-expand-size",
50 cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(Val: -1),
51 cl::Hidden);
52
53namespace {
54
55struct PreISelIntrinsicLowering {
56 const TargetMachine *TM;
57 const LibcallLoweringModuleAnalysisResult &ModuleLibcalls;
58 const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
59 const function_ref<TargetLibraryInfo &(Function &)> LookupTLI;
60
61 /// If this is true, assume it's preferably to leave memory intrinsic calls
62 /// for replacement with a library call later. Otherwise this depends on
63 /// TargetLoweringInfo availability of the corresponding function.
64 const bool UseMemIntrinsicLibFunc;
65
66 explicit PreISelIntrinsicLowering(
67 const TargetMachine *TM_,
68 const LibcallLoweringModuleAnalysisResult &ModuleLibcalls_,
69 function_ref<TargetTransformInfo &(Function &)> LookupTTI_,
70 function_ref<TargetLibraryInfo &(Function &)> LookupTLI_,
71 bool UseMemIntrinsicLibFunc_ = true)
72 : TM(TM_), ModuleLibcalls(ModuleLibcalls_), LookupTTI(LookupTTI_),
73 LookupTLI(LookupTLI_), UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {
74 }
75
76 static bool shouldExpandMemIntrinsicWithSize(Value *Size,
77 const TargetTransformInfo &TTI);
78 bool
79 expandMemIntrinsicUses(Function &F,
80 DenseMap<Constant *, GlobalVariable *> &CMap) const;
81 bool lowerIntrinsics(Module &M) const;
82};
83
84} // namespace
85
86template <class T> static bool forEachCall(Function &Intrin, T Callback) {
87 // Lowering all intrinsics in a function will delete multiple uses, so we
88 // can't use an early-inc-range. In case some remain, we don't want to look
89 // at them again. Unfortunately, Value::UseList is private, so we can't use a
90 // simple Use**. If LastUse is null, the next use to consider is
91 // Intrin.use_begin(), otherwise it's LastUse->getNext().
92 Use *LastUse = nullptr;
93 bool Changed = false;
94 while (!Intrin.use_empty() && (!LastUse || LastUse->getNext())) {
95 Use *U = LastUse ? LastUse->getNext() : &*Intrin.use_begin();
96 bool Removed = false;
97 // An intrinsic cannot have its address taken, so it cannot be an argument
98 // operand. It might be used as operand in debug metadata, though.
99 if (auto CI = dyn_cast<CallInst>(Val: U->getUser()))
100 Changed |= Removed = Callback(CI);
101 if (!Removed)
102 LastUse = U;
103 }
104 return Changed;
105}
106
107static bool lowerLoadRelative(Function &F) {
108 if (F.use_empty())
109 return false;
110
111 bool Changed = false;
112 Type *Int32Ty = Type::getInt32Ty(C&: F.getContext());
113
114 for (Use &U : llvm::make_early_inc_range(Range: F.uses())) {
115 auto CI = dyn_cast<CallInst>(Val: U.getUser());
116 if (!CI || CI->getCalledOperand() != &F)
117 continue;
118
119 IRBuilder<> B(CI);
120 Value *OffsetPtr =
121 B.CreatePtrAdd(Ptr: CI->getArgOperand(i: 0), Offset: CI->getArgOperand(i: 1));
122 Value *OffsetI32 = B.CreateAlignedLoad(Ty: Int32Ty, Ptr: OffsetPtr, Align: Align(4));
123
124 Value *ResultPtr = B.CreatePtrAdd(Ptr: CI->getArgOperand(i: 0), Offset: OffsetI32);
125
126 CI->replaceAllUsesWith(V: ResultPtr);
127 CI->eraseFromParent();
128 Changed = true;
129 }
130
131 return Changed;
132}
133
134// ObjCARC has knowledge about whether an obj-c runtime function needs to be
135// always tail-called or never tail-called.
136static CallInst::TailCallKind getOverridingTailCallKind(const Function &F) {
137 objcarc::ARCInstKind Kind = objcarc::GetFunctionClass(F: &F);
138 if (objcarc::IsAlwaysTail(Class: Kind))
139 return CallInst::TCK_Tail;
140 else if (objcarc::IsNeverTail(Class: Kind))
141 return CallInst::TCK_NoTail;
142 return CallInst::TCK_None;
143}
144
145static bool lowerObjCCall(Function &F, RTLIB::LibcallImpl NewFn,
146 bool setNonLazyBind = false) {
147 assert(IntrinsicInst::mayLowerToFunctionCall(F.getIntrinsicID()) &&
148 "Pre-ISel intrinsics do lower into regular function calls");
149 if (F.use_empty())
150 return false;
151
152 // FIXME: When RuntimeLibcalls is an analysis, check if the function is really
153 // supported, and go through RTLIB::Libcall.
154 StringRef NewFnName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: NewFn);
155
156 // If we haven't already looked up this function, check to see if the
157 // program already contains a function with this name.
158 Module *M = F.getParent();
159 FunctionCallee FCache =
160 M->getOrInsertFunction(Name: NewFnName, T: F.getFunctionType());
161
162 if (Function *Fn = dyn_cast<Function>(Val: FCache.getCallee())) {
163 Fn->setLinkage(F.getLinkage());
164 if (setNonLazyBind && !Fn->isWeakForLinker()) {
165 // If we have Native ARC, set nonlazybind attribute for these APIs for
166 // performance.
167 Fn->addFnAttr(Kind: Attribute::NonLazyBind);
168 }
169 }
170
171 CallInst::TailCallKind OverridingTCK = getOverridingTailCallKind(F);
172
173 for (Use &U : llvm::make_early_inc_range(Range: F.uses())) {
174 auto *CB = cast<CallBase>(Val: U.getUser());
175
176 if (CB->getCalledFunction() != &F) {
177 assert(objcarc::getAttachedARCFunction(CB) == &F &&
178 "use expected to be the argument of operand bundle "
179 "\"clang.arc.attachedcall\"");
180 U.set(FCache.getCallee());
181 continue;
182 }
183
184 auto *CI = cast<CallInst>(Val: CB);
185 assert(CI->getCalledFunction() && "Cannot lower an indirect call!");
186
187 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
188 SmallVector<Value *, 8> Args(CI->args());
189 SmallVector<llvm::OperandBundleDef, 1> BundleList;
190 CI->getOperandBundlesAsDefs(Defs&: BundleList);
191 CallInst *NewCI = Builder.CreateCall(Callee: FCache, Args, OpBundles: BundleList);
192 NewCI->setName(CI->getName());
193
194 // Try to set the most appropriate TailCallKind based on both the current
195 // attributes and the ones that we could get from ObjCARC's special
196 // knowledge of the runtime functions.
197 //
198 // std::max respects both requirements of notail and tail here:
199 // * notail on either the call or from ObjCARC becomes notail
200 // * tail on either side is stronger than none, but not notail
201 CallInst::TailCallKind TCK = CI->getTailCallKind();
202 NewCI->setTailCallKind(std::max(a: TCK, b: OverridingTCK));
203
204 // Transfer the 'returned' attribute from the intrinsic to the call site.
205 // By applying this only to intrinsic call sites, we avoid applying it to
206 // non-ARC explicit calls to things like objc_retain which have not been
207 // auto-upgraded to use the intrinsics.
208 unsigned Index;
209 if (F.getAttributes().hasAttrSomewhere(Kind: Attribute::Returned, Index: &Index) &&
210 Index)
211 NewCI->addParamAttr(ArgNo: Index - AttributeList::FirstArgIndex,
212 Kind: Attribute::Returned);
213
214 if (!CI->use_empty())
215 CI->replaceAllUsesWith(V: NewCI);
216 CI->eraseFromParent();
217 }
218
219 return true;
220}
221
222// TODO: Should refine based on estimated number of accesses (e.g. does it
223// require splitting based on alignment)
224bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(
225 Value *Size, const TargetTransformInfo &TTI) {
226 ConstantInt *CI = dyn_cast<ConstantInt>(Val: Size);
227 if (!CI)
228 return true;
229 uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences()
230 ? MemIntrinsicExpandSizeThresholdOpt
231 : TTI.getMaxMemIntrinsicInlineSizeThreshold();
232 uint64_t SizeVal = CI->getZExtValue();
233
234 // Treat a threshold of 0 as a special case to force expansion of all
235 // intrinsics, including size 0.
236 return SizeVal > Threshold || Threshold == 0;
237}
238
239static bool
240canEmitLibcall(const LibcallLoweringModuleAnalysisResult &ModuleLowering,
241 const TargetMachine *TM, Function *F, RTLIB::Libcall LC) {
242 // TODO: Should this consider the address space of the memcpy?
243 if (!TM)
244 return true;
245 const LibcallLoweringInfo &Lowering =
246 ModuleLowering.getLibcallLowering(Subtarget: *TM->getSubtargetImpl(*F));
247 return Lowering.getLibcallImpl(Call: LC) != RTLIB::Unsupported;
248}
249
250static bool
251canEmitMemcpy(const LibcallLoweringModuleAnalysisResult &ModuleLowering,
252 const TargetMachine *TM, Function *F) {
253 // TODO: Should this consider the address space of the memcpy?
254 if (!TM)
255 return true;
256 const LibcallLoweringInfo &Lowering =
257 ModuleLowering.getLibcallLowering(Subtarget: *TM->getSubtargetImpl(*F));
258 return Lowering.getMemcpyImpl() != RTLIB::Unsupported;
259}
260
261// Return a value appropriate for use with the memset_pattern16 libcall, if
262// possible and if we know how. (Adapted from equivalent helper in
263// LoopIdiomRecognize).
264static Constant *getMemSetPattern16Value(MemSetPatternInst *Inst,
265 const TargetLibraryInfo &TLI) {
266 // TODO: This could check for UndefValue because it can be merged into any
267 // other valid pattern.
268
269 // Don't emit libcalls if a non-default address space is being used.
270 if (Inst->getRawDest()->getType()->getPointerAddressSpace() != 0)
271 return nullptr;
272
273 Value *V = Inst->getValue();
274 Type *VTy = V->getType();
275 const DataLayout &DL = Inst->getDataLayout();
276 Module *M = Inst->getModule();
277
278 if (!isLibFuncEmittable(M, TLI: &TLI, TheLibFunc: LibFunc_memset_pattern16))
279 return nullptr;
280
281 // If the value isn't a constant, we can't promote it to being in a constant
282 // array. We could theoretically do a store to an alloca or something, but
283 // that doesn't seem worthwhile.
284 Constant *C = dyn_cast<Constant>(Val: V);
285 if (!C || isa<ConstantExpr>(Val: C))
286 return nullptr;
287
288 // Only handle simple values that are a power of two bytes in size.
289 uint64_t Size = DL.getTypeSizeInBits(Ty: VTy);
290 if (!DL.typeSizeEqualsStoreSize(Ty: VTy) || !isPowerOf2_64(Value: Size))
291 return nullptr;
292
293 // Don't care enough about darwin/ppc to implement this.
294 if (DL.isBigEndian())
295 return nullptr;
296
297 // Convert to size in bytes.
298 Size /= 8;
299
300 // TODO: If CI is larger than 16-bytes, we can try slicing it in half to see
301 // if the top and bottom are the same (e.g. for vectors and large integers).
302 if (Size > 16)
303 return nullptr;
304
305 // If the constant is exactly 16 bytes, just use it.
306 if (Size == 16)
307 return C;
308
309 // Otherwise, we'll use an array of the constants.
310 uint64_t ArraySize = 16 / Size;
311 ArrayType *AT = ArrayType::get(ElementType: V->getType(), NumElements: ArraySize);
312 return ConstantArray::get(T: AT, V: std::vector<Constant *>(ArraySize, C));
313}
314
315// TODO: Handle atomic memcpy and memcpy.inline
316// TODO: Pass ScalarEvolution
317bool PreISelIntrinsicLowering::expandMemIntrinsicUses(
318 Function &F, DenseMap<Constant *, GlobalVariable *> &CMap) const {
319 Intrinsic::ID ID = F.getIntrinsicID();
320 bool Changed = false;
321
322 for (User *U : llvm::make_early_inc_range(Range: F.users())) {
323 Instruction *Inst = cast<Instruction>(Val: U);
324
325 switch (ID) {
326 case Intrinsic::memcpy: {
327 auto *Memcpy = cast<MemCpyInst>(Val: Inst);
328 Function *ParentFunc = Memcpy->getFunction();
329 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
330 if (shouldExpandMemIntrinsicWithSize(Size: Memcpy->getLength(), TTI)) {
331 if (UseMemIntrinsicLibFunc &&
332 canEmitMemcpy(ModuleLowering: ModuleLibcalls, TM, F: ParentFunc))
333 break;
334
335 // TODO: For optsize, emit the loop into a separate function
336 expandMemCpyAsLoop(MemCpy: Memcpy, TTI);
337 Changed = true;
338 Memcpy->eraseFromParent();
339 }
340
341 break;
342 }
343 case Intrinsic::memcpy_inline: {
344 // Only expand llvm.memcpy.inline with non-constant length in this
345 // codepath, leaving the current SelectionDAG expansion for constant
346 // length memcpy intrinsics undisturbed.
347 auto *Memcpy = cast<MemCpyInst>(Val: Inst);
348 if (isa<ConstantInt>(Val: Memcpy->getLength()))
349 break;
350
351 Function *ParentFunc = Memcpy->getFunction();
352 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
353 expandMemCpyAsLoop(MemCpy: Memcpy, TTI);
354 Changed = true;
355 Memcpy->eraseFromParent();
356 break;
357 }
358 case Intrinsic::memmove: {
359 auto *Memmove = cast<MemMoveInst>(Val: Inst);
360 Function *ParentFunc = Memmove->getFunction();
361 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
362 if (shouldExpandMemIntrinsicWithSize(Size: Memmove->getLength(), TTI)) {
363 if (UseMemIntrinsicLibFunc &&
364 canEmitLibcall(ModuleLowering: ModuleLibcalls, TM, F: ParentFunc, LC: RTLIB::MEMMOVE))
365 break;
366
367 if (expandMemMoveAsLoop(MemMove: Memmove, TTI)) {
368 Changed = true;
369 Memmove->eraseFromParent();
370 }
371 }
372
373 break;
374 }
375 case Intrinsic::memset: {
376 auto *Memset = cast<MemSetInst>(Val: Inst);
377 Function *ParentFunc = Memset->getFunction();
378 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
379 if (shouldExpandMemIntrinsicWithSize(Size: Memset->getLength(), TTI)) {
380 if (UseMemIntrinsicLibFunc &&
381 canEmitLibcall(ModuleLowering: ModuleLibcalls, TM, F: ParentFunc, LC: RTLIB::MEMSET))
382 break;
383
384 expandMemSetAsLoop(MemSet: Memset);
385 Changed = true;
386 Memset->eraseFromParent();
387 }
388
389 break;
390 }
391 case Intrinsic::memset_inline: {
392 // Only expand llvm.memset.inline with non-constant length in this
393 // codepath, leaving the current SelectionDAG expansion for constant
394 // length memset intrinsics undisturbed.
395 auto *Memset = cast<MemSetInst>(Val: Inst);
396 if (isa<ConstantInt>(Val: Memset->getLength()))
397 break;
398
399 expandMemSetAsLoop(MemSet: Memset);
400 Changed = true;
401 Memset->eraseFromParent();
402 break;
403 }
404 case Intrinsic::experimental_memset_pattern: {
405 auto *Memset = cast<MemSetPatternInst>(Val: Inst);
406 const TargetLibraryInfo &TLI = LookupTLI(*Memset->getFunction());
407 Constant *PatternValue = getMemSetPattern16Value(Inst: Memset, TLI);
408 if (!PatternValue) {
409 // If it isn't possible to emit a memset_pattern16 libcall, expand to
410 // a loop instead.
411 expandMemSetPatternAsLoop(MemSet: Memset);
412 Changed = true;
413 Memset->eraseFromParent();
414 break;
415 }
416 // FIXME: There is currently no profitability calculation for emitting
417 // the libcall vs expanding the memset.pattern directly.
418 IRBuilder<> Builder(Inst);
419 Module *M = Memset->getModule();
420 const DataLayout &DL = Memset->getDataLayout();
421
422 Type *DestPtrTy = Memset->getRawDest()->getType();
423 Type *SizeTTy = TLI.getSizeTType(M: *M);
424 StringRef FuncName = "memset_pattern16";
425 FunctionCallee MSP = getOrInsertLibFunc(M, TLI, TheLibFunc: LibFunc_memset_pattern16,
426 RetTy: Builder.getVoidTy(), Args: DestPtrTy,
427 Args: Builder.getPtrTy(), Args: SizeTTy);
428 inferNonMandatoryLibFuncAttrs(M, Name: FuncName, TLI);
429
430 // Otherwise we should form a memset_pattern16. PatternValue is known
431 // to be an constant array of 16-bytes. Put the value into a mergable
432 // global.
433 assert(Memset->getRawDest()->getType()->getPointerAddressSpace() == 0 &&
434 "Should have skipped if non-zero AS");
435 GlobalVariable *GV;
436 auto It = CMap.find(Val: PatternValue);
437 if (It != CMap.end()) {
438 GV = It->second;
439 } else {
440 GV = new GlobalVariable(
441 *M, PatternValue->getType(), /*isConstant=*/true,
442 GlobalValue::PrivateLinkage, PatternValue, ".memset_pattern");
443 GV->setUnnamedAddr(
444 GlobalValue::UnnamedAddr::Global); // Ok to merge these.
445 // TODO: Consider relaxing alignment requirement.
446 GV->setAlignment(Align(16));
447 CMap[PatternValue] = GV;
448 }
449 Value *PatternPtr = GV;
450 Value *NumBytes = Builder.CreateMul(
451 LHS: TLI.getAsSizeT(V: DL.getTypeAllocSize(Ty: Memset->getValue()->getType()),
452 M: *M),
453 RHS: Builder.CreateZExtOrTrunc(V: Memset->getLength(), DestTy: SizeTTy));
454 CallInst *MemsetPattern16Call =
455 Builder.CreateCall(Callee: MSP, Args: {Memset->getRawDest(), PatternPtr, NumBytes});
456 MemsetPattern16Call->setAAMetadata(Memset->getAAMetadata());
457 // Preserve any call site attributes on the destination pointer
458 // argument (e.g. alignment).
459 AttrBuilder ArgAttrs(Memset->getContext(),
460 Memset->getAttributes().getParamAttrs(ArgNo: 0));
461 MemsetPattern16Call->setAttributes(
462 MemsetPattern16Call->getAttributes().addParamAttributes(
463 C&: Memset->getContext(), ArgNo: 0, B: ArgAttrs));
464 Changed = true;
465 Memset->eraseFromParent();
466 break;
467 }
468 default:
469 llvm_unreachable("unhandled intrinsic");
470 }
471 }
472
473 return Changed;
474}
475
476static bool expandProtectedFieldPtr(Function &Intr) {
477 Module &M = *Intr.getParent();
478
479 SmallPtrSet<GlobalValue *, 2> DSsToDeactivate;
480
481 Type *Int8Ty = Type::getInt8Ty(C&: M.getContext());
482 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
483 PointerType *PtrTy = PointerType::get(C&: M.getContext(), AddressSpace: 0);
484
485 Function *SignIntr =
486 Intrinsic::getOrInsertDeclaration(M: &M, id: Intrinsic::ptrauth_sign, Tys: {});
487 Function *AuthIntr =
488 Intrinsic::getOrInsertDeclaration(M: &M, id: Intrinsic::ptrauth_auth, Tys: {});
489
490 auto *EmuFnTy = FunctionType::get(Result: Int64Ty, Params: {Int64Ty, Int64Ty}, isVarArg: false);
491
492 auto CreateSign = [&](IRBuilder<> &B, Value *Val, Value *Disc,
493 OperandBundleDef DSBundle) {
494 Function *F = B.GetInsertBlock()->getParent();
495 Attribute FSAttr = F->getFnAttribute(Kind: "target-features");
496 if (FSAttr.isValid() && FSAttr.getValueAsString().contains(Other: "+pauth"))
497 return B.CreateCall(
498 Callee: SignIntr, Args: {Val, B.getInt32(/*AArch64PACKey::DA*/ C: 2), Disc}, OpBundles: DSBundle);
499 FunctionCallee EmuSignIntr =
500 M.getOrInsertFunction(Name: "__emupac_pacda", T: EmuFnTy);
501 return B.CreateCall(Callee: EmuSignIntr, Args: {Val, Disc}, OpBundles: DSBundle);
502 };
503
504 auto CreateAuth = [&](IRBuilder<> &B, Value *Val, Value *Disc,
505 OperandBundleDef DSBundle) {
506 Function *F = B.GetInsertBlock()->getParent();
507 Attribute FSAttr = F->getFnAttribute(Kind: "target-features");
508 if (FSAttr.isValid() && FSAttr.getValueAsString().contains(Other: "+pauth"))
509 return B.CreateCall(
510 Callee: AuthIntr, Args: {Val, B.getInt32(/*AArch64PACKey::DA*/ C: 2), Disc}, OpBundles: DSBundle);
511 FunctionCallee EmuAuthIntr =
512 M.getOrInsertFunction(Name: "__emupac_autda", T: EmuFnTy);
513 return B.CreateCall(Callee: EmuAuthIntr, Args: {Val, Disc}, OpBundles: DSBundle);
514 };
515
516 auto GetDeactivationSymbol = [&](CallInst *Call) -> GlobalValue * {
517 if (auto Bundle =
518 Call->getOperandBundle(ID: LLVMContext::OB_deactivation_symbol))
519 return cast<GlobalValue>(Val: Bundle->Inputs[0]);
520 return nullptr;
521 };
522
523 for (User *U : llvm::make_early_inc_range(Range: Intr.users())) {
524 auto *Call = cast<CallInst>(Val: U);
525
526 auto *Pointer = Call->getArgOperand(i: 0);
527 auto *Disc = Call->getArgOperand(i: 1);
528 bool UseHWEncoding =
529 cast<ConstantInt>(Val: Call->getArgOperand(i: 2))->getZExtValue();
530 if (!UseHWEncoding)
531 reportFatalUsageError(reason: "software encoding currently unsupported");
532
533 auto *DS = GetDeactivationSymbol(Call);
534 OperandBundleDef DSBundle("deactivation-symbol", DS);
535
536 for (Use &U : llvm::make_early_inc_range(Range: Call->uses())) {
537 // Insert code to encode each pointer stored to the pointer returned by
538 // the intrinsic.
539 if (auto *SI = dyn_cast<StoreInst>(Val: U.getUser())) {
540 if (U.getOperandNo() == 1 &&
541 isa<PointerType>(Val: SI->getValueOperand()->getType())) {
542 IRBuilder<> B(SI);
543 auto *SIValInt =
544 B.CreatePtrToInt(V: SI->getValueOperand(), DestTy: B.getInt64Ty());
545 Value *Sign = CreateSign(B, SIValInt, Disc, DSBundle);
546 SI->setOperand(i_nocapture: 0, Val_nocapture: B.CreateIntToPtr(V: Sign, DestTy: B.getPtrTy()));
547 SI->setOperand(i_nocapture: 1, Val_nocapture: Pointer);
548 continue;
549 }
550 }
551
552 // Insert code to decode each pointer loaded from the pointer returned by
553 // the intrinsic. This is the inverse of the encode operation implemented
554 // above.
555 if (auto *LI = dyn_cast<LoadInst>(Val: U.getUser())) {
556 if (isa<PointerType>(Val: LI->getType())) {
557 IRBuilder<> B(LI);
558 auto *NewLI = cast<LoadInst>(Val: LI->clone());
559 NewLI->setOperand(i_nocapture: 0, Val_nocapture: Pointer);
560 B.Insert(I: NewLI);
561 auto *LIInt = B.CreatePtrToInt(V: NewLI, DestTy: B.getInt64Ty());
562 Value *Auth = CreateAuth(B, LIInt, Disc, DSBundle);
563 LI->replaceAllUsesWith(V: B.CreateIntToPtr(V: Auth, DestTy: B.getPtrTy()));
564 LI->eraseFromParent();
565 continue;
566 }
567 }
568 // Comparisons against null cannot be used to recover the original
569 // pointer so we replace them with comparisons against the original
570 // pointer.
571 if (auto *CI = dyn_cast<ICmpInst>(Val: U.getUser())) {
572 if (auto *Op = dyn_cast<Constant>(Val: CI->getOperand(i_nocapture: 0))) {
573 if (Op->isNullValue()) {
574 CI->setOperand(i_nocapture: 1, Val_nocapture: Pointer);
575 continue;
576 }
577 }
578 if (auto *Op = dyn_cast<Constant>(Val: CI->getOperand(i_nocapture: 1))) {
579 if (Op->isNullValue()) {
580 CI->setOperand(i_nocapture: 0, Val_nocapture: Pointer);
581 continue;
582 }
583 }
584 }
585
586 // We couldn't rewrite away this use of the intrinsic. Replace it with the
587 // pointer operand, and arrange to define a deactivation symbol.
588 U.set(Pointer);
589 if (DS)
590 DSsToDeactivate.insert(Ptr: DS);
591 }
592
593 Call->eraseFromParent();
594 }
595
596 if (!DSsToDeactivate.empty()) {
597 // This is an AArch64 NOP instruction. When the deactivation symbol support
598 // is expanded to more architectures, there will likely need to be an API
599 // for retrieving this constant.
600 Constant *Nop =
601 ConstantExpr::getIntToPtr(C: ConstantInt::get(Ty: Int64Ty, V: 0xd503201f), Ty: PtrTy);
602 for (GlobalValue *OldDS : DSsToDeactivate) {
603 GlobalValue *DS = GlobalAlias::create(
604 Ty: Int8Ty, AddressSpace: 0, Linkage: GlobalValue::ExternalLinkage, Name: OldDS->getName(), Aliasee: Nop, Parent: &M);
605 DS->setVisibility(GlobalValue::HiddenVisibility);
606 DS->takeName(V: OldDS);
607 OldDS->replaceAllUsesWith(V: DS);
608 OldDS->eraseFromParent();
609 }
610 }
611 return true;
612}
613
614bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
615 // Map unique constants to globals.
616 DenseMap<Constant *, GlobalVariable *> CMap;
617 bool Changed = false;
618 for (Function &F : M) {
619 switch (F.getIntrinsicID()) {
620 default:
621 break;
622 case Intrinsic::memcpy:
623 case Intrinsic::memcpy_inline:
624 case Intrinsic::memmove:
625 case Intrinsic::memset:
626 case Intrinsic::memset_inline:
627 case Intrinsic::experimental_memset_pattern:
628 Changed |= expandMemIntrinsicUses(F, CMap);
629 break;
630 case Intrinsic::load_relative:
631 Changed |= lowerLoadRelative(F);
632 break;
633 case Intrinsic::is_constant:
634 case Intrinsic::objectsize:
635 Changed |= forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
636 Function *Parent = CI->getParent()->getParent();
637 TargetLibraryInfo &TLI = LookupTLI(*Parent);
638 // Intrinsics in unreachable code are not lowered.
639 bool Changed = lowerConstantIntrinsics(F&: *Parent, TLI, /*DT=*/nullptr);
640 return Changed;
641 });
642 break;
643#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
644 case Intrinsic::VPID:
645#include "llvm/IR/VPIntrinsics.def"
646 forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
647 Function *Parent = CI->getParent()->getParent();
648 const TargetTransformInfo &TTI = LookupTTI(*Parent);
649 auto *VPI = cast<VPIntrinsic>(Val: CI);
650 VPExpansionDetails ED = expandVectorPredicationIntrinsic(VPI&: *VPI, TTI);
651 // Expansion of VP intrinsics may change the IR but not actually
652 // replace the intrinsic, so update Changed for the pass
653 // and compute Removed for forEachCall.
654 Changed |= ED != VPExpansionDetails::IntrinsicUnchanged;
655 bool Removed = ED == VPExpansionDetails::IntrinsicReplaced;
656 return Removed;
657 });
658 break;
659 case Intrinsic::objc_autorelease:
660 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autorelease);
661 break;
662 case Intrinsic::objc_autoreleasePoolPop:
663 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autoreleasePoolPop);
664 break;
665 case Intrinsic::objc_autoreleasePoolPush:
666 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autoreleasePoolPush);
667 break;
668 case Intrinsic::objc_autoreleaseReturnValue:
669 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autoreleaseReturnValue);
670 break;
671 case Intrinsic::objc_copyWeak:
672 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_copyWeak);
673 break;
674 case Intrinsic::objc_destroyWeak:
675 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_destroyWeak);
676 break;
677 case Intrinsic::objc_initWeak:
678 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_initWeak);
679 break;
680 case Intrinsic::objc_loadWeak:
681 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_loadWeak);
682 break;
683 case Intrinsic::objc_loadWeakRetained:
684 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_loadWeakRetained);
685 break;
686 case Intrinsic::objc_moveWeak:
687 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_moveWeak);
688 break;
689 case Intrinsic::objc_release:
690 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_release, setNonLazyBind: true);
691 break;
692 case Intrinsic::objc_retain:
693 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retain, setNonLazyBind: true);
694 break;
695 case Intrinsic::objc_retainAutorelease:
696 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainAutorelease);
697 break;
698 case Intrinsic::objc_retainAutoreleaseReturnValue:
699 Changed |=
700 lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainAutoreleaseReturnValue);
701 break;
702 case Intrinsic::objc_retainAutoreleasedReturnValue:
703 Changed |=
704 lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainAutoreleasedReturnValue);
705 break;
706 case Intrinsic::objc_claimAutoreleasedReturnValue:
707 Changed |=
708 lowerObjCCall(F, NewFn: RTLIB::impl_objc_claimAutoreleasedReturnValue);
709 break;
710 case Intrinsic::objc_retainBlock:
711 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainBlock);
712 break;
713 case Intrinsic::objc_storeStrong:
714 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_storeStrong);
715 break;
716 case Intrinsic::objc_storeWeak:
717 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_storeWeak);
718 break;
719 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
720 Changed |=
721 lowerObjCCall(F, NewFn: RTLIB::impl_objc_unsafeClaimAutoreleasedReturnValue);
722 break;
723 case Intrinsic::objc_retainedObject:
724 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainedObject);
725 break;
726 case Intrinsic::objc_unretainedObject:
727 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_unretainedObject);
728 break;
729 case Intrinsic::objc_unretainedPointer:
730 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_unretainedPointer);
731 break;
732 case Intrinsic::objc_retain_autorelease:
733 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retain_autorelease);
734 break;
735 case Intrinsic::objc_sync_enter:
736 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_sync_enter);
737 break;
738 case Intrinsic::objc_sync_exit:
739 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_sync_exit);
740 break;
741 case Intrinsic::exp:
742 case Intrinsic::exp2:
743 case Intrinsic::log:
744 Changed |= forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
745 Type *Ty = CI->getArgOperand(i: 0)->getType();
746 if (!TM || !isa<ScalableVectorType>(Val: Ty))
747 return false;
748 const TargetLowering *TL = TM->getSubtargetImpl(F)->getTargetLowering();
749 unsigned Op = TL->IntrinsicIDToISD(ID: F.getIntrinsicID());
750 assert(Op != ISD::DELETED_NODE && "unsupported intrinsic");
751 if (!TL->isOperationExpand(Op, VT: EVT::getEVT(Ty)))
752 return false;
753 return lowerUnaryVectorIntrinsicAsLoop(M, CI);
754 });
755 break;
756 case Intrinsic::protected_field_ptr:
757 Changed |= expandProtectedFieldPtr(Intr&: F);
758 break;
759 }
760 }
761 return Changed;
762}
763
764namespace {
765
766class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
767public:
768 static char ID;
769
770 PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}
771
772 void getAnalysisUsage(AnalysisUsage &AU) const override {
773 AU.addRequired<TargetTransformInfoWrapperPass>();
774 AU.addRequired<TargetLibraryInfoWrapperPass>();
775 AU.addRequired<LibcallLoweringInfoWrapper>();
776 AU.addRequired<TargetPassConfig>();
777 }
778
779 bool runOnModule(Module &M) override {
780 const LibcallLoweringModuleAnalysisResult &ModuleLibcalls =
781 getAnalysis<LibcallLoweringInfoWrapper>().getResult(M);
782
783 auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {
784 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
785 };
786 auto LookupTLI = [this](Function &F) -> TargetLibraryInfo & {
787 return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
788 };
789
790 const auto *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
791 PreISelIntrinsicLowering Lowering(TM, ModuleLibcalls, LookupTTI, LookupTLI);
792 return Lowering.lowerIntrinsics(M);
793 }
794};
795
796} // end anonymous namespace
797
798char PreISelIntrinsicLoweringLegacyPass::ID;
799
800INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass,
801 "pre-isel-intrinsic-lowering",
802 "Pre-ISel Intrinsic Lowering", false, false)
803INITIALIZE_PASS_DEPENDENCY(LibcallLoweringInfoWrapper)
804INITIALIZE_PASS_DEPENDENCY(RuntimeLibraryInfoWrapper)
805INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
806INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
807INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
808INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
809 "pre-isel-intrinsic-lowering",
810 "Pre-ISel Intrinsic Lowering", false, false)
811
812ModulePass *llvm::createPreISelIntrinsicLoweringPass() {
813 return new PreISelIntrinsicLoweringLegacyPass();
814}
815
816PreservedAnalyses
817PreISelIntrinsicLoweringPass::run(Module &M, ModuleAnalysisManager &MAM) {
818 const LibcallLoweringModuleAnalysisResult &LibcallLowering =
819 MAM.getResult<LibcallLoweringModuleAnalysis>(IR&: M);
820
821 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
822
823 auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {
824 return FAM.getResult<TargetIRAnalysis>(IR&: F);
825 };
826 auto LookupTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
827 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
828 };
829
830 PreISelIntrinsicLowering Lowering(TM, LibcallLowering, LookupTTI, LookupTLI);
831 if (!Lowering.lowerIntrinsics(M))
832 return PreservedAnalyses::all();
833 else
834 return PreservedAnalyses::none();
835}
836