1//===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass implements IR lowering for the llvm.memcpy, llvm.memmove,
10// llvm.memset, llvm.load.relative and llvm.objc.* intrinsics.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/Analysis/ObjCARCInstKind.h"
17#include "llvm/Analysis/ObjCARCUtil.h"
18#include "llvm/Analysis/TargetLibraryInfo.h"
19#include "llvm/Analysis/TargetTransformInfo.h"
20#include "llvm/CodeGen/ExpandVectorPredication.h"
21#include "llvm/CodeGen/LibcallLoweringInfo.h"
22#include "llvm/CodeGen/Passes.h"
23#include "llvm/CodeGen/TargetLowering.h"
24#include "llvm/CodeGen/TargetPassConfig.h"
25#include "llvm/IR/Function.h"
26#include "llvm/IR/GlobalValue.h"
27#include "llvm/IR/IRBuilder.h"
28#include "llvm/IR/Instructions.h"
29#include "llvm/IR/IntrinsicInst.h"
30#include "llvm/IR/Metadata.h"
31#include "llvm/IR/Module.h"
32#include "llvm/IR/ProfDataUtils.h"
33#include "llvm/IR/RuntimeLibcalls.h"
34#include "llvm/IR/Type.h"
35#include "llvm/IR/Use.h"
36#include "llvm/InitializePasses.h"
37#include "llvm/Pass.h"
38#include "llvm/Support/Casting.h"
39#include "llvm/Target/TargetMachine.h"
40#include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
41#include "llvm/Transforms/Utils/BasicBlockUtils.h"
42#include "llvm/Transforms/Utils/BuildLibCalls.h"
43#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
44#include "llvm/Transforms/Utils/LowerVectorIntrinsics.h"
45
46using namespace llvm;
47
48#define DEBUG_TYPE "pre-isel-intrinsic-lowering"
49
50/// Threshold to leave statically sized memory intrinsic calls. Calls of known
51/// size larger than this will be expanded by the pass. Calls of unknown or
52/// lower size will be left for expansion in codegen.
53static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt(
54 "mem-intrinsic-expand-size",
55 cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(Val: -1),
56 cl::Hidden);
57
58namespace {
59
60struct PreISelIntrinsicLowering {
61 const TargetMachine *TM;
62 const LibcallLoweringModuleAnalysisResult &ModuleLibcalls;
63 const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
64 const function_ref<TargetLibraryInfo &(Function &)> LookupTLI;
65
66 /// If this is true, assume it's preferably to leave memory intrinsic calls
67 /// for replacement with a library call later. Otherwise this depends on
68 /// TargetLoweringInfo availability of the corresponding function.
69 const bool UseMemIntrinsicLibFunc;
70
71 explicit PreISelIntrinsicLowering(
72 const TargetMachine *TM_,
73 const LibcallLoweringModuleAnalysisResult &ModuleLibcalls_,
74 function_ref<TargetTransformInfo &(Function &)> LookupTTI_,
75 function_ref<TargetLibraryInfo &(Function &)> LookupTLI_,
76 bool UseMemIntrinsicLibFunc_ = true)
77 : TM(TM_), ModuleLibcalls(ModuleLibcalls_), LookupTTI(LookupTTI_),
78 LookupTLI(LookupTLI_), UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {
79 }
80
81 static bool shouldExpandMemIntrinsicWithSize(Value *Size,
82 const TargetTransformInfo &TTI);
83 bool
84 expandMemIntrinsicUses(Function &F,
85 DenseMap<Constant *, GlobalVariable *> &CMap) const;
86 bool lowerIntrinsics(Module &M) const;
87};
88
89} // namespace
90
91template <class T> static bool forEachCall(Function &Intrin, T Callback) {
92 // Lowering all intrinsics in a function will delete multiple uses, so we
93 // can't use an early-inc-range. In case some remain, we don't want to look
94 // at them again. Unfortunately, Value::UseList is private, so we can't use a
95 // simple Use**. If LastUse is null, the next use to consider is
96 // Intrin.use_begin(), otherwise it's LastUse->getNext().
97 Use *LastUse = nullptr;
98 bool Changed = false;
99 while (!Intrin.use_empty() && (!LastUse || LastUse->getNext())) {
100 Use *U = LastUse ? LastUse->getNext() : &*Intrin.use_begin();
101 bool Removed = false;
102 // An intrinsic cannot have its address taken, so it cannot be an argument
103 // operand. It might be used as operand in debug metadata, though.
104 if (auto CI = dyn_cast<CallInst>(Val: U->getUser()))
105 Changed |= Removed = Callback(CI);
106 if (!Removed)
107 LastUse = U;
108 }
109 return Changed;
110}
111
112static bool lowerLoadRelative(Function &F) {
113 if (F.use_empty())
114 return false;
115
116 bool Changed = false;
117 Type *Int32Ty = Type::getInt32Ty(C&: F.getContext());
118
119 for (Use &U : llvm::make_early_inc_range(Range: F.uses())) {
120 auto CI = dyn_cast<CallInst>(Val: U.getUser());
121 if (!CI || CI->getCalledOperand() != &F)
122 continue;
123
124 IRBuilder<> B(CI);
125 Value *OffsetPtr =
126 B.CreatePtrAdd(Ptr: CI->getArgOperand(i: 0), Offset: CI->getArgOperand(i: 1));
127 Value *OffsetI32 = B.CreateAlignedLoad(Ty: Int32Ty, Ptr: OffsetPtr, Align: Align(4));
128
129 Value *ResultPtr = B.CreatePtrAdd(Ptr: CI->getArgOperand(i: 0), Offset: OffsetI32);
130
131 CI->replaceAllUsesWith(V: ResultPtr);
132 CI->eraseFromParent();
133 Changed = true;
134 }
135
136 return Changed;
137}
138
139// ObjCARC has knowledge about whether an obj-c runtime function needs to be
140// always tail-called or never tail-called.
141static CallInst::TailCallKind getOverridingTailCallKind(const Function &F) {
142 objcarc::ARCInstKind Kind = objcarc::GetFunctionClass(F: &F);
143 if (objcarc::IsAlwaysTail(Class: Kind))
144 return CallInst::TCK_Tail;
145 else if (objcarc::IsNeverTail(Class: Kind))
146 return CallInst::TCK_NoTail;
147 return CallInst::TCK_None;
148}
149
150static bool lowerObjCCall(Function &F, RTLIB::LibcallImpl NewFn,
151 bool setNonLazyBind = false) {
152 assert(IntrinsicInst::mayLowerToFunctionCall(F.getIntrinsicID()) &&
153 "Pre-ISel intrinsics do lower into regular function calls");
154 if (F.use_empty())
155 return false;
156
157 // FIXME: When RuntimeLibcalls is an analysis, check if the function is really
158 // supported, and go through RTLIB::Libcall.
159 StringRef NewFnName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: NewFn);
160
161 // If we haven't already looked up this function, check to see if the
162 // program already contains a function with this name.
163 Module *M = F.getParent();
164 FunctionCallee FCache =
165 M->getOrInsertFunction(Name: NewFnName, T: F.getFunctionType());
166
167 if (Function *Fn = dyn_cast<Function>(Val: FCache.getCallee())) {
168 Fn->setLinkage(F.getLinkage());
169 if (setNonLazyBind && !Fn->isWeakForLinker()) {
170 // If we have Native ARC, set nonlazybind attribute for these APIs for
171 // performance.
172 Fn->addFnAttr(Kind: Attribute::NonLazyBind);
173 }
174 }
175
176 CallInst::TailCallKind OverridingTCK = getOverridingTailCallKind(F);
177
178 for (Use &U : llvm::make_early_inc_range(Range: F.uses())) {
179 auto *CB = cast<CallBase>(Val: U.getUser());
180
181 if (CB->getCalledFunction() != &F) {
182 assert(objcarc::getAttachedARCFunction(CB) == &F &&
183 "use expected to be the argument of operand bundle "
184 "\"clang.arc.attachedcall\"");
185 U.set(FCache.getCallee());
186 continue;
187 }
188
189 auto *CI = cast<CallInst>(Val: CB);
190 assert(CI->getCalledFunction() && "Cannot lower an indirect call!");
191
192 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
193 SmallVector<Value *, 8> Args(CI->args());
194 SmallVector<llvm::OperandBundleDef, 1> BundleList;
195 CI->getOperandBundlesAsDefs(Defs&: BundleList);
196 CallInst *NewCI = Builder.CreateCall(Callee: FCache, Args, OpBundles: BundleList);
197 NewCI->setName(CI->getName());
198
199 // Try to set the most appropriate TailCallKind based on both the current
200 // attributes and the ones that we could get from ObjCARC's special
201 // knowledge of the runtime functions.
202 //
203 // std::max respects both requirements of notail and tail here:
204 // * notail on either the call or from ObjCARC becomes notail
205 // * tail on either side is stronger than none, but not notail
206 CallInst::TailCallKind TCK = CI->getTailCallKind();
207 NewCI->setTailCallKind(std::max(a: TCK, b: OverridingTCK));
208
209 // Transfer the 'returned' attribute from the intrinsic to the call site.
210 // By applying this only to intrinsic call sites, we avoid applying it to
211 // non-ARC explicit calls to things like objc_retain which have not been
212 // auto-upgraded to use the intrinsics.
213 unsigned Index;
214 if (F.getAttributes().hasAttrSomewhere(Kind: Attribute::Returned, Index: &Index) &&
215 Index)
216 NewCI->addParamAttr(ArgNo: Index - AttributeList::FirstArgIndex,
217 Kind: Attribute::Returned);
218
219 if (!CI->use_empty())
220 CI->replaceAllUsesWith(V: NewCI);
221 CI->eraseFromParent();
222 }
223
224 return true;
225}
226
227// TODO: Should refine based on estimated number of accesses (e.g. does it
228// require splitting based on alignment)
229bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(
230 Value *Size, const TargetTransformInfo &TTI) {
231 ConstantInt *CI = dyn_cast<ConstantInt>(Val: Size);
232 if (!CI)
233 return true;
234 uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences()
235 ? MemIntrinsicExpandSizeThresholdOpt
236 : TTI.getMaxMemIntrinsicInlineSizeThreshold();
237 uint64_t SizeVal = CI->getZExtValue();
238
239 // Treat a threshold of 0 as a special case to force expansion of all
240 // intrinsics, including size 0.
241 return SizeVal > Threshold || Threshold == 0;
242}
243
244static bool
245canEmitLibcall(const LibcallLoweringModuleAnalysisResult &ModuleLowering,
246 const TargetMachine *TM, Function *F, RTLIB::Libcall LC) {
247 // TODO: Should this consider the address space of the memcpy?
248 if (!TM)
249 return true;
250 const LibcallLoweringInfo &Lowering =
251 ModuleLowering.getLibcallLowering(Subtarget: *TM->getSubtargetImpl(*F));
252 return Lowering.getLibcallImpl(Call: LC) != RTLIB::Unsupported;
253}
254
255static bool
256canEmitMemcpy(const LibcallLoweringModuleAnalysisResult &ModuleLowering,
257 const TargetMachine *TM, Function *F) {
258 // TODO: Should this consider the address space of the memcpy?
259 if (!TM)
260 return true;
261 const LibcallLoweringInfo &Lowering =
262 ModuleLowering.getLibcallLowering(Subtarget: *TM->getSubtargetImpl(*F));
263 return Lowering.getMemcpyImpl() != RTLIB::Unsupported;
264}
265
266// Return a value appropriate for use with the memset_pattern16 libcall, if
267// possible and if we know how. (Adapted from equivalent helper in
268// LoopIdiomRecognize).
269static Constant *getMemSetPattern16Value(MemSetPatternInst *Inst,
270 const TargetLibraryInfo &TLI) {
271 // TODO: This could check for UndefValue because it can be merged into any
272 // other valid pattern.
273
274 // Don't emit libcalls if a non-default address space is being used.
275 if (Inst->getRawDest()->getType()->getPointerAddressSpace() != 0)
276 return nullptr;
277
278 Value *V = Inst->getValue();
279 Type *VTy = V->getType();
280 const DataLayout &DL = Inst->getDataLayout();
281 Module *M = Inst->getModule();
282
283 if (!isLibFuncEmittable(M, TLI: &TLI, TheLibFunc: LibFunc_memset_pattern16))
284 return nullptr;
285
286 // If the value isn't a constant, we can't promote it to being in a constant
287 // array. We could theoretically do a store to an alloca or something, but
288 // that doesn't seem worthwhile.
289 Constant *C = dyn_cast<Constant>(Val: V);
290 if (!C || isa<ConstantExpr>(Val: C))
291 return nullptr;
292
293 // Only handle simple values that are a power of two bytes in size.
294 uint64_t Size = DL.getTypeSizeInBits(Ty: VTy);
295 if (!DL.typeSizeEqualsStoreSize(Ty: VTy) || !isPowerOf2_64(Value: Size))
296 return nullptr;
297
298 // Don't care enough about darwin/ppc to implement this.
299 if (DL.isBigEndian())
300 return nullptr;
301
302 // Convert to size in bytes.
303 Size /= 8;
304
305 // TODO: If CI is larger than 16-bytes, we can try slicing it in half to see
306 // if the top and bottom are the same (e.g. for vectors and large integers).
307 if (Size > 16)
308 return nullptr;
309
310 // If the constant is exactly 16 bytes, just use it.
311 if (Size == 16)
312 return C;
313
314 // Otherwise, we'll use an array of the constants.
315 uint64_t ArraySize = 16 / Size;
316 ArrayType *AT = ArrayType::get(ElementType: V->getType(), NumElements: ArraySize);
317 return ConstantArray::get(T: AT, V: std::vector<Constant *>(ArraySize, C));
318}
319
320// TODO: Handle atomic memcpy and memcpy.inline
321// TODO: Pass ScalarEvolution
322bool PreISelIntrinsicLowering::expandMemIntrinsicUses(
323 Function &F, DenseMap<Constant *, GlobalVariable *> &CMap) const {
324 Intrinsic::ID ID = F.getIntrinsicID();
325 bool Changed = false;
326
327 for (User *U : llvm::make_early_inc_range(Range: F.users())) {
328 Instruction *Inst = cast<Instruction>(Val: U);
329
330 switch (ID) {
331 case Intrinsic::memcpy: {
332 auto *Memcpy = cast<MemCpyInst>(Val: Inst);
333 Function *ParentFunc = Memcpy->getFunction();
334 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
335 if (shouldExpandMemIntrinsicWithSize(Size: Memcpy->getLength(), TTI)) {
336 if (UseMemIntrinsicLibFunc &&
337 canEmitMemcpy(ModuleLowering: ModuleLibcalls, TM, F: ParentFunc))
338 break;
339
340 // TODO: For optsize, emit the loop into a separate function
341 expandMemCpyAsLoop(MemCpy: Memcpy, TTI);
342 Changed = true;
343 Memcpy->eraseFromParent();
344 }
345
346 break;
347 }
348 case Intrinsic::memcpy_inline: {
349 // Only expand llvm.memcpy.inline with non-constant length in this
350 // codepath, leaving the current SelectionDAG expansion for constant
351 // length memcpy intrinsics undisturbed.
352 auto *Memcpy = cast<MemCpyInst>(Val: Inst);
353 if (isa<ConstantInt>(Val: Memcpy->getLength()))
354 break;
355
356 Function *ParentFunc = Memcpy->getFunction();
357 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
358 expandMemCpyAsLoop(MemCpy: Memcpy, TTI);
359 Changed = true;
360 Memcpy->eraseFromParent();
361 break;
362 }
363 case Intrinsic::memmove: {
364 auto *Memmove = cast<MemMoveInst>(Val: Inst);
365 Function *ParentFunc = Memmove->getFunction();
366 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
367 if (shouldExpandMemIntrinsicWithSize(Size: Memmove->getLength(), TTI)) {
368 if (UseMemIntrinsicLibFunc &&
369 canEmitLibcall(ModuleLowering: ModuleLibcalls, TM, F: ParentFunc, LC: RTLIB::MEMMOVE))
370 break;
371
372 if (expandMemMoveAsLoop(MemMove: Memmove, TTI)) {
373 Changed = true;
374 Memmove->eraseFromParent();
375 }
376 }
377
378 break;
379 }
380 case Intrinsic::memset: {
381 auto *Memset = cast<MemSetInst>(Val: Inst);
382 Function *ParentFunc = Memset->getFunction();
383 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
384 if (shouldExpandMemIntrinsicWithSize(Size: Memset->getLength(), TTI)) {
385 if (UseMemIntrinsicLibFunc &&
386 canEmitLibcall(ModuleLowering: ModuleLibcalls, TM, F: ParentFunc, LC: RTLIB::MEMSET))
387 break;
388
389 expandMemSetAsLoop(MemSet: Memset, TTI);
390 Changed = true;
391 Memset->eraseFromParent();
392 }
393
394 break;
395 }
396 case Intrinsic::memset_inline: {
397 // Only expand llvm.memset.inline with non-constant length in this
398 // codepath, leaving the current SelectionDAG expansion for constant
399 // length memset intrinsics undisturbed.
400 auto *Memset = cast<MemSetInst>(Val: Inst);
401 if (isa<ConstantInt>(Val: Memset->getLength()))
402 break;
403
404 Function *ParentFunc = Memset->getFunction();
405 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
406 expandMemSetAsLoop(MemSet: Memset, TTI);
407 Changed = true;
408 Memset->eraseFromParent();
409 break;
410 }
411 case Intrinsic::experimental_memset_pattern: {
412 auto *Memset = cast<MemSetPatternInst>(Val: Inst);
413 const TargetLibraryInfo &TLI = LookupTLI(*Memset->getFunction());
414 Constant *PatternValue = getMemSetPattern16Value(Inst: Memset, TLI);
415 if (!PatternValue) {
416 // If it isn't possible to emit a memset_pattern16 libcall, expand to
417 // a loop instead.
418 expandMemSetPatternAsLoop(MemSet: Memset);
419 Changed = true;
420 Memset->eraseFromParent();
421 break;
422 }
423 // FIXME: There is currently no profitability calculation for emitting
424 // the libcall vs expanding the memset.pattern directly.
425 IRBuilder<> Builder(Inst);
426 Module *M = Memset->getModule();
427 const DataLayout &DL = Memset->getDataLayout();
428
429 Type *DestPtrTy = Memset->getRawDest()->getType();
430 Type *SizeTTy = TLI.getSizeTType(M: *M);
431 StringRef FuncName = "memset_pattern16";
432 FunctionCallee MSP = getOrInsertLibFunc(M, TLI, TheLibFunc: LibFunc_memset_pattern16,
433 RetTy: Builder.getVoidTy(), Args: DestPtrTy,
434 Args: Builder.getPtrTy(), Args: SizeTTy);
435 inferNonMandatoryLibFuncAttrs(M, Name: FuncName, TLI);
436
437 // Otherwise we should form a memset_pattern16. PatternValue is known
438 // to be an constant array of 16-bytes. Put the value into a mergable
439 // global.
440 assert(Memset->getRawDest()->getType()->getPointerAddressSpace() == 0 &&
441 "Should have skipped if non-zero AS");
442 GlobalVariable *GV;
443 auto It = CMap.find(Val: PatternValue);
444 if (It != CMap.end()) {
445 GV = It->second;
446 } else {
447 GV = new GlobalVariable(
448 *M, PatternValue->getType(), /*isConstant=*/true,
449 GlobalValue::PrivateLinkage, PatternValue, ".memset_pattern");
450 GV->setUnnamedAddr(
451 GlobalValue::UnnamedAddr::Global); // Ok to merge these.
452 // TODO: Consider relaxing alignment requirement.
453 GV->setAlignment(Align(16));
454 CMap[PatternValue] = GV;
455 }
456 Value *PatternPtr = GV;
457 Value *NumBytes = Builder.CreateMul(
458 LHS: TLI.getAsSizeT(V: DL.getTypeAllocSize(Ty: Memset->getValue()->getType()),
459 M: *M),
460 RHS: Builder.CreateZExtOrTrunc(V: Memset->getLength(), DestTy: SizeTTy));
461 CallInst *MemsetPattern16Call =
462 Builder.CreateCall(Callee: MSP, Args: {Memset->getRawDest(), PatternPtr, NumBytes});
463 MemsetPattern16Call->setAAMetadata(Memset->getAAMetadata());
464 // Preserve any call site attributes on the destination pointer
465 // argument (e.g. alignment).
466 AttrBuilder ArgAttrs(Memset->getContext(),
467 Memset->getAttributes().getParamAttrs(ArgNo: 0));
468 MemsetPattern16Call->setAttributes(
469 MemsetPattern16Call->getAttributes().addParamAttributes(
470 C&: Memset->getContext(), ArgNo: 0, B: ArgAttrs));
471 Changed = true;
472 Memset->eraseFromParent();
473 break;
474 }
475 default:
476 llvm_unreachable("unhandled intrinsic");
477 }
478 }
479
480 return Changed;
481}
482
483static bool expandProtectedFieldPtr(Function &Intr) {
484 Module &M = *Intr.getParent();
485
486 SmallPtrSet<GlobalValue *, 2> DSsToDeactivate;
487
488 Type *Int8Ty = Type::getInt8Ty(C&: M.getContext());
489 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
490 PointerType *PtrTy = PointerType::get(C&: M.getContext(), AddressSpace: 0);
491
492 Function *SignIntr =
493 Intrinsic::getOrInsertDeclaration(M: &M, id: Intrinsic::ptrauth_sign, Tys: {});
494 Function *AuthIntr =
495 Intrinsic::getOrInsertDeclaration(M: &M, id: Intrinsic::ptrauth_auth, Tys: {});
496
497 auto *EmuFnTy = FunctionType::get(Result: Int64Ty, Params: {Int64Ty, Int64Ty}, isVarArg: false);
498
499 auto CreateSign = [&](IRBuilder<> &B, Value *Val, Value *Disc,
500 OperandBundleDef DSBundle) {
501 Function *F = B.GetInsertBlock()->getParent();
502 Attribute FSAttr = F->getFnAttribute(Kind: "target-features");
503 if (FSAttr.isValid() && FSAttr.getValueAsString().contains(Other: "+pauth"))
504 return B.CreateCall(
505 Callee: SignIntr, Args: {Val, B.getInt32(/*AArch64PACKey::DA*/ C: 2), Disc}, OpBundles: DSBundle);
506 FunctionCallee EmuSignIntr =
507 M.getOrInsertFunction(Name: "__emupac_pacda", T: EmuFnTy);
508 return B.CreateCall(Callee: EmuSignIntr, Args: {Val, Disc}, OpBundles: DSBundle);
509 };
510
511 auto CreateAuth = [&](IRBuilder<> &B, Value *Val, Value *Disc,
512 OperandBundleDef DSBundle) {
513 Function *F = B.GetInsertBlock()->getParent();
514 Attribute FSAttr = F->getFnAttribute(Kind: "target-features");
515 if (FSAttr.isValid() && FSAttr.getValueAsString().contains(Other: "+pauth"))
516 return B.CreateCall(
517 Callee: AuthIntr, Args: {Val, B.getInt32(/*AArch64PACKey::DA*/ C: 2), Disc}, OpBundles: DSBundle);
518 FunctionCallee EmuAuthIntr =
519 M.getOrInsertFunction(Name: "__emupac_autda", T: EmuFnTy);
520 return B.CreateCall(Callee: EmuAuthIntr, Args: {Val, Disc}, OpBundles: DSBundle);
521 };
522
523 auto GetDeactivationSymbol = [&](CallInst *Call) -> GlobalValue * {
524 if (auto Bundle =
525 Call->getOperandBundle(ID: LLVMContext::OB_deactivation_symbol))
526 return cast<GlobalValue>(Val: Bundle->Inputs[0]);
527 return nullptr;
528 };
529
530 for (User *U : llvm::make_early_inc_range(Range: Intr.users())) {
531 auto *Call = cast<CallInst>(Val: U);
532
533 auto *Pointer = Call->getArgOperand(i: 0);
534 auto *Disc = Call->getArgOperand(i: 1);
535 bool UseHWEncoding =
536 cast<ConstantInt>(Val: Call->getArgOperand(i: 2))->getZExtValue();
537 if (!UseHWEncoding)
538 reportFatalUsageError(reason: "software encoding currently unsupported");
539
540 auto *DS = GetDeactivationSymbol(Call);
541 OperandBundleDef DSBundle("deactivation-symbol", DS);
542
543 for (Use &U : llvm::make_early_inc_range(Range: Call->uses())) {
544 // Insert code to encode each pointer stored to the pointer returned by
545 // the intrinsic.
546 if (auto *SI = dyn_cast<StoreInst>(Val: U.getUser())) {
547 if (U.getOperandNo() == 1 &&
548 isa<PointerType>(Val: SI->getValueOperand()->getType())) {
549 IRBuilder<> B(SI);
550 auto *SIValInt =
551 B.CreatePtrToInt(V: SI->getValueOperand(), DestTy: B.getInt64Ty());
552 Value *Sign = CreateSign(B, SIValInt, Disc, DSBundle);
553 SI->setOperand(i_nocapture: 0, Val_nocapture: B.CreateIntToPtr(V: Sign, DestTy: B.getPtrTy()));
554 SI->setOperand(i_nocapture: 1, Val_nocapture: Pointer);
555 continue;
556 }
557 }
558
559 // Insert code to decode each pointer loaded from the pointer returned by
560 // the intrinsic. This is the inverse of the encode operation implemented
561 // above.
562 if (auto *LI = dyn_cast<LoadInst>(Val: U.getUser())) {
563 if (isa<PointerType>(Val: LI->getType())) {
564 IRBuilder<> B(LI);
565 auto *NewLI = cast<LoadInst>(Val: LI->clone());
566 NewLI->setOperand(i_nocapture: 0, Val_nocapture: Pointer);
567 B.Insert(I: NewLI);
568 auto *LIInt = B.CreatePtrToInt(V: NewLI, DestTy: B.getInt64Ty());
569 Value *Auth = CreateAuth(B, LIInt, Disc, DSBundle);
570 LI->replaceAllUsesWith(V: B.CreateIntToPtr(V: Auth, DestTy: B.getPtrTy()));
571 LI->eraseFromParent();
572 continue;
573 }
574 }
575 // Comparisons against null cannot be used to recover the original
576 // pointer so we replace them with comparisons against the original
577 // pointer.
578 if (auto *CI = dyn_cast<ICmpInst>(Val: U.getUser())) {
579 if (auto *Op = dyn_cast<Constant>(Val: CI->getOperand(i_nocapture: 0))) {
580 if (Op->isNullValue()) {
581 CI->setOperand(i_nocapture: 1, Val_nocapture: Pointer);
582 continue;
583 }
584 }
585 if (auto *Op = dyn_cast<Constant>(Val: CI->getOperand(i_nocapture: 1))) {
586 if (Op->isNullValue()) {
587 CI->setOperand(i_nocapture: 0, Val_nocapture: Pointer);
588 continue;
589 }
590 }
591 }
592
593 // We couldn't rewrite away this use of the intrinsic. Replace it with the
594 // pointer operand, and arrange to define a deactivation symbol.
595 U.set(Pointer);
596 if (DS)
597 DSsToDeactivate.insert(Ptr: DS);
598 }
599
600 Call->eraseFromParent();
601 }
602
603 if (!DSsToDeactivate.empty()) {
604 // This is an AArch64 NOP instruction. When the deactivation symbol support
605 // is expanded to more architectures, there will likely need to be an API
606 // for retrieving this constant.
607 Constant *Nop =
608 ConstantExpr::getIntToPtr(C: ConstantInt::get(Ty: Int64Ty, V: 0xd503201f), Ty: PtrTy);
609 for (GlobalValue *OldDS : DSsToDeactivate) {
610 GlobalValue *DS = GlobalAlias::create(
611 Ty: Int8Ty, AddressSpace: 0, Linkage: GlobalValue::ExternalLinkage, Name: OldDS->getName(), Aliasee: Nop, Parent: &M);
612 DS->setVisibility(GlobalValue::HiddenVisibility);
613 DS->takeName(V: OldDS);
614 OldDS->replaceAllUsesWith(V: DS);
615 OldDS->eraseFromParent();
616 }
617 }
618 return true;
619}
620
621static bool expandCondLoop(Function &Intr) {
622 for (User *U : llvm::make_early_inc_range(Range: Intr.users())) {
623 auto *Call = cast<CallInst>(Val: U);
624
625 auto *Br = cast<BranchInst>(
626 Val: SplitBlockAndInsertIfThen(Cond: Call->getArgOperand(i: 0), SplitBefore: Call, Unreachable: false,
627 BranchWeights: getExplicitlyUnknownBranchWeightsIfProfiled(
628 F&: *Call->getFunction(), DEBUG_TYPE)));
629 Br->setSuccessor(idx: 0, NewSucc: Br->getParent());
630 Call->eraseFromParent();
631 }
632 return true;
633}
634
635static bool expandLoopTrap(Function &Intr) {
636 for (User *U : make_early_inc_range(Range: Intr.users())) {
637 auto *Call = cast<CallInst>(Val: U);
638 if (!Call->getParent()->isEntryBlock() &&
639 std::all_of(first: Call->getParent()->begin(), last: BasicBlock::iterator(Call),
640 pred: [](Instruction &I) { return !I.mayHaveSideEffects(); })) {
641 for (auto *BB : predecessors(BB: Call->getParent())) {
642 auto *BI = dyn_cast<BranchInst>(Val: BB->getTerminator());
643 if (!BI || BI->isUnconditional())
644 continue;
645 IRBuilder<> B(BI);
646 Value *Cond;
647 // The looptrap can either be on the true branch or the false branch.
648 // We insert the cond loop before the branch, which uses the branch's
649 // original condition for going to the looptrap as its condition, and
650 // force the branch to take whichever path does not lead to the
651 // looptrap, as the original path to the looptrap is now unreachable
652 // thanks to the cond loop. The codegenprepare pass will clean up our
653 // "unconditional conditional branch" by combining the two basic blocks
654 // if possible, or replacing it with an unconditional branch.
655 if (BI->getSuccessor(i: 0) == Call->getParent()) {
656 // The looptrap is on the true branch.
657 Cond = BI->getCondition();
658 BI->setCondition(ConstantInt::getFalse(Context&: BI->getContext()));
659 } else {
660 // The looptrap is on the false branch, which means that we need to
661 // invert the condition.
662 Cond = B.CreateNot(V: BI->getCondition());
663 BI->setCondition(ConstantInt::getTrue(Context&: BI->getContext()));
664 }
665 B.CreateIntrinsic(ID: Intrinsic::cond_loop, Args: Cond);
666 }
667 }
668 IRBuilder<> B(Call);
669 B.CreateIntrinsic(ID: Intrinsic::cond_loop,
670 Args: ConstantInt::getTrue(Context&: Call->getContext()));
671 Call->eraseFromParent();
672 }
673 return true;
674}
675
676bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
677 // Map unique constants to globals.
678 DenseMap<Constant *, GlobalVariable *> CMap;
679 bool Changed = false;
680 for (Function &F : M) {
681 switch (F.getIntrinsicID()) {
682 default:
683 break;
684 case Intrinsic::memcpy:
685 case Intrinsic::memcpy_inline:
686 case Intrinsic::memmove:
687 case Intrinsic::memset:
688 case Intrinsic::memset_inline:
689 case Intrinsic::experimental_memset_pattern:
690 Changed |= expandMemIntrinsicUses(F, CMap);
691 break;
692 case Intrinsic::load_relative:
693 Changed |= lowerLoadRelative(F);
694 break;
695 case Intrinsic::is_constant:
696 case Intrinsic::objectsize:
697 Changed |= forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
698 Function *Parent = CI->getParent()->getParent();
699 TargetLibraryInfo &TLI = LookupTLI(*Parent);
700 // Intrinsics in unreachable code are not lowered.
701 bool Changed = lowerConstantIntrinsics(F&: *Parent, TLI, /*DT=*/nullptr);
702 return Changed;
703 });
704 break;
705#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
706 case Intrinsic::VPID:
707#include "llvm/IR/VPIntrinsics.def"
708 forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
709 Function *Parent = CI->getParent()->getParent();
710 const TargetTransformInfo &TTI = LookupTTI(*Parent);
711 auto *VPI = cast<VPIntrinsic>(Val: CI);
712 VPExpansionDetails ED = expandVectorPredicationIntrinsic(VPI&: *VPI, TTI);
713 // Expansion of VP intrinsics may change the IR but not actually
714 // replace the intrinsic, so update Changed for the pass
715 // and compute Removed for forEachCall.
716 Changed |= ED != VPExpansionDetails::IntrinsicUnchanged;
717 bool Removed = ED == VPExpansionDetails::IntrinsicReplaced;
718 return Removed;
719 });
720 break;
721 case Intrinsic::objc_autorelease:
722 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autorelease);
723 break;
724 case Intrinsic::objc_autoreleasePoolPop:
725 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autoreleasePoolPop);
726 break;
727 case Intrinsic::objc_autoreleasePoolPush:
728 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autoreleasePoolPush);
729 break;
730 case Intrinsic::objc_autoreleaseReturnValue:
731 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_autoreleaseReturnValue);
732 break;
733 case Intrinsic::objc_copyWeak:
734 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_copyWeak);
735 break;
736 case Intrinsic::objc_destroyWeak:
737 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_destroyWeak);
738 break;
739 case Intrinsic::objc_initWeak:
740 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_initWeak);
741 break;
742 case Intrinsic::objc_loadWeak:
743 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_loadWeak);
744 break;
745 case Intrinsic::objc_loadWeakRetained:
746 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_loadWeakRetained);
747 break;
748 case Intrinsic::objc_moveWeak:
749 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_moveWeak);
750 break;
751 case Intrinsic::objc_release:
752 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_release, setNonLazyBind: true);
753 break;
754 case Intrinsic::objc_retain:
755 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retain, setNonLazyBind: true);
756 break;
757 case Intrinsic::objc_retainAutorelease:
758 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainAutorelease);
759 break;
760 case Intrinsic::objc_retainAutoreleaseReturnValue:
761 Changed |=
762 lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainAutoreleaseReturnValue);
763 break;
764 case Intrinsic::objc_retainAutoreleasedReturnValue:
765 Changed |=
766 lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainAutoreleasedReturnValue);
767 break;
768 case Intrinsic::objc_claimAutoreleasedReturnValue:
769 Changed |=
770 lowerObjCCall(F, NewFn: RTLIB::impl_objc_claimAutoreleasedReturnValue);
771 break;
772 case Intrinsic::objc_retainBlock:
773 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainBlock);
774 break;
775 case Intrinsic::objc_storeStrong:
776 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_storeStrong);
777 break;
778 case Intrinsic::objc_storeWeak:
779 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_storeWeak);
780 break;
781 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
782 Changed |=
783 lowerObjCCall(F, NewFn: RTLIB::impl_objc_unsafeClaimAutoreleasedReturnValue);
784 break;
785 case Intrinsic::objc_retainedObject:
786 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retainedObject);
787 break;
788 case Intrinsic::objc_unretainedObject:
789 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_unretainedObject);
790 break;
791 case Intrinsic::objc_unretainedPointer:
792 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_unretainedPointer);
793 break;
794 case Intrinsic::objc_retain_autorelease:
795 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_retain_autorelease);
796 break;
797 case Intrinsic::objc_sync_enter:
798 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_sync_enter);
799 break;
800 case Intrinsic::objc_sync_exit:
801 Changed |= lowerObjCCall(F, NewFn: RTLIB::impl_objc_sync_exit);
802 break;
803 case Intrinsic::exp:
804 case Intrinsic::exp2:
805 case Intrinsic::log:
806 Changed |= forEachCall(Intrin&: F, Callback: [&](CallInst *CI) {
807 Type *Ty = CI->getArgOperand(i: 0)->getType();
808 if (!TM || !isa<ScalableVectorType>(Val: Ty))
809 return false;
810 const TargetLowering *TL = TM->getSubtargetImpl(F)->getTargetLowering();
811 unsigned Op = TL->IntrinsicIDToISD(ID: F.getIntrinsicID());
812 assert(Op != ISD::DELETED_NODE && "unsupported intrinsic");
813 if (!TL->isOperationExpand(Op, VT: EVT::getEVT(Ty)))
814 return false;
815 return lowerUnaryVectorIntrinsicAsLoop(M, CI);
816 });
817 break;
818 case Intrinsic::protected_field_ptr:
819 Changed |= expandProtectedFieldPtr(Intr&: F);
820 break;
821 case Intrinsic::cond_loop:
822 if (!TM->canLowerCondLoop())
823 Changed |= expandCondLoop(Intr&: F);
824 break;
825 case Intrinsic::looptrap:
826 Changed |= expandLoopTrap(Intr&: F);
827 if (!TM->canLowerCondLoop())
828 if (auto *CondLoop = M.getFunction(Name: "llvm.cond.loop"))
829 Changed |= expandCondLoop(Intr&: *CondLoop);
830 break;
831 }
832 }
833 return Changed;
834}
835
836namespace {
837
838class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
839public:
840 static char ID;
841
842 PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}
843
844 void getAnalysisUsage(AnalysisUsage &AU) const override {
845 AU.addRequired<TargetTransformInfoWrapperPass>();
846 AU.addRequired<TargetLibraryInfoWrapperPass>();
847 AU.addRequired<LibcallLoweringInfoWrapper>();
848 AU.addRequired<TargetPassConfig>();
849 }
850
851 bool runOnModule(Module &M) override {
852 const LibcallLoweringModuleAnalysisResult &ModuleLibcalls =
853 getAnalysis<LibcallLoweringInfoWrapper>().getResult(M);
854
855 auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {
856 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
857 };
858 auto LookupTLI = [this](Function &F) -> TargetLibraryInfo & {
859 return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
860 };
861
862 const auto *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
863 PreISelIntrinsicLowering Lowering(TM, ModuleLibcalls, LookupTTI, LookupTLI);
864 return Lowering.lowerIntrinsics(M);
865 }
866};
867
868} // end anonymous namespace
869
870char PreISelIntrinsicLoweringLegacyPass::ID;
871
872INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass,
873 "pre-isel-intrinsic-lowering",
874 "Pre-ISel Intrinsic Lowering", false, false)
875INITIALIZE_PASS_DEPENDENCY(LibcallLoweringInfoWrapper)
876INITIALIZE_PASS_DEPENDENCY(RuntimeLibraryInfoWrapper)
877INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
878INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
879INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
880INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
881 "pre-isel-intrinsic-lowering",
882 "Pre-ISel Intrinsic Lowering", false, false)
883
884ModulePass *llvm::createPreISelIntrinsicLoweringPass() {
885 return new PreISelIntrinsicLoweringLegacyPass();
886}
887
888PreservedAnalyses
889PreISelIntrinsicLoweringPass::run(Module &M, ModuleAnalysisManager &MAM) {
890 const LibcallLoweringModuleAnalysisResult &LibcallLowering =
891 MAM.getResult<LibcallLoweringModuleAnalysis>(IR&: M);
892
893 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
894
895 auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {
896 return FAM.getResult<TargetIRAnalysis>(IR&: F);
897 };
898 auto LookupTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
899 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
900 };
901
902 PreISelIntrinsicLowering Lowering(TM, LibcallLowering, LookupTTI, LookupTLI);
903 if (!Lowering.lowerIntrinsics(M))
904 return PreservedAnalyses::all();
905 else
906 return PreservedAnalyses::none();
907}
908