1//===-- IntrinsicLowering.cpp - Intrinsic Lowering default implementation -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the IntrinsicLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/IntrinsicLowering.h"
14#include "llvm/ADT/SmallVector.h"
15#include "llvm/IR/Constants.h"
16#include "llvm/IR/DataLayout.h"
17#include "llvm/IR/DerivedTypes.h"
18#include "llvm/IR/IRBuilder.h"
19#include "llvm/IR/Module.h"
20#include "llvm/IR/Type.h"
21#include "llvm/Support/ErrorHandling.h"
22#include "llvm/Support/raw_ostream.h"
23using namespace llvm;
24
25/// This function is used when we want to lower an intrinsic call to a call of
26/// an external function. This handles hard cases such as when there was already
27/// a prototype for the external function, but that prototype doesn't match the
28/// arguments we expect to pass in.
29template <class ArgIt>
30static CallInst *ReplaceCallWith(const char *NewFn, CallInst *CI,
31 ArgIt ArgBegin, ArgIt ArgEnd,
32 Type *RetTy) {
33 // If we haven't already looked up this function, check to see if the
34 // program already contains a function with this name.
35 Module *M = CI->getModule();
36 // Get or insert the definition now.
37 std::vector<Type *> ParamTys;
38 for (ArgIt I = ArgBegin; I != ArgEnd; ++I)
39 ParamTys.push_back((*I)->getType());
40 FunctionCallee FCache =
41 M->getOrInsertFunction(Name: NewFn, T: FunctionType::get(Result: RetTy, Params: ParamTys, isVarArg: false));
42
43 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
44 SmallVector<Value *, 8> Args(ArgBegin, ArgEnd);
45 CallInst *NewCI = Builder.CreateCall(Callee: FCache, Args);
46 NewCI->setName(CI->getName());
47 if (!CI->use_empty())
48 CI->replaceAllUsesWith(V: NewCI);
49 return NewCI;
50}
51
52/// Emit the code to lower bswap of V before the specified instruction IP.
53static Value *LowerBSWAP(LLVMContext &Context, Value *V, Instruction *IP) {
54 assert(V->getType()->isIntOrIntVectorTy() && "Can't bswap a non-integer type!");
55
56 unsigned BitSize = V->getType()->getScalarSizeInBits();
57
58 IRBuilder<> Builder(IP);
59
60 switch(BitSize) {
61 default: llvm_unreachable("Unhandled type size of value to byteswap!");
62 case 16: {
63 Value *Tmp1 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
64 Name: "bswap.2");
65 Value *Tmp2 = Builder.CreateLShr(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
66 Name: "bswap.1");
67 V = Builder.CreateOr(LHS: Tmp1, RHS: Tmp2, Name: "bswap.i16");
68 break;
69 }
70 case 32: {
71 Value *Tmp4 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 24),
72 Name: "bswap.4");
73 Value *Tmp3 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
74 Name: "bswap.3");
75 Value *Tmp2 = Builder.CreateLShr(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
76 Name: "bswap.2");
77 Value *Tmp1 = Builder.CreateLShr(LHS: V,RHS: ConstantInt::get(Ty: V->getType(), V: 24),
78 Name: "bswap.1");
79 Tmp3 = Builder.CreateAnd(LHS: Tmp3,
80 RHS: ConstantInt::get(Ty: V->getType(), V: 0xFF0000),
81 Name: "bswap.and3");
82 Tmp2 = Builder.CreateAnd(LHS: Tmp2,
83 RHS: ConstantInt::get(Ty: V->getType(), V: 0xFF00),
84 Name: "bswap.and2");
85 Tmp4 = Builder.CreateOr(LHS: Tmp4, RHS: Tmp3, Name: "bswap.or1");
86 Tmp2 = Builder.CreateOr(LHS: Tmp2, RHS: Tmp1, Name: "bswap.or2");
87 V = Builder.CreateOr(LHS: Tmp4, RHS: Tmp2, Name: "bswap.i32");
88 break;
89 }
90 case 64: {
91 Value *Tmp8 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 56),
92 Name: "bswap.8");
93 Value *Tmp7 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 40),
94 Name: "bswap.7");
95 Value *Tmp6 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 24),
96 Name: "bswap.6");
97 Value *Tmp5 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
98 Name: "bswap.5");
99 Value* Tmp4 = Builder.CreateLShr(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
100 Name: "bswap.4");
101 Value* Tmp3 = Builder.CreateLShr(LHS: V,
102 RHS: ConstantInt::get(Ty: V->getType(), V: 24),
103 Name: "bswap.3");
104 Value* Tmp2 = Builder.CreateLShr(LHS: V,
105 RHS: ConstantInt::get(Ty: V->getType(), V: 40),
106 Name: "bswap.2");
107 Value* Tmp1 = Builder.CreateLShr(LHS: V,
108 RHS: ConstantInt::get(Ty: V->getType(), V: 56),
109 Name: "bswap.1");
110 Tmp7 = Builder.CreateAnd(LHS: Tmp7,
111 RHS: ConstantInt::get(Ty: V->getType(),
112 V: 0xFF000000000000ULL),
113 Name: "bswap.and7");
114 Tmp6 = Builder.CreateAnd(LHS: Tmp6,
115 RHS: ConstantInt::get(Ty: V->getType(),
116 V: 0xFF0000000000ULL),
117 Name: "bswap.and6");
118 Tmp5 = Builder.CreateAnd(LHS: Tmp5,
119 RHS: ConstantInt::get(Ty: V->getType(),
120 V: 0xFF00000000ULL),
121 Name: "bswap.and5");
122 Tmp4 = Builder.CreateAnd(LHS: Tmp4,
123 RHS: ConstantInt::get(Ty: V->getType(),
124 V: 0xFF000000ULL),
125 Name: "bswap.and4");
126 Tmp3 = Builder.CreateAnd(LHS: Tmp3,
127 RHS: ConstantInt::get(Ty: V->getType(),
128 V: 0xFF0000ULL),
129 Name: "bswap.and3");
130 Tmp2 = Builder.CreateAnd(LHS: Tmp2,
131 RHS: ConstantInt::get(Ty: V->getType(),
132 V: 0xFF00ULL),
133 Name: "bswap.and2");
134 Tmp8 = Builder.CreateOr(LHS: Tmp8, RHS: Tmp7, Name: "bswap.or1");
135 Tmp6 = Builder.CreateOr(LHS: Tmp6, RHS: Tmp5, Name: "bswap.or2");
136 Tmp4 = Builder.CreateOr(LHS: Tmp4, RHS: Tmp3, Name: "bswap.or3");
137 Tmp2 = Builder.CreateOr(LHS: Tmp2, RHS: Tmp1, Name: "bswap.or4");
138 Tmp8 = Builder.CreateOr(LHS: Tmp8, RHS: Tmp6, Name: "bswap.or5");
139 Tmp4 = Builder.CreateOr(LHS: Tmp4, RHS: Tmp2, Name: "bswap.or6");
140 V = Builder.CreateOr(LHS: Tmp8, RHS: Tmp4, Name: "bswap.i64");
141 break;
142 }
143 }
144 return V;
145}
146
147/// Emit the code to lower ctpop of V before the specified instruction IP.
148static Value *LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP) {
149 assert(V->getType()->isIntegerTy() && "Can't ctpop a non-integer type!");
150
151 static const uint64_t MaskValues[6] = {
152 0x5555555555555555ULL, 0x3333333333333333ULL,
153 0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL,
154 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL
155 };
156
157 IRBuilder<> Builder(IP);
158
159 unsigned BitSize = V->getType()->getPrimitiveSizeInBits();
160 unsigned WordSize = (BitSize + 63) / 64;
161 Value *Count = ConstantInt::get(Ty: V->getType(), V: 0);
162
163 for (unsigned n = 0; n < WordSize; ++n) {
164 Value *PartValue = V;
165 for (unsigned i = 1, ct = 0; i < (BitSize>64 ? 64 : BitSize);
166 i <<= 1, ++ct) {
167 Value *MaskCst = ConstantInt::get(Ty: V->getType(), V: MaskValues[ct]);
168 Value *LHS = Builder.CreateAnd(LHS: PartValue, RHS: MaskCst, Name: "cppop.and1");
169 Value *VShift = Builder.CreateLShr(LHS: PartValue,
170 RHS: ConstantInt::get(Ty: V->getType(), V: i),
171 Name: "ctpop.sh");
172 Value *RHS = Builder.CreateAnd(LHS: VShift, RHS: MaskCst, Name: "cppop.and2");
173 PartValue = Builder.CreateAdd(LHS, RHS, Name: "ctpop.step");
174 }
175 Count = Builder.CreateAdd(LHS: PartValue, RHS: Count, Name: "ctpop.part");
176 if (BitSize > 64) {
177 V = Builder.CreateLShr(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 64),
178 Name: "ctpop.part.sh");
179 BitSize -= 64;
180 }
181 }
182
183 return Count;
184}
185
186/// Emit the code to lower ctlz of V before the specified instruction IP.
187static Value *LowerCTLZ(LLVMContext &Context, Value *V, Instruction *IP) {
188
189 IRBuilder<> Builder(IP);
190
191 unsigned BitSize = V->getType()->getPrimitiveSizeInBits();
192 for (unsigned i = 1; i < BitSize; i <<= 1) {
193 Value *ShVal = ConstantInt::get(Ty: V->getType(), V: i);
194 ShVal = Builder.CreateLShr(LHS: V, RHS: ShVal, Name: "ctlz.sh");
195 V = Builder.CreateOr(LHS: V, RHS: ShVal, Name: "ctlz.step");
196 }
197
198 V = Builder.CreateNot(V);
199 return LowerCTPOP(Context, V, IP);
200}
201
202static void ReplaceFPIntrinsicWithCall(CallInst *CI, const char *Fname,
203 const char *Dname,
204 const char *LDname) {
205 switch (CI->getArgOperand(i: 0)->getType()->getTypeID()) {
206 default: llvm_unreachable("Invalid type in intrinsic");
207 case Type::FloatTyID:
208 ReplaceCallWith(NewFn: Fname, CI, ArgBegin: CI->arg_begin(), ArgEnd: CI->arg_end(),
209 RetTy: Type::getFloatTy(C&: CI->getContext()));
210 break;
211 case Type::DoubleTyID:
212 ReplaceCallWith(NewFn: Dname, CI, ArgBegin: CI->arg_begin(), ArgEnd: CI->arg_end(),
213 RetTy: Type::getDoubleTy(C&: CI->getContext()));
214 break;
215 case Type::X86_FP80TyID:
216 case Type::FP128TyID:
217 case Type::PPC_FP128TyID:
218 ReplaceCallWith(NewFn: LDname, CI, ArgBegin: CI->arg_begin(), ArgEnd: CI->arg_end(),
219 RetTy: CI->getArgOperand(i: 0)->getType());
220 break;
221 }
222}
223
224void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) {
225 IRBuilder<> Builder(CI);
226 LLVMContext &Context = CI->getContext();
227
228 const Function *Callee = CI->getCalledFunction();
229 assert(Callee && "Cannot lower an indirect call!");
230
231 switch (Callee->getIntrinsicID()) {
232 case Intrinsic::not_intrinsic:
233 report_fatal_error(reason: "Cannot lower a call to a non-intrinsic function '"+
234 Callee->getName() + "'!");
235 default:
236 report_fatal_error(reason: "Code generator does not support intrinsic function '"+
237 Callee->getName()+"'!");
238
239 case Intrinsic::expect: {
240 // Just replace __builtin_expect(exp, c) with EXP.
241 Value *V = CI->getArgOperand(i: 0);
242 CI->replaceAllUsesWith(V);
243 break;
244 }
245
246 case Intrinsic::allow_runtime_check:
247 case Intrinsic::allow_ubsan_check:
248 CI->replaceAllUsesWith(V: ConstantInt::getTrue(Ty: CI->getType()));
249 return;
250
251 case Intrinsic::ctpop:
252 CI->replaceAllUsesWith(V: LowerCTPOP(Context, V: CI->getArgOperand(i: 0), IP: CI));
253 break;
254
255 case Intrinsic::bswap:
256 CI->replaceAllUsesWith(V: LowerBSWAP(Context, V: CI->getArgOperand(i: 0), IP: CI));
257 break;
258
259 case Intrinsic::ctlz:
260 CI->replaceAllUsesWith(V: LowerCTLZ(Context, V: CI->getArgOperand(i: 0), IP: CI));
261 break;
262
263 case Intrinsic::cttz: {
264 // cttz(x) -> ctpop(~X & (X-1))
265 Value *Src = CI->getArgOperand(i: 0);
266 Value *NotSrc = Builder.CreateNot(V: Src);
267 NotSrc->setName(Src->getName() + ".not");
268 Value *SrcM1 = ConstantInt::get(Ty: Src->getType(), V: 1);
269 SrcM1 = Builder.CreateSub(LHS: Src, RHS: SrcM1);
270 Src = LowerCTPOP(Context, V: Builder.CreateAnd(LHS: NotSrc, RHS: SrcM1), IP: CI);
271 CI->replaceAllUsesWith(V: Src);
272 break;
273 }
274
275 case Intrinsic::stacksave:
276 case Intrinsic::stackrestore: {
277 if (!Warned)
278 errs() << "WARNING: this target does not support the llvm.stack"
279 << (Callee->getIntrinsicID() == Intrinsic::stacksave ?
280 "save" : "restore") << " intrinsic.\n";
281 Warned = true;
282 if (Callee->getIntrinsicID() == Intrinsic::stacksave)
283 CI->replaceAllUsesWith(V: Constant::getNullValue(Ty: CI->getType()));
284 break;
285 }
286
287 case Intrinsic::get_dynamic_area_offset:
288 errs() << "WARNING: this target does not support the custom llvm.get."
289 "dynamic.area.offset. It is being lowered to a constant 0\n";
290 // Just lower it to a constant 0 because for most targets
291 // @llvm.get.dynamic.area.offset is lowered to zero.
292 CI->replaceAllUsesWith(V: ConstantInt::get(Ty: CI->getType(), V: 0));
293 break;
294 case Intrinsic::returnaddress:
295 case Intrinsic::frameaddress:
296 errs() << "WARNING: this target does not support the llvm."
297 << (Callee->getIntrinsicID() == Intrinsic::returnaddress ?
298 "return" : "frame") << "address intrinsic.\n";
299 CI->replaceAllUsesWith(
300 V: ConstantPointerNull::get(T: cast<PointerType>(Val: CI->getType())));
301 break;
302 case Intrinsic::addressofreturnaddress:
303 errs() << "WARNING: this target does not support the "
304 "llvm.addressofreturnaddress intrinsic.\n";
305 CI->replaceAllUsesWith(
306 V: ConstantPointerNull::get(T: cast<PointerType>(Val: CI->getType())));
307 break;
308
309 case Intrinsic::prefetch:
310 break; // Simply strip out prefetches on unsupported architectures
311
312 case Intrinsic::pcmarker:
313 break; // Simply strip out pcmarker on unsupported architectures
314 case Intrinsic::readcyclecounter: {
315 errs() << "WARNING: this target does not support the llvm.readcyclecoun"
316 << "ter intrinsic. It is being lowered to a constant 0\n";
317 CI->replaceAllUsesWith(V: ConstantInt::get(Ty: Type::getInt64Ty(C&: Context), V: 0));
318 break;
319 }
320 case Intrinsic::readsteadycounter: {
321 errs() << "WARNING: this target does not support the llvm.readsteadycounter"
322 << " intrinsic. It is being lowered to a constant 0\n";
323 CI->replaceAllUsesWith(V: ConstantInt::get(Ty: Type::getInt64Ty(C&: Context), V: 0));
324 break;
325 }
326
327 case Intrinsic::dbg_declare:
328 case Intrinsic::dbg_label:
329 break; // Simply strip out debugging intrinsics
330
331 case Intrinsic::eh_typeid_for:
332 // Return something different to eh_selector.
333 CI->replaceAllUsesWith(V: ConstantInt::get(Ty: CI->getType(), V: 1));
334 break;
335
336 case Intrinsic::annotation:
337 case Intrinsic::ptr_annotation:
338 // Just drop the annotation, but forward the value
339 CI->replaceAllUsesWith(V: CI->getOperand(i_nocapture: 0));
340 break;
341
342 case Intrinsic::assume:
343 case Intrinsic::experimental_noalias_scope_decl:
344 case Intrinsic::var_annotation:
345 break; // Strip out these intrinsics
346
347 case Intrinsic::memcpy: {
348 Type *IntPtr = DL.getIntPtrType(C&: Context);
349 Value *Size = Builder.CreateIntCast(V: CI->getArgOperand(i: 2), DestTy: IntPtr,
350 /* isSigned */ false);
351 Value *Ops[3];
352 Ops[0] = CI->getArgOperand(i: 0);
353 Ops[1] = CI->getArgOperand(i: 1);
354 Ops[2] = Size;
355 ReplaceCallWith(NewFn: "memcpy", CI, ArgBegin: Ops, ArgEnd: Ops+3, RetTy: CI->getArgOperand(i: 0)->getType());
356 break;
357 }
358 case Intrinsic::memmove: {
359 Type *IntPtr = DL.getIntPtrType(C&: Context);
360 Value *Size = Builder.CreateIntCast(V: CI->getArgOperand(i: 2), DestTy: IntPtr,
361 /* isSigned */ false);
362 Value *Ops[3];
363 Ops[0] = CI->getArgOperand(i: 0);
364 Ops[1] = CI->getArgOperand(i: 1);
365 Ops[2] = Size;
366 ReplaceCallWith(NewFn: "memmove", CI, ArgBegin: Ops, ArgEnd: Ops+3, RetTy: CI->getArgOperand(i: 0)->getType());
367 break;
368 }
369 case Intrinsic::memset: {
370 Value *Op0 = CI->getArgOperand(i: 0);
371 Type *IntPtr = DL.getIntPtrType(Op0->getType());
372 Value *Size = Builder.CreateIntCast(V: CI->getArgOperand(i: 2), DestTy: IntPtr,
373 /* isSigned */ false);
374 Value *Ops[3];
375 Ops[0] = Op0;
376 // Extend the amount to i32.
377 Ops[1] = Builder.CreateIntCast(V: CI->getArgOperand(i: 1),
378 DestTy: Type::getInt32Ty(C&: Context),
379 /* isSigned */ false);
380 Ops[2] = Size;
381 ReplaceCallWith(NewFn: "memset", CI, ArgBegin: Ops, ArgEnd: Ops+3, RetTy: CI->getArgOperand(i: 0)->getType());
382 break;
383 }
384 case Intrinsic::sqrt: {
385 ReplaceFPIntrinsicWithCall(CI, Fname: "sqrtf", Dname: "sqrt", LDname: "sqrtl");
386 break;
387 }
388 case Intrinsic::log: {
389 ReplaceFPIntrinsicWithCall(CI, Fname: "logf", Dname: "log", LDname: "logl");
390 break;
391 }
392 case Intrinsic::log2: {
393 ReplaceFPIntrinsicWithCall(CI, Fname: "log2f", Dname: "log2", LDname: "log2l");
394 break;
395 }
396 case Intrinsic::log10: {
397 ReplaceFPIntrinsicWithCall(CI, Fname: "log10f", Dname: "log10", LDname: "log10l");
398 break;
399 }
400 case Intrinsic::exp: {
401 ReplaceFPIntrinsicWithCall(CI, Fname: "expf", Dname: "exp", LDname: "expl");
402 break;
403 }
404 case Intrinsic::exp2: {
405 ReplaceFPIntrinsicWithCall(CI, Fname: "exp2f", Dname: "exp2", LDname: "exp2l");
406 break;
407 }
408 case Intrinsic::pow: {
409 ReplaceFPIntrinsicWithCall(CI, Fname: "powf", Dname: "pow", LDname: "powl");
410 break;
411 }
412 case Intrinsic::sin: {
413 ReplaceFPIntrinsicWithCall(CI, Fname: "sinf", Dname: "sin", LDname: "sinl");
414 break;
415 }
416 case Intrinsic::cos: {
417 ReplaceFPIntrinsicWithCall(CI, Fname: "cosf", Dname: "cos", LDname: "cosl");
418 break;
419 }
420 case Intrinsic::floor: {
421 ReplaceFPIntrinsicWithCall(CI, Fname: "floorf", Dname: "floor", LDname: "floorl");
422 break;
423 }
424 case Intrinsic::ceil: {
425 ReplaceFPIntrinsicWithCall(CI, Fname: "ceilf", Dname: "ceil", LDname: "ceill");
426 break;
427 }
428 case Intrinsic::trunc: {
429 ReplaceFPIntrinsicWithCall(CI, Fname: "truncf", Dname: "trunc", LDname: "truncl");
430 break;
431 }
432 case Intrinsic::round: {
433 ReplaceFPIntrinsicWithCall(CI, Fname: "roundf", Dname: "round", LDname: "roundl");
434 break;
435 }
436 case Intrinsic::roundeven: {
437 ReplaceFPIntrinsicWithCall(CI, Fname: "roundevenf", Dname: "roundeven", LDname: "roundevenl");
438 break;
439 }
440 case Intrinsic::copysign: {
441 ReplaceFPIntrinsicWithCall(CI, Fname: "copysignf", Dname: "copysign", LDname: "copysignl");
442 break;
443 }
444 case Intrinsic::get_rounding:
445 // Lower to "round to the nearest"
446 if (!CI->getType()->isVoidTy())
447 CI->replaceAllUsesWith(V: ConstantInt::get(Ty: CI->getType(), V: 1));
448 break;
449 case Intrinsic::invariant_start:
450 case Intrinsic::lifetime_start:
451 // Discard region information.
452 CI->replaceAllUsesWith(V: UndefValue::get(T: CI->getType()));
453 break;
454 case Intrinsic::invariant_end:
455 case Intrinsic::lifetime_end:
456 // Discard region information.
457 break;
458 }
459
460 assert(CI->use_empty() &&
461 "Lowering should have eliminated any uses of the intrinsic call!");
462 CI->eraseFromParent();
463}
464
465bool IntrinsicLowering::LowerToByteSwap(CallInst *CI) {
466 // Verify this is a simple bswap.
467 if (CI->arg_size() != 1 || CI->getType() != CI->getArgOperand(i: 0)->getType() ||
468 !CI->getType()->isIntegerTy())
469 return false;
470
471 IntegerType *Ty = dyn_cast<IntegerType>(Val: CI->getType());
472 if (!Ty)
473 return false;
474
475 // Okay, we can do this xform, do so now.
476 Module *M = CI->getModule();
477 Function *Int = Intrinsic::getDeclaration(M, id: Intrinsic::bswap, Tys: Ty);
478
479 Value *Op = CI->getArgOperand(i: 0);
480 Op = CallInst::Create(Func: Int, Args: Op, NameStr: CI->getName(), InsertBefore: CI->getIterator());
481
482 CI->replaceAllUsesWith(V: Op);
483 CI->eraseFromParent();
484 return true;
485}
486