1//===-- IntrinsicLowering.cpp - Intrinsic Lowering default implementation -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the IntrinsicLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/IntrinsicLowering.h"
14#include "llvm/ADT/SmallVector.h"
15#include "llvm/IR/Constants.h"
16#include "llvm/IR/DataLayout.h"
17#include "llvm/IR/DerivedTypes.h"
18#include "llvm/IR/IRBuilder.h"
19#include "llvm/IR/Module.h"
20#include "llvm/IR/Type.h"
21#include "llvm/Support/ErrorHandling.h"
22#include "llvm/Support/raw_ostream.h"
23using namespace llvm;
24
25/// This function is used when we want to lower an intrinsic call to a call of
26/// an external function. This handles hard cases such as when there was already
27/// a prototype for the external function, but that prototype doesn't match the
28/// arguments we expect to pass in.
29template <class ArgIt>
30static CallInst *ReplaceCallWith(const char *NewFn, CallInst *CI,
31 ArgIt ArgBegin, ArgIt ArgEnd,
32 Type *RetTy) {
33 // If we haven't already looked up this function, check to see if the
34 // program already contains a function with this name.
35 Module *M = CI->getModule();
36 // Get or insert the definition now.
37 std::vector<Type *> ParamTys;
38 for (ArgIt I = ArgBegin; I != ArgEnd; ++I)
39 ParamTys.push_back((*I)->getType());
40 FunctionCallee FCache =
41 M->getOrInsertFunction(Name: NewFn, T: FunctionType::get(Result: RetTy, Params: ParamTys, isVarArg: false));
42
43 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
44 SmallVector<Value *, 8> Args(ArgBegin, ArgEnd);
45 CallInst *NewCI = Builder.CreateCall(Callee: FCache, Args);
46 NewCI->setName(CI->getName());
47 if (!CI->use_empty())
48 CI->replaceAllUsesWith(V: NewCI);
49 return NewCI;
50}
51
52/// Emit the code to lower bswap of V before the specified instruction IP.
53static Value *LowerBSWAP(LLVMContext &Context, Value *V, Instruction *IP) {
54 assert(V->getType()->isIntOrIntVectorTy() && "Can't bswap a non-integer type!");
55
56 unsigned BitSize = V->getType()->getScalarSizeInBits();
57
58 IRBuilder<> Builder(IP);
59
60 switch(BitSize) {
61 default: llvm_unreachable("Unhandled type size of value to byteswap!");
62 case 16: {
63 Value *Tmp1 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
64 Name: "bswap.2");
65 Value *Tmp2 = Builder.CreateLShr(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
66 Name: "bswap.1");
67 V = Builder.CreateOr(LHS: Tmp1, RHS: Tmp2, Name: "bswap.i16");
68 break;
69 }
70 case 32: {
71 Value *Tmp4 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 24),
72 Name: "bswap.4");
73 Value *Tmp3 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
74 Name: "bswap.3");
75 Value *Tmp2 = Builder.CreateLShr(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
76 Name: "bswap.2");
77 Value *Tmp1 = Builder.CreateLShr(LHS: V,RHS: ConstantInt::get(Ty: V->getType(), V: 24),
78 Name: "bswap.1");
79 Tmp3 = Builder.CreateAnd(LHS: Tmp3,
80 RHS: ConstantInt::get(Ty: V->getType(), V: 0xFF0000),
81 Name: "bswap.and3");
82 Tmp2 = Builder.CreateAnd(LHS: Tmp2,
83 RHS: ConstantInt::get(Ty: V->getType(), V: 0xFF00),
84 Name: "bswap.and2");
85 Tmp4 = Builder.CreateOr(LHS: Tmp4, RHS: Tmp3, Name: "bswap.or1");
86 Tmp2 = Builder.CreateOr(LHS: Tmp2, RHS: Tmp1, Name: "bswap.or2");
87 V = Builder.CreateOr(LHS: Tmp4, RHS: Tmp2, Name: "bswap.i32");
88 break;
89 }
90 case 64: {
91 Value *Tmp8 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 56),
92 Name: "bswap.8");
93 Value *Tmp7 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 40),
94 Name: "bswap.7");
95 Value *Tmp6 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 24),
96 Name: "bswap.6");
97 Value *Tmp5 = Builder.CreateShl(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
98 Name: "bswap.5");
99 Value* Tmp4 = Builder.CreateLShr(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 8),
100 Name: "bswap.4");
101 Value* Tmp3 = Builder.CreateLShr(LHS: V,
102 RHS: ConstantInt::get(Ty: V->getType(), V: 24),
103 Name: "bswap.3");
104 Value* Tmp2 = Builder.CreateLShr(LHS: V,
105 RHS: ConstantInt::get(Ty: V->getType(), V: 40),
106 Name: "bswap.2");
107 Value* Tmp1 = Builder.CreateLShr(LHS: V,
108 RHS: ConstantInt::get(Ty: V->getType(), V: 56),
109 Name: "bswap.1");
110 Tmp7 = Builder.CreateAnd(LHS: Tmp7,
111 RHS: ConstantInt::get(Ty: V->getType(),
112 V: 0xFF000000000000ULL),
113 Name: "bswap.and7");
114 Tmp6 = Builder.CreateAnd(LHS: Tmp6,
115 RHS: ConstantInt::get(Ty: V->getType(),
116 V: 0xFF0000000000ULL),
117 Name: "bswap.and6");
118 Tmp5 = Builder.CreateAnd(LHS: Tmp5,
119 RHS: ConstantInt::get(Ty: V->getType(),
120 V: 0xFF00000000ULL),
121 Name: "bswap.and5");
122 Tmp4 = Builder.CreateAnd(LHS: Tmp4,
123 RHS: ConstantInt::get(Ty: V->getType(),
124 V: 0xFF000000ULL),
125 Name: "bswap.and4");
126 Tmp3 = Builder.CreateAnd(LHS: Tmp3,
127 RHS: ConstantInt::get(Ty: V->getType(),
128 V: 0xFF0000ULL),
129 Name: "bswap.and3");
130 Tmp2 = Builder.CreateAnd(LHS: Tmp2,
131 RHS: ConstantInt::get(Ty: V->getType(),
132 V: 0xFF00ULL),
133 Name: "bswap.and2");
134 Tmp8 = Builder.CreateOr(LHS: Tmp8, RHS: Tmp7, Name: "bswap.or1");
135 Tmp6 = Builder.CreateOr(LHS: Tmp6, RHS: Tmp5, Name: "bswap.or2");
136 Tmp4 = Builder.CreateOr(LHS: Tmp4, RHS: Tmp3, Name: "bswap.or3");
137 Tmp2 = Builder.CreateOr(LHS: Tmp2, RHS: Tmp1, Name: "bswap.or4");
138 Tmp8 = Builder.CreateOr(LHS: Tmp8, RHS: Tmp6, Name: "bswap.or5");
139 Tmp4 = Builder.CreateOr(LHS: Tmp4, RHS: Tmp2, Name: "bswap.or6");
140 V = Builder.CreateOr(LHS: Tmp8, RHS: Tmp4, Name: "bswap.i64");
141 break;
142 }
143 }
144 return V;
145}
146
147/// Emit the code to lower ctpop of V before the specified instruction IP.
148static Value *LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP) {
149 assert(V->getType()->isIntegerTy() && "Can't ctpop a non-integer type!");
150
151 static const uint64_t MaskValues[6] = {
152 0x5555555555555555ULL, 0x3333333333333333ULL,
153 0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL,
154 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL
155 };
156
157 IRBuilder<> Builder(IP);
158
159 unsigned BitSize = V->getType()->getPrimitiveSizeInBits();
160 unsigned WordSize = (BitSize + 63) / 64;
161 Value *Count = ConstantInt::get(Ty: V->getType(), V: 0);
162
163 for (unsigned n = 0; n < WordSize; ++n) {
164 Value *PartValue = V;
165 for (unsigned i = 1, ct = 0; i < (BitSize>64 ? 64 : BitSize);
166 i <<= 1, ++ct) {
167 Value *MaskCst = ConstantInt::get(Ty: V->getType(), V: MaskValues[ct]);
168 Value *LHS = Builder.CreateAnd(LHS: PartValue, RHS: MaskCst, Name: "cppop.and1");
169 Value *VShift = Builder.CreateLShr(LHS: PartValue,
170 RHS: ConstantInt::get(Ty: V->getType(), V: i),
171 Name: "ctpop.sh");
172 Value *RHS = Builder.CreateAnd(LHS: VShift, RHS: MaskCst, Name: "cppop.and2");
173 PartValue = Builder.CreateAdd(LHS, RHS, Name: "ctpop.step");
174 }
175 Count = Builder.CreateAdd(LHS: PartValue, RHS: Count, Name: "ctpop.part");
176 if (BitSize > 64) {
177 V = Builder.CreateLShr(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 64),
178 Name: "ctpop.part.sh");
179 BitSize -= 64;
180 }
181 }
182
183 return Count;
184}
185
186/// Emit the code to lower ctlz of V before the specified instruction IP.
187static Value *LowerCTLZ(LLVMContext &Context, Value *V, Instruction *IP) {
188
189 IRBuilder<> Builder(IP);
190
191 unsigned BitSize = V->getType()->getPrimitiveSizeInBits();
192 for (unsigned i = 1; i < BitSize; i <<= 1) {
193 Value *ShVal = ConstantInt::get(Ty: V->getType(), V: i);
194 ShVal = Builder.CreateLShr(LHS: V, RHS: ShVal, Name: "ctlz.sh");
195 V = Builder.CreateOr(LHS: V, RHS: ShVal, Name: "ctlz.step");
196 }
197
198 V = Builder.CreateNot(V);
199 return LowerCTPOP(Context, V, IP);
200}
201
202static void ReplaceFPIntrinsicWithCall(CallInst *CI, const char *Fname,
203 const char *Dname,
204 const char *LDname) {
205 switch (CI->getArgOperand(i: 0)->getType()->getTypeID()) {
206 default: llvm_unreachable("Invalid type in intrinsic");
207 case Type::FloatTyID:
208 ReplaceCallWith(NewFn: Fname, CI, ArgBegin: CI->arg_begin(), ArgEnd: CI->arg_end(),
209 RetTy: Type::getFloatTy(C&: CI->getContext()));
210 break;
211 case Type::DoubleTyID:
212 ReplaceCallWith(NewFn: Dname, CI, ArgBegin: CI->arg_begin(), ArgEnd: CI->arg_end(),
213 RetTy: Type::getDoubleTy(C&: CI->getContext()));
214 break;
215 case Type::X86_FP80TyID:
216 case Type::FP128TyID:
217 case Type::PPC_FP128TyID:
218 ReplaceCallWith(NewFn: LDname, CI, ArgBegin: CI->arg_begin(), ArgEnd: CI->arg_end(),
219 RetTy: CI->getArgOperand(i: 0)->getType());
220 break;
221 }
222}
223
224void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) {
225 IRBuilder<> Builder(CI);
226 LLVMContext &Context = CI->getContext();
227
228 const Function *Callee = CI->getCalledFunction();
229 assert(Callee && "Cannot lower an indirect call!");
230
231 switch (Callee->getIntrinsicID()) {
232 case Intrinsic::not_intrinsic:
233 report_fatal_error(reason: "Cannot lower a call to a non-intrinsic function '"+
234 Callee->getName() + "'!");
235 default:
236 report_fatal_error(reason: "Code generator does not support intrinsic function '"+
237 Callee->getName()+"'!");
238
239 case Intrinsic::expect:
240 case Intrinsic::expect_with_probability: {
241 // Just replace __builtin_expect(exp, c) and
242 // __builtin_expect_with_probability(exp, c, p) with EXP.
243 Value *V = CI->getArgOperand(i: 0);
244 CI->replaceAllUsesWith(V);
245 break;
246 }
247
248 case Intrinsic::allow_runtime_check:
249 case Intrinsic::allow_ubsan_check:
250 CI->replaceAllUsesWith(V: ConstantInt::getTrue(Ty: CI->getType()));
251 return;
252
253 case Intrinsic::ctpop:
254 CI->replaceAllUsesWith(V: LowerCTPOP(Context, V: CI->getArgOperand(i: 0), IP: CI));
255 break;
256
257 case Intrinsic::bswap:
258 CI->replaceAllUsesWith(V: LowerBSWAP(Context, V: CI->getArgOperand(i: 0), IP: CI));
259 break;
260
261 case Intrinsic::ctlz:
262 CI->replaceAllUsesWith(V: LowerCTLZ(Context, V: CI->getArgOperand(i: 0), IP: CI));
263 break;
264
265 case Intrinsic::cttz: {
266 // cttz(x) -> ctpop(~X & (X-1))
267 Value *Src = CI->getArgOperand(i: 0);
268 Value *NotSrc = Builder.CreateNot(V: Src);
269 NotSrc->setName(Src->getName() + ".not");
270 Value *SrcM1 = ConstantInt::get(Ty: Src->getType(), V: 1);
271 SrcM1 = Builder.CreateSub(LHS: Src, RHS: SrcM1);
272 Src = LowerCTPOP(Context, V: Builder.CreateAnd(LHS: NotSrc, RHS: SrcM1), IP: CI);
273 CI->replaceAllUsesWith(V: Src);
274 break;
275 }
276
277 case Intrinsic::stacksave:
278 case Intrinsic::stackrestore: {
279 if (!Warned)
280 errs() << "WARNING: this target does not support the llvm.stack"
281 << (Callee->getIntrinsicID() == Intrinsic::stacksave ?
282 "save" : "restore") << " intrinsic.\n";
283 Warned = true;
284 if (Callee->getIntrinsicID() == Intrinsic::stacksave)
285 CI->replaceAllUsesWith(V: Constant::getNullValue(Ty: CI->getType()));
286 break;
287 }
288
289 case Intrinsic::get_dynamic_area_offset:
290 errs() << "WARNING: this target does not support the custom llvm.get."
291 "dynamic.area.offset. It is being lowered to a constant 0\n";
292 // Just lower it to a constant 0 because for most targets
293 // @llvm.get.dynamic.area.offset is lowered to zero.
294 CI->replaceAllUsesWith(V: ConstantInt::get(Ty: CI->getType(), V: 0));
295 break;
296 case Intrinsic::returnaddress:
297 case Intrinsic::frameaddress:
298 errs() << "WARNING: this target does not support the llvm."
299 << (Callee->getIntrinsicID() == Intrinsic::returnaddress ?
300 "return" : "frame") << "address intrinsic.\n";
301 CI->replaceAllUsesWith(
302 V: ConstantPointerNull::get(T: cast<PointerType>(Val: CI->getType())));
303 break;
304 case Intrinsic::addressofreturnaddress:
305 errs() << "WARNING: this target does not support the "
306 "llvm.addressofreturnaddress intrinsic.\n";
307 CI->replaceAllUsesWith(
308 V: ConstantPointerNull::get(T: cast<PointerType>(Val: CI->getType())));
309 break;
310
311 case Intrinsic::prefetch:
312 break; // Simply strip out prefetches on unsupported architectures
313
314 case Intrinsic::pcmarker:
315 break; // Simply strip out pcmarker on unsupported architectures
316 case Intrinsic::readcyclecounter: {
317 errs() << "WARNING: this target does not support the llvm.readcyclecoun"
318 << "ter intrinsic. It is being lowered to a constant 0\n";
319 CI->replaceAllUsesWith(V: ConstantInt::get(Ty: Type::getInt64Ty(C&: Context), V: 0));
320 break;
321 }
322 case Intrinsic::readsteadycounter: {
323 errs() << "WARNING: this target does not support the llvm.readsteadycounter"
324 << " intrinsic. It is being lowered to a constant 0\n";
325 CI->replaceAllUsesWith(V: ConstantInt::get(Ty: Type::getInt64Ty(C&: Context), V: 0));
326 break;
327 }
328
329 case Intrinsic::dbg_declare:
330 case Intrinsic::dbg_label:
331 break; // Simply strip out debugging intrinsics
332
333 case Intrinsic::eh_typeid_for:
334 // Return something different to eh_selector.
335 CI->replaceAllUsesWith(V: ConstantInt::get(Ty: CI->getType(), V: 1));
336 break;
337
338 case Intrinsic::annotation:
339 case Intrinsic::ptr_annotation:
340 // Just drop the annotation, but forward the value
341 CI->replaceAllUsesWith(V: CI->getOperand(i_nocapture: 0));
342 break;
343
344 case Intrinsic::assume:
345 case Intrinsic::experimental_noalias_scope_decl:
346 case Intrinsic::var_annotation:
347 break; // Strip out these intrinsics
348
349 case Intrinsic::memcpy: {
350 Type *IntPtr = DL.getIntPtrType(C&: Context);
351 Value *Size = Builder.CreateIntCast(V: CI->getArgOperand(i: 2), DestTy: IntPtr,
352 /* isSigned */ false);
353 Value *Ops[3];
354 Ops[0] = CI->getArgOperand(i: 0);
355 Ops[1] = CI->getArgOperand(i: 1);
356 Ops[2] = Size;
357 ReplaceCallWith(NewFn: "memcpy", CI, ArgBegin: Ops, ArgEnd: Ops+3, RetTy: CI->getArgOperand(i: 0)->getType());
358 break;
359 }
360 case Intrinsic::memmove: {
361 Type *IntPtr = DL.getIntPtrType(C&: Context);
362 Value *Size = Builder.CreateIntCast(V: CI->getArgOperand(i: 2), DestTy: IntPtr,
363 /* isSigned */ false);
364 Value *Ops[3];
365 Ops[0] = CI->getArgOperand(i: 0);
366 Ops[1] = CI->getArgOperand(i: 1);
367 Ops[2] = Size;
368 ReplaceCallWith(NewFn: "memmove", CI, ArgBegin: Ops, ArgEnd: Ops+3, RetTy: CI->getArgOperand(i: 0)->getType());
369 break;
370 }
371 case Intrinsic::memset: {
372 Value *Op0 = CI->getArgOperand(i: 0);
373 Type *IntPtr = DL.getIntPtrType(Op0->getType());
374 Value *Size = Builder.CreateIntCast(V: CI->getArgOperand(i: 2), DestTy: IntPtr,
375 /* isSigned */ false);
376 Value *Ops[3];
377 Ops[0] = Op0;
378 // Extend the amount to i32.
379 Ops[1] = Builder.CreateIntCast(V: CI->getArgOperand(i: 1),
380 DestTy: Type::getInt32Ty(C&: Context),
381 /* isSigned */ false);
382 Ops[2] = Size;
383 ReplaceCallWith(NewFn: "memset", CI, ArgBegin: Ops, ArgEnd: Ops+3, RetTy: CI->getArgOperand(i: 0)->getType());
384 break;
385 }
386 case Intrinsic::sqrt: {
387 ReplaceFPIntrinsicWithCall(CI, Fname: "sqrtf", Dname: "sqrt", LDname: "sqrtl");
388 break;
389 }
390 case Intrinsic::log: {
391 ReplaceFPIntrinsicWithCall(CI, Fname: "logf", Dname: "log", LDname: "logl");
392 break;
393 }
394 case Intrinsic::log2: {
395 ReplaceFPIntrinsicWithCall(CI, Fname: "log2f", Dname: "log2", LDname: "log2l");
396 break;
397 }
398 case Intrinsic::log10: {
399 ReplaceFPIntrinsicWithCall(CI, Fname: "log10f", Dname: "log10", LDname: "log10l");
400 break;
401 }
402 case Intrinsic::exp: {
403 ReplaceFPIntrinsicWithCall(CI, Fname: "expf", Dname: "exp", LDname: "expl");
404 break;
405 }
406 case Intrinsic::exp2: {
407 ReplaceFPIntrinsicWithCall(CI, Fname: "exp2f", Dname: "exp2", LDname: "exp2l");
408 break;
409 }
410 case Intrinsic::pow: {
411 ReplaceFPIntrinsicWithCall(CI, Fname: "powf", Dname: "pow", LDname: "powl");
412 break;
413 }
414 case Intrinsic::sin: {
415 ReplaceFPIntrinsicWithCall(CI, Fname: "sinf", Dname: "sin", LDname: "sinl");
416 break;
417 }
418 case Intrinsic::cos: {
419 ReplaceFPIntrinsicWithCall(CI, Fname: "cosf", Dname: "cos", LDname: "cosl");
420 break;
421 }
422 case Intrinsic::floor: {
423 ReplaceFPIntrinsicWithCall(CI, Fname: "floorf", Dname: "floor", LDname: "floorl");
424 break;
425 }
426 case Intrinsic::ceil: {
427 ReplaceFPIntrinsicWithCall(CI, Fname: "ceilf", Dname: "ceil", LDname: "ceill");
428 break;
429 }
430 case Intrinsic::trunc: {
431 ReplaceFPIntrinsicWithCall(CI, Fname: "truncf", Dname: "trunc", LDname: "truncl");
432 break;
433 }
434 case Intrinsic::round: {
435 ReplaceFPIntrinsicWithCall(CI, Fname: "roundf", Dname: "round", LDname: "roundl");
436 break;
437 }
438 case Intrinsic::roundeven: {
439 ReplaceFPIntrinsicWithCall(CI, Fname: "roundevenf", Dname: "roundeven", LDname: "roundevenl");
440 break;
441 }
442 case Intrinsic::copysign: {
443 ReplaceFPIntrinsicWithCall(CI, Fname: "copysignf", Dname: "copysign", LDname: "copysignl");
444 break;
445 }
446 case Intrinsic::get_rounding:
447 // Lower to "round to the nearest"
448 if (!CI->getType()->isVoidTy())
449 CI->replaceAllUsesWith(V: ConstantInt::get(Ty: CI->getType(), V: 1));
450 break;
451 case Intrinsic::invariant_start:
452 case Intrinsic::lifetime_start:
453 // Discard region information.
454 CI->replaceAllUsesWith(V: PoisonValue::get(T: CI->getType()));
455 break;
456 case Intrinsic::invariant_end:
457 case Intrinsic::lifetime_end:
458 // Discard region information.
459 break;
460 }
461
462 assert(CI->use_empty() &&
463 "Lowering should have eliminated any uses of the intrinsic call!");
464 CI->eraseFromParent();
465}
466
467bool IntrinsicLowering::LowerToByteSwap(CallInst *CI) {
468 // Verify this is a simple bswap.
469 if (CI->arg_size() != 1 || CI->getType() != CI->getArgOperand(i: 0)->getType() ||
470 !CI->getType()->isIntegerTy())
471 return false;
472
473 IntegerType *Ty = dyn_cast<IntegerType>(Val: CI->getType());
474 if (!Ty)
475 return false;
476
477 // Okay, we can do this xform, do so now.
478 Module *M = CI->getModule();
479 Function *Int = Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::bswap, Tys: Ty);
480
481 Value *Op = CI->getArgOperand(i: 0);
482 Op = CallInst::Create(Func: Int, Args: Op, NameStr: CI->getName(), InsertBefore: CI->getIterator());
483
484 CI->replaceAllUsesWith(V: Op);
485 CI->eraseFromParent();
486 return true;
487}
488