1//===- AMDGPULibCalls.cpp -------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file does AMD library function optimizations.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "AMDGPULibFunc.h"
16#include "AMDGPUSubtarget.h"
17#include "llvm/Analysis/AssumptionCache.h"
18#include "llvm/Analysis/TargetLibraryInfo.h"
19#include "llvm/Analysis/ValueTracking.h"
20#include "llvm/IR/AttributeMask.h"
21#include "llvm/IR/Dominators.h"
22#include "llvm/IR/IRBuilder.h"
23#include "llvm/IR/MDBuilder.h"
24#include "llvm/IR/PatternMatch.h"
25#include <cmath>
26
27#define DEBUG_TYPE "amdgpu-simplifylib"
28
29using namespace llvm;
30using namespace llvm::PatternMatch;
31
32static cl::list<std::string> UseNative("amdgpu-use-native",
33 cl::desc("Comma separated list of functions to replace with native, or all"),
34 cl::CommaSeparated, cl::ValueOptional,
35 cl::Hidden);
36
37#define MATH_PI numbers::pi
38#define MATH_E numbers::e
39#define MATH_SQRT2 numbers::sqrt2
40#define MATH_SQRT1_2 numbers::inv_sqrt2
41
42enum class PowKind { Pow, PowR, PowN, RootN };
43
44namespace llvm {
45
46class AMDGPULibCalls {
47private:
48 const AMDGPUOptions &CLOpts;
49 SimplifyQuery SQ;
50
51 using FuncInfo = llvm::AMDGPULibFunc;
52
53 // -fuse-native.
54 bool AllNative = false;
55
56 bool useNativeFunc(const StringRef F) const;
57
58 // Return a pointer (pointer expr) to the function if function definition with
59 // "FuncName" exists. It may create a new function prototype in pre-link mode.
60 FunctionCallee getFunction(Module *M, const FuncInfo &fInfo);
61
62 /// Wrapper around getFunction which tries to use a faster variant if
63 /// available, and falls back to a less fast option.
64 ///
65 /// Return a replacement function for \p fInfo that has float-typed fast
66 /// variants. \p NewFunc is a base replacement function to use. \p
67 /// NewFuncFastVariant is a faster version to use if the calling context knows
68 /// it's legal. If there is no fast variant to use, \p NewFuncFastVariant
69 /// should be EI_NONE.
70 FunctionCallee getFloatFastVariant(Module *M, const FuncInfo &fInfo,
71 FuncInfo &newInfo,
72 AMDGPULibFunc::EFuncId NewFunc,
73 AMDGPULibFunc::EFuncId NewFuncFastVariant);
74
75 bool parseFunctionName(const StringRef &FMangledName, FuncInfo &FInfo);
76
77 bool TDOFold(CallInst *CI, const FuncInfo &FInfo);
78
79 /* Specialized optimizations */
80
81 // pow/powr/pown
82 bool fold_pow(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo);
83
84 /// Peform a fast math expansion of pow, powr, pown or rootn.
85 bool expandFastPow(FPMathOperator *FPOp, IRBuilder<> &B, PowKind Kind);
86
87 bool tryOptimizePow(FPMathOperator *FPOp, IRBuilder<> &B,
88 const FuncInfo &FInfo);
89
90 // rootn
91 bool fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo);
92
93 // -fuse-native for sincos
94 bool sincosUseNative(CallInst *aCI, const FuncInfo &FInfo);
95
96 // evaluate calls if calls' arguments are constants.
97 bool evaluateScalarMathFunc(const FuncInfo &FInfo, APFloat &Res0,
98 APFloat &Res1, Constant *copr0, Constant *copr1);
99 bool evaluateCall(CallInst *aCI, const FuncInfo &FInfo);
100
101 /// Insert a value to sincos function \p Fsincos. Returns (value of sin, value
102 /// of cos, sincos call).
103 std::tuple<Value *, Value *, Value *> insertSinCos(Value *Arg,
104 FastMathFlags FMF,
105 IRBuilder<> &B,
106 FunctionCallee Fsincos);
107
108 // sin/cos
109 bool fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo);
110
111 // __read_pipe/__write_pipe
112 bool fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,
113 const FuncInfo &FInfo);
114
115 /// Substitute a call to a known libcall with an intrinsic call. If \p
116 /// AllowMinSize is true, allow the replacement in a minsize function.
117 bool shouldReplaceLibcallWithIntrinsic(const CallInst *CI,
118 bool AllowMinSizeF32 = false,
119 bool AllowF64 = false,
120 bool AllowStrictFP = false);
121 void replaceLibCallWithSimpleIntrinsic(IRBuilder<> &B, CallInst *CI,
122 Intrinsic::ID IntrID);
123
124 bool tryReplaceLibcallWithSimpleIntrinsic(IRBuilder<> &B, CallInst *CI,
125 Intrinsic::ID IntrID,
126 bool AllowMinSizeF32 = false,
127 bool AllowF64 = false,
128 bool AllowStrictFP = false);
129
130protected:
131 bool isUnsafeFiniteOnlyMath(const FPMathOperator *FPOp) const;
132
133 bool canIncreasePrecisionOfConstantFold(const FPMathOperator *FPOp) const;
134
135 static void replaceCall(Instruction *I, Value *With) {
136 I->replaceAllUsesWith(V: With);
137 I->eraseFromParent();
138 }
139
140 static void replaceCall(FPMathOperator *I, Value *With) {
141 replaceCall(I: cast<Instruction>(Val: I), With);
142 }
143
144public:
145 AMDGPULibCalls(Function &F, FunctionAnalysisManager &FAM);
146
147 bool fold(CallInst *CI);
148
149 void initNativeFuncs();
150
151 // Replace a normal math function call with that native version
152 bool useNative(CallInst *CI);
153};
154
155} // end namespace llvm
156
157template <typename IRB>
158static CallInst *CreateCallEx(IRB &B, FunctionCallee Callee, Value *Arg,
159 const Twine &Name = "") {
160 CallInst *R = B.CreateCall(Callee, Arg, Name);
161 if (Function *F = dyn_cast<Function>(Val: Callee.getCallee()))
162 R->setCallingConv(F->getCallingConv());
163 return R;
164}
165
166template <typename IRB>
167static CallInst *CreateCallEx2(IRB &B, FunctionCallee Callee, Value *Arg1,
168 Value *Arg2, const Twine &Name = "") {
169 CallInst *R = B.CreateCall(Callee, {Arg1, Arg2}, Name);
170 if (Function *F = dyn_cast<Function>(Val: Callee.getCallee()))
171 R->setCallingConv(F->getCallingConv());
172 return R;
173}
174
175static FunctionType *getPownType(FunctionType *FT) {
176 Type *PowNExpTy = Type::getInt32Ty(C&: FT->getContext());
177 if (VectorType *VecTy = dyn_cast<VectorType>(Val: FT->getReturnType()))
178 PowNExpTy = VectorType::get(ElementType: PowNExpTy, EC: VecTy->getElementCount());
179
180 return FunctionType::get(Result: FT->getReturnType(),
181 Params: {FT->getParamType(i: 0), PowNExpTy}, isVarArg: false);
182}
183
184// Data structures for table-driven optimizations.
185// FuncTbl works for both f32 and f64 functions with 1 input argument
186
187struct TableEntry {
188 double result;
189 double input;
190};
191
192/* a list of {result, input} */
193static const TableEntry tbl_acos[] = {
194 {MATH_PI / 2.0, .input: 0.0},
195 {MATH_PI / 2.0, .input: -0.0},
196 {.result: 0.0, .input: 1.0},
197 {MATH_PI, .input: -1.0}
198};
199static const TableEntry tbl_acosh[] = {
200 {.result: 0.0, .input: 1.0}
201};
202static const TableEntry tbl_acospi[] = {
203 {.result: 0.5, .input: 0.0},
204 {.result: 0.5, .input: -0.0},
205 {.result: 0.0, .input: 1.0},
206 {.result: 1.0, .input: -1.0}
207};
208static const TableEntry tbl_asin[] = {
209 {.result: 0.0, .input: 0.0},
210 {.result: -0.0, .input: -0.0},
211 {MATH_PI / 2.0, .input: 1.0},
212 {.result: -MATH_PI / 2.0, .input: -1.0}
213};
214static const TableEntry tbl_asinh[] = {
215 {.result: 0.0, .input: 0.0},
216 {.result: -0.0, .input: -0.0}
217};
218static const TableEntry tbl_asinpi[] = {
219 {.result: 0.0, .input: 0.0},
220 {.result: -0.0, .input: -0.0},
221 {.result: 0.5, .input: 1.0},
222 {.result: -0.5, .input: -1.0}
223};
224static const TableEntry tbl_atan[] = {
225 {.result: 0.0, .input: 0.0},
226 {.result: -0.0, .input: -0.0},
227 {MATH_PI / 4.0, .input: 1.0},
228 {.result: -MATH_PI / 4.0, .input: -1.0}
229};
230static const TableEntry tbl_atanh[] = {
231 {.result: 0.0, .input: 0.0},
232 {.result: -0.0, .input: -0.0}
233};
234static const TableEntry tbl_atanpi[] = {
235 {.result: 0.0, .input: 0.0},
236 {.result: -0.0, .input: -0.0},
237 {.result: 0.25, .input: 1.0},
238 {.result: -0.25, .input: -1.0}
239};
240static const TableEntry tbl_cbrt[] = {
241 {.result: 0.0, .input: 0.0},
242 {.result: -0.0, .input: -0.0},
243 {.result: 1.0, .input: 1.0},
244 {.result: -1.0, .input: -1.0},
245};
246static const TableEntry tbl_cos[] = {
247 {.result: 1.0, .input: 0.0},
248 {.result: 1.0, .input: -0.0}
249};
250static const TableEntry tbl_cosh[] = {
251 {.result: 1.0, .input: 0.0},
252 {.result: 1.0, .input: -0.0}
253};
254static const TableEntry tbl_cospi[] = {
255 {.result: 1.0, .input: 0.0},
256 {.result: 1.0, .input: -0.0}
257};
258static const TableEntry tbl_erfc[] = {
259 {.result: 1.0, .input: 0.0},
260 {.result: 1.0, .input: -0.0}
261};
262static const TableEntry tbl_erf[] = {
263 {.result: 0.0, .input: 0.0},
264 {.result: -0.0, .input: -0.0}
265};
266static const TableEntry tbl_exp[] = {
267 {.result: 1.0, .input: 0.0},
268 {.result: 1.0, .input: -0.0},
269 {MATH_E, .input: 1.0}
270};
271static const TableEntry tbl_exp2[] = {
272 {.result: 1.0, .input: 0.0},
273 {.result: 1.0, .input: -0.0},
274 {.result: 2.0, .input: 1.0}
275};
276static const TableEntry tbl_exp10[] = {
277 {.result: 1.0, .input: 0.0},
278 {.result: 1.0, .input: -0.0},
279 {.result: 10.0, .input: 1.0}
280};
281static const TableEntry tbl_expm1[] = {
282 {.result: 0.0, .input: 0.0},
283 {.result: -0.0, .input: -0.0}
284};
285static const TableEntry tbl_log[] = {
286 {.result: 0.0, .input: 1.0},
287 {.result: 1.0, MATH_E}
288};
289static const TableEntry tbl_log2[] = {
290 {.result: 0.0, .input: 1.0},
291 {.result: 1.0, .input: 2.0}
292};
293static const TableEntry tbl_log10[] = {
294 {.result: 0.0, .input: 1.0},
295 {.result: 1.0, .input: 10.0}
296};
297static const TableEntry tbl_rsqrt[] = {
298 {.result: 1.0, .input: 1.0},
299 {MATH_SQRT1_2, .input: 2.0}
300};
301static const TableEntry tbl_sin[] = {
302 {.result: 0.0, .input: 0.0},
303 {.result: -0.0, .input: -0.0}
304};
305static const TableEntry tbl_sinh[] = {
306 {.result: 0.0, .input: 0.0},
307 {.result: -0.0, .input: -0.0}
308};
309static const TableEntry tbl_sinpi[] = {
310 {.result: 0.0, .input: 0.0},
311 {.result: -0.0, .input: -0.0}
312};
313static const TableEntry tbl_sqrt[] = {
314 {.result: 0.0, .input: 0.0},
315 {.result: 1.0, .input: 1.0},
316 {MATH_SQRT2, .input: 2.0}
317};
318static const TableEntry tbl_tan[] = {
319 {.result: 0.0, .input: 0.0},
320 {.result: -0.0, .input: -0.0}
321};
322static const TableEntry tbl_tanh[] = {
323 {.result: 0.0, .input: 0.0},
324 {.result: -0.0, .input: -0.0}
325};
326static const TableEntry tbl_tanpi[] = {
327 {.result: 0.0, .input: 0.0},
328 {.result: -0.0, .input: -0.0}
329};
330static const TableEntry tbl_tgamma[] = {
331 {.result: 1.0, .input: 1.0},
332 {.result: 1.0, .input: 2.0},
333 {.result: 2.0, .input: 3.0},
334 {.result: 6.0, .input: 4.0}
335};
336
337static bool HasNative(AMDGPULibFunc::EFuncId id) {
338 switch(id) {
339 case AMDGPULibFunc::EI_DIVIDE:
340 case AMDGPULibFunc::EI_COS:
341 case AMDGPULibFunc::EI_EXP:
342 case AMDGPULibFunc::EI_EXP2:
343 case AMDGPULibFunc::EI_EXP10:
344 case AMDGPULibFunc::EI_LOG:
345 case AMDGPULibFunc::EI_LOG2:
346 case AMDGPULibFunc::EI_LOG10:
347 case AMDGPULibFunc::EI_POWR:
348 case AMDGPULibFunc::EI_RECIP:
349 case AMDGPULibFunc::EI_RSQRT:
350 case AMDGPULibFunc::EI_SIN:
351 case AMDGPULibFunc::EI_SINCOS:
352 case AMDGPULibFunc::EI_SQRT:
353 case AMDGPULibFunc::EI_TAN:
354 return true;
355 default:;
356 }
357 return false;
358}
359
360using TableRef = ArrayRef<TableEntry>;
361
362static TableRef getOptTable(AMDGPULibFunc::EFuncId id) {
363 switch(id) {
364 case AMDGPULibFunc::EI_ACOS: return TableRef(tbl_acos);
365 case AMDGPULibFunc::EI_ACOSH: return TableRef(tbl_acosh);
366 case AMDGPULibFunc::EI_ACOSPI: return TableRef(tbl_acospi);
367 case AMDGPULibFunc::EI_ASIN: return TableRef(tbl_asin);
368 case AMDGPULibFunc::EI_ASINH: return TableRef(tbl_asinh);
369 case AMDGPULibFunc::EI_ASINPI: return TableRef(tbl_asinpi);
370 case AMDGPULibFunc::EI_ATAN: return TableRef(tbl_atan);
371 case AMDGPULibFunc::EI_ATANH: return TableRef(tbl_atanh);
372 case AMDGPULibFunc::EI_ATANPI: return TableRef(tbl_atanpi);
373 case AMDGPULibFunc::EI_CBRT: return TableRef(tbl_cbrt);
374 case AMDGPULibFunc::EI_NCOS:
375 case AMDGPULibFunc::EI_COS: return TableRef(tbl_cos);
376 case AMDGPULibFunc::EI_COSH: return TableRef(tbl_cosh);
377 case AMDGPULibFunc::EI_COSPI: return TableRef(tbl_cospi);
378 case AMDGPULibFunc::EI_ERFC: return TableRef(tbl_erfc);
379 case AMDGPULibFunc::EI_ERF: return TableRef(tbl_erf);
380 case AMDGPULibFunc::EI_EXP: return TableRef(tbl_exp);
381 case AMDGPULibFunc::EI_NEXP2:
382 case AMDGPULibFunc::EI_EXP2: return TableRef(tbl_exp2);
383 case AMDGPULibFunc::EI_EXP10: return TableRef(tbl_exp10);
384 case AMDGPULibFunc::EI_EXPM1: return TableRef(tbl_expm1);
385 case AMDGPULibFunc::EI_LOG: return TableRef(tbl_log);
386 case AMDGPULibFunc::EI_NLOG2:
387 case AMDGPULibFunc::EI_LOG2: return TableRef(tbl_log2);
388 case AMDGPULibFunc::EI_LOG10: return TableRef(tbl_log10);
389 case AMDGPULibFunc::EI_NRSQRT:
390 case AMDGPULibFunc::EI_RSQRT: return TableRef(tbl_rsqrt);
391 case AMDGPULibFunc::EI_NSIN:
392 case AMDGPULibFunc::EI_SIN: return TableRef(tbl_sin);
393 case AMDGPULibFunc::EI_SINH: return TableRef(tbl_sinh);
394 case AMDGPULibFunc::EI_SINPI: return TableRef(tbl_sinpi);
395 case AMDGPULibFunc::EI_NSQRT:
396 case AMDGPULibFunc::EI_SQRT: return TableRef(tbl_sqrt);
397 case AMDGPULibFunc::EI_TAN: return TableRef(tbl_tan);
398 case AMDGPULibFunc::EI_TANH: return TableRef(tbl_tanh);
399 case AMDGPULibFunc::EI_TANPI: return TableRef(tbl_tanpi);
400 case AMDGPULibFunc::EI_TGAMMA: return TableRef(tbl_tgamma);
401 default:;
402 }
403 return TableRef();
404}
405
406static inline int getVecSize(const AMDGPULibFunc& FInfo) {
407 return FInfo.getLeads()[0].VectorSize;
408}
409
410static inline AMDGPULibFunc::EType getArgType(const AMDGPULibFunc& FInfo) {
411 return (AMDGPULibFunc::EType)FInfo.getLeads()[0].ArgType;
412}
413
414FunctionCallee AMDGPULibCalls::getFunction(Module *M, const FuncInfo &fInfo) {
415 // If we are doing PreLinkOpt, the function is external. So it is safe to
416 // use getOrInsertFunction() at this stage.
417
418 return CLOpts.prelink ? AMDGPULibFunc::getOrInsertFunction(M, fInfo)
419 : AMDGPULibFunc::getFunction(M, fInfo);
420}
421
422FunctionCallee AMDGPULibCalls::getFloatFastVariant(
423 Module *M, const FuncInfo &fInfo, FuncInfo &newInfo,
424 AMDGPULibFunc::EFuncId NewFunc, AMDGPULibFunc::EFuncId FastVariant) {
425 assert(NewFunc != FastVariant);
426
427 if (FastVariant != AMDGPULibFunc::EI_NONE &&
428 getArgType(FInfo: fInfo) == AMDGPULibFunc::F32) {
429 newInfo = AMDGPULibFunc(FastVariant, fInfo);
430 if (FunctionCallee NewCallee = getFunction(M, fInfo: newInfo))
431 return NewCallee;
432 }
433
434 newInfo = AMDGPULibFunc(NewFunc, fInfo);
435 return getFunction(M, fInfo: newInfo);
436}
437
438bool AMDGPULibCalls::parseFunctionName(const StringRef &FMangledName,
439 FuncInfo &FInfo) {
440 return AMDGPULibFunc::parse(MangledName: FMangledName, Ptr&: FInfo);
441}
442
443bool AMDGPULibCalls::isUnsafeFiniteOnlyMath(const FPMathOperator *FPOp) const {
444 return FPOp->hasApproxFunc() && FPOp->hasNoNaNs() && FPOp->hasNoInfs();
445}
446
447bool AMDGPULibCalls::canIncreasePrecisionOfConstantFold(
448 const FPMathOperator *FPOp) const {
449 // TODO: Refine to approxFunc or contract
450 return FPOp->isFast();
451}
452
453AMDGPULibCalls::AMDGPULibCalls(Function &F, FunctionAnalysisManager &FAM)
454 : CLOpts(F.getContext().getLibraryOptions().get<AMDGPUOptions>()),
455 SQ(F.getDataLayout(), &FAM.getResult<TargetLibraryAnalysis>(IR&: F),
456 FAM.getCachedResult<DominatorTreeAnalysis>(IR&: F),
457 &FAM.getResult<AssumptionAnalysis>(IR&: F)) {}
458
459bool AMDGPULibCalls::useNativeFunc(const StringRef F) const {
460 return AllNative || llvm::is_contained(Range&: UseNative, Element: F);
461}
462
463void AMDGPULibCalls::initNativeFuncs() {
464 AllNative = useNativeFunc(F: "all") ||
465 (UseNative.getNumOccurrences() && UseNative.size() == 1 &&
466 UseNative.begin()->empty());
467}
468
469bool AMDGPULibCalls::sincosUseNative(CallInst *aCI, const FuncInfo &FInfo) {
470 bool native_sin = useNativeFunc(F: "sin");
471 bool native_cos = useNativeFunc(F: "cos");
472
473 if (native_sin && native_cos) {
474 Module *M = aCI->getModule();
475 Value *opr0 = aCI->getArgOperand(i: 0);
476
477 AMDGPULibFunc nf;
478 nf.getLeads()[0].ArgType = FInfo.getLeads()[0].ArgType;
479 nf.getLeads()[0].VectorSize = FInfo.getLeads()[0].VectorSize;
480
481 nf.setPrefix(AMDGPULibFunc::NATIVE);
482 nf.setId(AMDGPULibFunc::EI_SIN);
483 FunctionCallee sinExpr = getFunction(M, fInfo: nf);
484
485 nf.setPrefix(AMDGPULibFunc::NATIVE);
486 nf.setId(AMDGPULibFunc::EI_COS);
487 FunctionCallee cosExpr = getFunction(M, fInfo: nf);
488 if (sinExpr && cosExpr) {
489 Value *sinval =
490 CallInst::Create(Func: sinExpr, Args: opr0, NameStr: "splitsin", InsertBefore: aCI->getIterator());
491 Value *cosval =
492 CallInst::Create(Func: cosExpr, Args: opr0, NameStr: "splitcos", InsertBefore: aCI->getIterator());
493 new StoreInst(cosval, aCI->getArgOperand(i: 1), aCI->getIterator());
494
495 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI
496 << " with native version of sin/cos");
497
498 replaceCall(I: aCI, With: sinval);
499 return true;
500 }
501 }
502 return false;
503}
504
505bool AMDGPULibCalls::useNative(CallInst *aCI) {
506 Function *Callee = aCI->getCalledFunction();
507 if (!Callee || aCI->isNoBuiltin())
508 return false;
509
510 FuncInfo FInfo;
511 if (!parseFunctionName(FMangledName: Callee->getName(), FInfo) || !FInfo.isMangled() ||
512 FInfo.getPrefix() != AMDGPULibFunc::NOPFX ||
513 getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(id: FInfo.getId()) ||
514 !(AllNative || useNativeFunc(F: FInfo.getName()))) {
515 return false;
516 }
517
518 if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS)
519 return sincosUseNative(aCI, FInfo);
520
521 FInfo.setPrefix(AMDGPULibFunc::NATIVE);
522 FunctionCallee F = getFunction(M: aCI->getModule(), fInfo: FInfo);
523 if (!F)
524 return false;
525
526 aCI->setCalledFunction(F);
527 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI
528 << " with native version");
529 return true;
530}
531
532// Clang emits call of __read_pipe_2 or __read_pipe_4 for OpenCL read_pipe
533// builtin, with appended type size and alignment arguments, where 2 or 4
534// indicates the original number of arguments. The library has optimized version
535// of __read_pipe_2/__read_pipe_4 when the type size and alignment has the same
536// power of 2 value. This function transforms __read_pipe_2 to __read_pipe_2_N
537// for such cases where N is the size in bytes of the type (N = 1, 2, 4, 8, ...,
538// 128). The same for __read_pipe_4, write_pipe_2, and write_pipe_4.
539bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,
540 const FuncInfo &FInfo) {
541 auto *Callee = CI->getCalledFunction();
542 if (!Callee->isDeclaration())
543 return false;
544
545 assert(Callee->hasName() && "Invalid read_pipe/write_pipe function");
546 auto *M = Callee->getParent();
547 std::string Name = std::string(Callee->getName());
548 auto NumArg = CI->arg_size();
549 if (NumArg != 4 && NumArg != 6)
550 return false;
551 ConstantInt *PacketSize =
552 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: NumArg - 2));
553 ConstantInt *PacketAlign =
554 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: NumArg - 1));
555 if (!PacketSize || !PacketAlign)
556 return false;
557
558 uint64_t Size = PacketSize->getZExtValue();
559 Align Alignment = PacketAlign->getAlignValue();
560 if (Alignment != Size)
561 return false;
562
563 unsigned PtrArgLoc = CI->arg_size() - 3;
564 Value *PtrArg = CI->getArgOperand(i: PtrArgLoc);
565 Type *PtrTy = PtrArg->getType();
566
567 SmallVector<llvm::Type *, 6> ArgTys;
568 for (unsigned I = 0; I != PtrArgLoc; ++I)
569 ArgTys.push_back(Elt: CI->getArgOperand(i: I)->getType());
570 ArgTys.push_back(Elt: PtrTy);
571
572 Name = Name + "_" + std::to_string(val: Size);
573 auto *FTy = FunctionType::get(Result: Callee->getReturnType(),
574 Params: ArrayRef<Type *>(ArgTys), isVarArg: false);
575 AMDGPULibFunc NewLibFunc(Name, FTy);
576 FunctionCallee F = AMDGPULibFunc::getOrInsertFunction(M, fInfo: NewLibFunc);
577 if (!F)
578 return false;
579
580 SmallVector<Value *, 6> Args;
581 for (unsigned I = 0; I != PtrArgLoc; ++I)
582 Args.push_back(Elt: CI->getArgOperand(i: I));
583 Args.push_back(Elt: PtrArg);
584
585 auto *NCI = B.CreateCall(Callee: F, Args);
586 NCI->setAttributes(CI->getAttributes());
587 CI->replaceAllUsesWith(V: NCI);
588 CI->dropAllReferences();
589 CI->eraseFromParent();
590
591 return true;
592}
593
594// This function returns false if no change; return true otherwise.
595bool AMDGPULibCalls::fold(CallInst *CI) {
596 Function *Callee = CI->getCalledFunction();
597 // Ignore indirect calls.
598 if (!Callee || Callee->isIntrinsic() || CI->isNoBuiltin())
599 return false;
600
601 FuncInfo FInfo;
602 if (!parseFunctionName(FMangledName: Callee->getName(), FInfo))
603 return false;
604
605 // Further check the number of arguments to see if they match.
606 // TODO: Check calling convention matches too
607 if (!FInfo.isCompatibleSignature(M: *Callee->getParent(), FuncTy: CI->getFunctionType()))
608 return false;
609
610 LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << '\n');
611
612 if (TDOFold(CI, FInfo))
613 return true;
614
615 IRBuilder<> B(CI);
616 if (CI->isStrictFP())
617 B.setIsFPConstrained(true);
618
619 if (FPMathOperator *FPOp = dyn_cast<FPMathOperator>(Val: CI)) {
620 // Under unsafe-math, evaluate calls if possible.
621 // According to Brian Sumner, we can do this for all f32 function calls
622 // using host's double function calls.
623 if (canIncreasePrecisionOfConstantFold(FPOp) && evaluateCall(aCI: CI, FInfo))
624 return true;
625
626 // Copy fast flags from the original call.
627 FastMathFlags FMF = FPOp->getFastMathFlags();
628 B.setFastMathFlags(FMF);
629
630 // Specialized optimizations for each function call.
631 //
632 // TODO: Handle native functions
633 switch (FInfo.getId()) {
634 case AMDGPULibFunc::EI_EXP:
635 if (FMF.none())
636 return false;
637 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::exp,
638 AllowMinSizeF32: FMF.approxFunc());
639 case AMDGPULibFunc::EI_EXP2:
640 if (FMF.none())
641 return false;
642 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::exp2,
643 AllowMinSizeF32: FMF.approxFunc());
644 case AMDGPULibFunc::EI_LOG:
645 if (FMF.none())
646 return false;
647 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::log,
648 AllowMinSizeF32: FMF.approxFunc());
649 case AMDGPULibFunc::EI_LOG2:
650 if (FMF.none())
651 return false;
652 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::log2,
653 AllowMinSizeF32: FMF.approxFunc());
654 case AMDGPULibFunc::EI_LOG10:
655 if (FMF.none())
656 return false;
657 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::log10,
658 AllowMinSizeF32: FMF.approxFunc());
659 case AMDGPULibFunc::EI_FMIN:
660 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::minnum,
661 AllowMinSizeF32: true, AllowF64: true);
662 case AMDGPULibFunc::EI_FMAX:
663 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::maxnum,
664 AllowMinSizeF32: true, AllowF64: true);
665 case AMDGPULibFunc::EI_FMA:
666 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::fma, AllowMinSizeF32: true,
667 AllowF64: true);
668 case AMDGPULibFunc::EI_MAD:
669 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::fmuladd,
670 AllowMinSizeF32: true, AllowF64: true);
671 case AMDGPULibFunc::EI_FABS:
672 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::fabs, AllowMinSizeF32: true,
673 AllowF64: true, AllowStrictFP: true);
674 case AMDGPULibFunc::EI_COPYSIGN:
675 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::copysign,
676 AllowMinSizeF32: true, AllowF64: true, AllowStrictFP: true);
677 case AMDGPULibFunc::EI_FLOOR:
678 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::floor, AllowMinSizeF32: true,
679 AllowF64: true);
680 case AMDGPULibFunc::EI_CEIL:
681 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::ceil, AllowMinSizeF32: true,
682 AllowF64: true);
683 case AMDGPULibFunc::EI_TRUNC:
684 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::trunc, AllowMinSizeF32: true,
685 AllowF64: true);
686 case AMDGPULibFunc::EI_RINT:
687 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::rint, AllowMinSizeF32: true,
688 AllowF64: true);
689 case AMDGPULibFunc::EI_ROUND:
690 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::round, AllowMinSizeF32: true,
691 AllowF64: true);
692 case AMDGPULibFunc::EI_LDEXP: {
693 if (!shouldReplaceLibcallWithIntrinsic(CI, AllowMinSizeF32: true, AllowF64: true))
694 return false;
695
696 Value *Arg1 = CI->getArgOperand(i: 1);
697 if (VectorType *VecTy = dyn_cast<VectorType>(Val: CI->getType());
698 VecTy && !isa<VectorType>(Val: Arg1->getType())) {
699 Value *SplatArg1 = B.CreateVectorSplat(EC: VecTy->getElementCount(), V: Arg1);
700 CI->setArgOperand(i: 1, v: SplatArg1);
701 }
702
703 CI->setCalledFunction(Intrinsic::getOrInsertDeclaration(
704 M: CI->getModule(), id: Intrinsic::ldexp,
705 OverloadTys: {CI->getType(), CI->getArgOperand(i: 1)->getType()}));
706 CI->setCallingConv(CallingConv::C);
707 return true;
708 }
709 case AMDGPULibFunc::EI_POW:
710 case AMDGPULibFunc::EI_POW_FAST:
711 return tryOptimizePow(FPOp, B, FInfo);
712 case AMDGPULibFunc::EI_POWR:
713 case AMDGPULibFunc::EI_POWR_FAST: {
714 if (fold_pow(FPOp, B, FInfo))
715 return true;
716 if (!FMF.approxFunc())
717 return false;
718
719 if (FInfo.getId() == AMDGPULibFunc::EI_POWR && FMF.approxFunc() &&
720 getArgType(FInfo) == AMDGPULibFunc::F32) {
721 Module *M = Callee->getParent();
722 AMDGPULibFunc PowrFastInfo(AMDGPULibFunc::EI_POWR_FAST, FInfo);
723 if (FunctionCallee PowrFastFunc = getFunction(M, fInfo: PowrFastInfo)) {
724 CI->setCalledFunction(PowrFastFunc);
725 return true;
726 }
727 }
728
729 if (!shouldReplaceLibcallWithIntrinsic(CI))
730 return false;
731 return expandFastPow(FPOp, B, Kind: PowKind::PowR);
732 }
733 case AMDGPULibFunc::EI_POWN:
734 case AMDGPULibFunc::EI_POWN_FAST: {
735 if (fold_pow(FPOp, B, FInfo))
736 return true;
737 if (!FMF.approxFunc())
738 return false;
739
740 if (FInfo.getId() == AMDGPULibFunc::EI_POWN &&
741 getArgType(FInfo) == AMDGPULibFunc::F32) {
742 Module *M = Callee->getParent();
743 AMDGPULibFunc PownFastInfo(AMDGPULibFunc::EI_POWN_FAST, FInfo);
744 if (FunctionCallee PownFastFunc = getFunction(M, fInfo: PownFastInfo)) {
745 CI->setCalledFunction(PownFastFunc);
746 return true;
747 }
748 }
749
750 if (!shouldReplaceLibcallWithIntrinsic(CI))
751 return false;
752 return expandFastPow(FPOp, B, Kind: PowKind::PowN);
753 }
754 case AMDGPULibFunc::EI_ROOTN:
755 case AMDGPULibFunc::EI_ROOTN_FAST: {
756 if (fold_rootn(FPOp, B, FInfo))
757 return true;
758 if (!FMF.approxFunc())
759 return false;
760
761 if (getArgType(FInfo) == AMDGPULibFunc::F32) {
762 Module *M = Callee->getParent();
763 AMDGPULibFunc RootnFastInfo(AMDGPULibFunc::EI_ROOTN_FAST, FInfo);
764 if (FunctionCallee RootnFastFunc = getFunction(M, fInfo: RootnFastInfo)) {
765 CI->setCalledFunction(RootnFastFunc);
766 return true;
767 }
768 }
769
770 return expandFastPow(FPOp, B, Kind: PowKind::RootN);
771 }
772 case AMDGPULibFunc::EI_SQRT:
773 // TODO: Allow with strictfp + constrained intrinsic
774 return tryReplaceLibcallWithSimpleIntrinsic(
775 B, CI, IntrID: Intrinsic::sqrt, AllowMinSizeF32: true, AllowF64: true, /*AllowStrictFP=*/false);
776 case AMDGPULibFunc::EI_COS:
777 case AMDGPULibFunc::EI_SIN:
778 return fold_sincos(FPOp, B, FInfo);
779 default:
780 break;
781 }
782 } else {
783 // Specialized optimizations for each function call
784 switch (FInfo.getId()) {
785 case AMDGPULibFunc::EI_READ_PIPE_2:
786 case AMDGPULibFunc::EI_READ_PIPE_4:
787 case AMDGPULibFunc::EI_WRITE_PIPE_2:
788 case AMDGPULibFunc::EI_WRITE_PIPE_4:
789 return fold_read_write_pipe(CI, B, FInfo);
790 default:
791 break;
792 }
793 }
794
795 return false;
796}
797
798static Constant *getConstantFloat(const ArrayRef<APFloat> Values,
799 const Type *Ty) {
800
801 assert(Ty->isSingleValueType() &&
802 "Type must either be a scalar or a vector.");
803 assert((!Ty->isVectorTy() || Ty->isScalableTy() ||
804 Values.size() == cast<FixedVectorType>(Ty)->getNumElements()) &&
805 "Unexpected number of constant values.");
806 assert((Ty->isVectorTy() || Values.size() == 1) &&
807 "Expected exactly one constant value");
808
809 Type *ElemTy = Ty->getScalarType();
810 const fltSemantics &FltSem = ElemTy->getFltSemantics();
811
812 SmallVector<Constant *, 4> ConstValues;
813 ConstValues.reserve(N: Values.size());
814 for (APFloat APF : Values) {
815 bool Unused;
816 APF.convert(ToSemantics: FltSem, RM: APFloat::rmNearestTiesToEven, losesInfo: &Unused);
817 ConstValues.push_back(Elt: ConstantFP::get(Ty: ElemTy, V: APF));
818 }
819
820 return Ty->isVectorTy() ? ConstantVector::get(V: ConstValues) : ConstValues[0];
821}
822
823bool AMDGPULibCalls::TDOFold(CallInst *CI, const FuncInfo &FInfo) {
824 // Table-Driven optimization
825 const TableRef tr = getOptTable(id: FInfo.getId());
826 if (tr.empty())
827 return false;
828
829 int const sz = (int)tr.size();
830 Value *opr0 = CI->getArgOperand(i: 0);
831
832 int vecSize = getVecSize(FInfo);
833 if (vecSize > 1) {
834 // Vector version
835 Constant *CV = dyn_cast<Constant>(Val: opr0);
836 if (CV && CV->getType()->isVectorTy()) {
837 SmallVector<APFloat, 4> Values;
838 Values.reserve(N: vecSize);
839 for (int eltNo = 0; eltNo < vecSize; ++eltNo) {
840 // A lane may be undef or poison, in which case there is nothing to
841 // look up in the table.
842 ConstantFP *eltval = dyn_cast_or_null<ConstantFP>(
843 Val: CV->getAggregateElement(Elt: (unsigned)eltNo));
844 if (!eltval)
845 return false;
846 auto MatchingRow = llvm::find_if(Range: tr, P: [eltval](const TableEntry &entry) {
847 return eltval->isExactlyValue(V: entry.input);
848 });
849 if (MatchingRow == tr.end())
850 return false;
851 Values.push_back(Elt: APFloat(MatchingRow->result));
852 }
853 Constant *NewValues = getConstantFloat(Values, Ty: CI->getType());
854 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *NewValues << "\n");
855 replaceCall(I: CI, With: NewValues);
856 return true;
857 }
858 } else {
859 // Scalar version
860 if (ConstantFP *CF = dyn_cast<ConstantFP>(Val: opr0)) {
861 for (int i = 0; i < sz; ++i) {
862 if (CF->isExactlyValue(V: tr[i].input)) {
863 Value *nval = ConstantFP::get(Ty: CF->getType(), V: tr[i].result);
864 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");
865 replaceCall(I: CI, With: nval);
866 return true;
867 }
868 }
869 }
870 }
871
872 return false;
873}
874
875namespace llvm {
876static double log2(double V) {
877#if _XOPEN_SOURCE >= 600 || defined(_ISOC99_SOURCE) || _POSIX_C_SOURCE >= 200112L
878 return ::log2(x: V);
879#else
880 return log(V) / numbers::ln2;
881#endif
882}
883} // namespace llvm
884
885bool AMDGPULibCalls::fold_pow(FPMathOperator *FPOp, IRBuilder<> &B,
886 const FuncInfo &FInfo) {
887 assert((FInfo.getId() == AMDGPULibFunc::EI_POW ||
888 FInfo.getId() == AMDGPULibFunc::EI_POW_FAST ||
889 FInfo.getId() == AMDGPULibFunc::EI_POWR ||
890 FInfo.getId() == AMDGPULibFunc::EI_POWR_FAST ||
891 FInfo.getId() == AMDGPULibFunc::EI_POWN ||
892 FInfo.getId() == AMDGPULibFunc::EI_POWN_FAST) &&
893 "fold_pow: encounter a wrong function call");
894
895 Module *M = B.getModule();
896 Type *eltType = FPOp->getType()->getScalarType();
897 Value *opr0 = FPOp->getOperand(i: 0);
898 Value *opr1 = FPOp->getOperand(i: 1);
899
900 const APFloat *CF = nullptr;
901 const APInt *CINT = nullptr;
902 if (!match(V: opr1, P: m_APFloatAllowPoison(Res&: CF)))
903 match(V: opr1, P: m_APIntAllowPoison(Res&: CINT));
904
905 // 0x1111111 means that we don't do anything for this call.
906 int ci_opr1 = (CINT ? (int)CINT->getSExtValue() : 0x1111111);
907
908 // OpenCL powr(x<0, y) = NaN, but the folds below would turn it into a
909 // finite number. Skip them unless NaNs are ignored or the base is known
910 // non-negative.
911 bool IsPowr = FInfo.getId() == AMDGPULibFunc::EI_POWR ||
912 FInfo.getId() == AMDGPULibFunc::EI_POWR_FAST;
913 bool SkipConstantFolds =
914 IsPowr && !FPOp->hasNoNaNs() &&
915 !cannotBeOrderedLessThanZero(
916 V: opr0, SQ: SQ.getWithInstruction(I: cast<Instruction>(Val: FPOp)));
917
918 if (CF && (CF->isExactlyValue(V: 0.5) || CF->isExactlyValue(V: -0.5))) {
919 // pow[r](x, [-]0.5) = sqrt(x) / rsqrt(x)
920 //
921 // sqrt/rsqrt and pow disagree on two negative inputs:
922 // pow(-Inf, 0.5) == +Inf but sqrt(-Inf) == NaN (ninf case)
923 // pow(-0.0, 0.5) == +0.0 but sqrt(-0.0) == -0.0 (nsz case)
924 // powr requires x >= 0 by the OpenCL spec, so -Inf is undefined behaviour
925 // and the ninf check can be skipped for powr/powr_fast. -0.0 is a valid
926 // input for powr since -0.0 >= 0 by IEEE comparison, so nsz is still
927 // required for all variants. sqrt/rsqrt already return NaN for a
928 // negative base like powr does, so this fold skips the base-sign check.
929 if (FPOp->hasNoSignedZeros() && (IsPowr || FPOp->hasNoInfs())) {
930 bool issqrt = CF->isExactlyValue(V: 0.5);
931 if (FunctionCallee FPExpr =
932 getFunction(M, fInfo: AMDGPULibFunc(issqrt ? AMDGPULibFunc::EI_SQRT
933 : AMDGPULibFunc::EI_RSQRT,
934 FInfo))) {
935 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << FInfo.getName()
936 << '(' << *opr0 << ")\n");
937 Value *nval = CreateCallEx(B, Callee: FPExpr, Arg: opr0,
938 Name: issqrt ? "__pow2sqrt" : "__pow2rsqrt");
939 replaceCall(I: FPOp, With: nval);
940 return true;
941 }
942 }
943 }
944
945 if (!SkipConstantFolds) {
946 if ((CF && CF->isZero()) || (CINT && ci_opr1 == 0)) {
947 // pow/powr/pown(x, 0) == 1
948 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1\n");
949 Constant *cnval = ConstantFP::get(Ty: eltType, V: 1.0);
950 if (getVecSize(FInfo) > 1) {
951 cnval = ConstantDataVector::getSplat(NumElts: getVecSize(FInfo), Elt: cnval);
952 }
953 replaceCall(I: FPOp, With: cnval);
954 return true;
955 }
956 if ((CF && CF->isOne()) || (CINT && ci_opr1 == 1)) {
957 // pow/powr/pown(x, 1.0) = x
958 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << "\n");
959 replaceCall(I: FPOp, With: opr0);
960 return true;
961 }
962 if ((CF && CF->isExactlyValue(V: 2.0)) || (CINT && ci_opr1 == 2)) {
963 // pow/powr/pown(x, 2.0) = x*x
964 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << " * "
965 << *opr0 << "\n");
966 Value *nval = B.CreateFMul(L: opr0, R: opr0, Name: "__pow2");
967 replaceCall(I: FPOp, With: nval);
968 return true;
969 }
970 if ((CF && CF->isMinusOne()) || (CINT && ci_opr1 == -1)) {
971 // pow/powr/pown(x, -1.0) = 1.0/x
972 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1 / " << *opr0 << "\n");
973 Constant *cnval = ConstantFP::get(Ty: eltType, V: 1.0);
974 if (getVecSize(FInfo) > 1) {
975 cnval = ConstantDataVector::getSplat(NumElts: getVecSize(FInfo), Elt: cnval);
976 }
977 Value *nval = B.CreateFDiv(L: cnval, R: opr0, Name: "__powrecip");
978 replaceCall(I: FPOp, With: nval);
979 return true;
980 }
981 }
982
983 if (!isUnsafeFiniteOnlyMath(FPOp))
984 return false;
985
986 // Unsafe Math optimization
987
988 // Remember that ci_opr1 is set if opr1 is integral
989 if (CF) {
990 double dval = (getArgType(FInfo) == AMDGPULibFunc::F32)
991 ? (double)CF->convertToFloat()
992 : CF->convertToDouble();
993 int ival = (int)dval;
994 if ((double)ival == dval) {
995 ci_opr1 = ival;
996 } else
997 ci_opr1 = 0x11111111;
998 }
999
1000 // pow/powr/pown(x, c) = [1/](x*x*..x); where
1001 // trunc(c) == c && the number of x == c && |c| <= 12
1002 unsigned abs_opr1 = (ci_opr1 < 0) ? -ci_opr1 : ci_opr1;
1003 if (abs_opr1 <= 12) {
1004 Constant *cnval;
1005 Value *nval;
1006 if (abs_opr1 == 0) {
1007 cnval = ConstantFP::get(Ty: eltType, V: 1.0);
1008 if (getVecSize(FInfo) > 1) {
1009 cnval = ConstantDataVector::getSplat(NumElts: getVecSize(FInfo), Elt: cnval);
1010 }
1011 nval = cnval;
1012 } else {
1013 Value *valx2 = nullptr;
1014 nval = nullptr;
1015 while (abs_opr1 > 0) {
1016 valx2 = valx2 ? B.CreateFMul(L: valx2, R: valx2, Name: "__powx2") : opr0;
1017 if (abs_opr1 & 1) {
1018 nval = nval ? B.CreateFMul(L: nval, R: valx2, Name: "__powprod") : valx2;
1019 }
1020 abs_opr1 >>= 1;
1021 }
1022 }
1023
1024 if (ci_opr1 < 0) {
1025 cnval = ConstantFP::get(Ty: eltType, V: 1.0);
1026 if (getVecSize(FInfo) > 1) {
1027 cnval = ConstantDataVector::getSplat(NumElts: getVecSize(FInfo), Elt: cnval);
1028 }
1029 nval = B.CreateFDiv(L: cnval, R: nval, Name: "__1powprod");
1030 }
1031 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> "
1032 << ((ci_opr1 < 0) ? "1/prod(" : "prod(") << *opr0
1033 << ")\n");
1034 replaceCall(I: FPOp, With: nval);
1035 return true;
1036 }
1037
1038 // If we should use the generic intrinsic instead of emitting a libcall
1039 const bool ShouldUseIntrinsic = eltType->isFloatTy() || eltType->isHalfTy();
1040
1041 // powr ---> exp2(y * log2(x))
1042 // pown/pow ---> powr(fabs(x), y) | (x & ((int)y << 31))
1043 FunctionCallee ExpExpr;
1044 if (ShouldUseIntrinsic)
1045 ExpExpr = Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::exp2,
1046 OverloadTys: {FPOp->getType()});
1047 else {
1048 ExpExpr = getFunction(M, fInfo: AMDGPULibFunc(AMDGPULibFunc::EI_EXP2, FInfo));
1049 if (!ExpExpr)
1050 return false;
1051 }
1052
1053 bool needlog = false;
1054 bool needabs = false;
1055 bool needcopysign = false;
1056 Constant *cnval = nullptr;
1057 if (getVecSize(FInfo) == 1) {
1058 CF = nullptr;
1059 match(V: opr0, P: m_APFloatAllowPoison(Res&: CF));
1060
1061 if (CF) {
1062 double V = (getArgType(FInfo) == AMDGPULibFunc::F32)
1063 ? (double)CF->convertToFloat()
1064 : CF->convertToDouble();
1065
1066 V = log2(V: std::abs(x: V));
1067 cnval = ConstantFP::get(Ty: eltType, V);
1068 needcopysign = (FInfo.getId() != AMDGPULibFunc::EI_POWR &&
1069 FInfo.getId() != AMDGPULibFunc::EI_POWR_FAST) &&
1070 CF->isNegative();
1071 } else {
1072 needlog = true;
1073 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR &&
1074 FInfo.getId() != AMDGPULibFunc::EI_POWR_FAST;
1075 }
1076 } else {
1077 ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(Val: opr0);
1078
1079 if (!CDV) {
1080 needlog = true;
1081 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR &&
1082 FInfo.getId() != AMDGPULibFunc::EI_POWR_FAST;
1083 } else {
1084 assert ((int)CDV->getNumElements() == getVecSize(FInfo) &&
1085 "Wrong vector size detected");
1086
1087 SmallVector<double, 0> DVal;
1088 for (int i=0; i < getVecSize(FInfo); ++i) {
1089 double V = CDV->getElementAsAPFloat(i).convertToDouble();
1090 if (V < 0.0) needcopysign = true;
1091 V = log2(V: std::abs(x: V));
1092 DVal.push_back(Elt: V);
1093 }
1094 if (getArgType(FInfo) == AMDGPULibFunc::F32) {
1095 SmallVector<float, 0> FVal;
1096 for (double D : DVal)
1097 FVal.push_back(Elt: (float)D);
1098 ArrayRef<float> tmp(FVal);
1099 cnval = ConstantDataVector::get(Context&: M->getContext(), Elts: tmp);
1100 } else {
1101 ArrayRef<double> tmp(DVal);
1102 cnval = ConstantDataVector::get(Context&: M->getContext(), Elts: tmp);
1103 }
1104 }
1105 }
1106
1107 if (needcopysign && (FInfo.getId() == AMDGPULibFunc::EI_POW ||
1108 FInfo.getId() == AMDGPULibFunc::EI_POW_FAST)) {
1109 // We cannot handle corner cases for a general pow() function, give up
1110 // unless y is a constant integral value. Then proceed as if it were pown.
1111 if (!isKnownIntegral(V: opr1, SQ: SQ.getWithInstruction(I: cast<Instruction>(Val: FPOp)),
1112 FMF: FPOp->getFastMathFlags()))
1113 return false;
1114 }
1115
1116 Value *nval;
1117 if (needabs) {
1118 nval = B.CreateFAbs(V: opr0, FMFSource: nullptr, Name: "__fabs");
1119 } else {
1120 nval = cnval ? cnval : opr0;
1121 }
1122 if (needlog) {
1123 FunctionCallee LogExpr;
1124 if (ShouldUseIntrinsic) {
1125 LogExpr = Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::log2,
1126 OverloadTys: {FPOp->getType()});
1127 } else {
1128 LogExpr = getFunction(M, fInfo: AMDGPULibFunc(AMDGPULibFunc::EI_LOG2, FInfo));
1129 if (!LogExpr)
1130 return false;
1131 }
1132
1133 nval = CreateCallEx(B,Callee: LogExpr, Arg: nval, Name: "__log2");
1134 }
1135
1136 if (FInfo.getId() == AMDGPULibFunc::EI_POWN ||
1137 FInfo.getId() == AMDGPULibFunc::EI_POWN_FAST) {
1138 // convert int(32) to fp(f32 or f64)
1139 opr1 = B.CreateSIToFP(V: opr1, DestTy: nval->getType(), Name: "pownI2F");
1140 }
1141 nval = B.CreateFMul(L: opr1, R: nval, Name: "__ylogx");
1142
1143 CallInst *Exp2Call = CreateCallEx(B, Callee: ExpExpr, Arg: nval, Name: "__exp2");
1144
1145 // TODO: Generalized fpclass logic for pow
1146 FPClassTest KnownNot = FPClassTest::fcNegative;
1147 if (FPOp->hasNoNaNs())
1148 KnownNot |= FPClassTest::fcNan;
1149
1150 Exp2Call->addRetAttr(
1151 Attr: Attribute::getWithNoFPClass(Context&: Exp2Call->getContext(), Mask: KnownNot));
1152 nval = Exp2Call;
1153
1154 if (needcopysign) {
1155 Type *nTyS =
1156 B.getIntNTy(N: static_cast<unsigned>(eltType->getPrimitiveSizeInBits()));
1157 Type *nTy = FPOp->getType()->getWithNewType(EltTy: nTyS);
1158 Value *opr_n = FPOp->getOperand(i: 1);
1159 if (opr_n->getType()->getScalarType()->isIntegerTy())
1160 opr_n = B.CreateZExtOrTrunc(V: opr_n, DestTy: nTy, Name: "__ytou");
1161 else
1162 opr_n = B.CreateFPToSI(V: opr1, DestTy: nTy, Name: "__ytou");
1163
1164 unsigned size = nTy->getScalarSizeInBits();
1165 Value *sign = B.CreateShl(LHS: opr_n, RHS: size-1, Name: "__yeven");
1166 sign = B.CreateAnd(LHS: B.CreateBitCast(V: opr0, DestTy: nTy), RHS: sign, Name: "__pow_sign");
1167
1168 nval = B.CreateCopySign(LHS: nval, RHS: B.CreateBitCast(V: sign, DestTy: nval->getType()),
1169 FMFSource: nullptr, Name: "__pow_sign");
1170 }
1171
1172 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> "
1173 << "exp2(" << *opr1 << " * log2(" << *opr0 << "))\n");
1174 replaceCall(I: FPOp, With: nval);
1175
1176 return true;
1177}
1178
1179bool AMDGPULibCalls::fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B,
1180 const FuncInfo &FInfo) {
1181 Value *opr0 = FPOp->getOperand(i: 0);
1182 Value *opr1 = FPOp->getOperand(i: 1);
1183
1184 const APInt *CINT = nullptr;
1185 if (!match(V: opr1, P: m_APIntAllowPoison(Res&: CINT)))
1186 return false;
1187
1188 Function *Parent = B.GetInsertBlock()->getParent();
1189
1190 int ci_opr1 = (int)CINT->getSExtValue();
1191 if (ci_opr1 == 1 && !Parent->hasFnAttribute(Kind: Attribute::StrictFP)) {
1192 // rootn(x, 1) = x
1193 //
1194 // TODO: Insert constrained canonicalize for strictfp case.
1195 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << '\n');
1196 replaceCall(I: FPOp, With: opr0);
1197 return true;
1198 }
1199
1200 Module *M = B.getModule();
1201
1202 CallInst *CI = cast<CallInst>(Val: FPOp);
1203
1204 // rootn and sqrt disagree on signed-zero / -Inf inputs (e.g. rootn(-0.0, 2)
1205 // is +0.0, sqrt(-0.0) is -0.0), so require nsz/ninf.
1206 bool FMFOkForSqrt = FPOp->hasNoSignedZeros() && FPOp->hasNoInfs();
1207
1208 if (ci_opr1 == 2 && FMFOkForSqrt &&
1209 shouldReplaceLibcallWithIntrinsic(CI,
1210 /*AllowMinSizeF32=*/true,
1211 /*AllowF64=*/true)) {
1212 // rootn(x, 2) = sqrt(x)
1213 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> sqrt(" << *opr0 << ")\n");
1214
1215 Value *NewCall = B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, Op: opr0, FMFSource: CI);
1216 NewCall->takeName(V: CI);
1217
1218 // OpenCL rootn has a looser ulp of 2 requirement than sqrt, so add some
1219 // metadata.
1220 MDBuilder MDHelper(M->getContext());
1221 MDNode *FPMD = MDHelper.createFPMath(Accuracy: std::max(a: FPOp->getFPAccuracy(), b: 2.0f));
1222 if (auto *NewCallI = dyn_cast<Instruction>(Val: NewCall))
1223 NewCallI->setMetadata(KindID: LLVMContext::MD_fpmath, Node: FPMD);
1224
1225 replaceCall(I: CI, With: NewCall);
1226 return true;
1227 }
1228
1229 if (ci_opr1 == 3) { // rootn(x, 3) = cbrt(x)
1230 if (FunctionCallee FPExpr =
1231 getFunction(M, fInfo: AMDGPULibFunc(AMDGPULibFunc::EI_CBRT, FInfo))) {
1232 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> cbrt(" << *opr0
1233 << ")\n");
1234 Value *nval = CreateCallEx(B,Callee: FPExpr, Arg: opr0, Name: "__rootn2cbrt");
1235 replaceCall(I: FPOp, With: nval);
1236 return true;
1237 }
1238 } else if (ci_opr1 == -1) { // rootn(x, -1) = 1.0/x
1239 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1.0 / " << *opr0 << "\n");
1240 Value *nval = B.CreateFDiv(L: ConstantFP::get(Ty: opr0->getType(), V: 1.0),
1241 R: opr0,
1242 Name: "__rootn2div");
1243 replaceCall(I: FPOp, With: nval);
1244 return true;
1245 }
1246
1247 if (ci_opr1 == -2 && FMFOkForSqrt &&
1248 shouldReplaceLibcallWithIntrinsic(CI,
1249 /*AllowMinSizeF32=*/true,
1250 /*AllowF64=*/true)) {
1251 // rootn(x, -2) = rsqrt(x)
1252
1253 // The original rootn had looser ulp requirements than the resultant sqrt
1254 // and fdiv.
1255 MDBuilder MDHelper(M->getContext());
1256 MDNode *FPMD = MDHelper.createFPMath(Accuracy: std::max(a: FPOp->getFPAccuracy(), b: 2.0f));
1257
1258 // TODO: Could handle strictfp but need to fix strict sqrt emission
1259 FastMathFlags FMF = FPOp->getFastMathFlags();
1260 FMF.setAllowContract(true);
1261
1262 Value *Sqrt = B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, Op: opr0, FMFSource: CI);
1263 Instruction *RSqrt = cast<Instruction>(
1264 Val: B.CreateFDiv(L: ConstantFP::get(Ty: opr0->getType(), V: 1.0), R: Sqrt));
1265 if (auto *SqrtI = dyn_cast<Instruction>(Val: Sqrt))
1266 SqrtI->setFastMathFlags(FMF);
1267 RSqrt->setFastMathFlags(FMF);
1268 RSqrt->setMetadata(KindID: LLVMContext::MD_fpmath, Node: FPMD);
1269
1270 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> rsqrt(" << *opr0
1271 << ")\n");
1272 replaceCall(I: CI, With: RSqrt);
1273 return true;
1274 }
1275
1276 return false;
1277}
1278
1279// is_integer(y) => trunc(y) == y
1280static Value *emitIsInteger(IRBuilder<> &B, Value *Y) {
1281 Value *TruncY = B.CreateUnaryIntrinsic(ID: Intrinsic::trunc, Op: Y);
1282 return B.CreateFCmpOEQ(LHS: TruncY, RHS: Y);
1283}
1284
1285static Value *emitIsEvenInteger(IRBuilder<> &B, Value *Y) {
1286 // Even integers are still integers after division by 2.
1287 auto *HalfY = B.CreateFMul(L: Y, R: ConstantFP::get(Ty: Y->getType(), V: 0.5));
1288 return emitIsInteger(B, Y: HalfY);
1289}
1290
1291// is_odd_integer(y) => is_integer(y) && !is_even_integer(y)
1292static Value *emitIsOddInteger(IRBuilder<> &B, Value *Y) {
1293 Value *IsIntY = emitIsInteger(B, Y);
1294 Value *IsEvenY = emitIsEvenInteger(B, Y);
1295 Value *NotEvenY = B.CreateNot(V: IsEvenY);
1296 return B.CreateAnd(LHS: IsIntY, RHS: NotEvenY);
1297}
1298
1299// isinf(val) => fabs(val) == +inf
1300static Value *emitIsInf(IRBuilder<> &B, Value *val) {
1301 auto *fabsVal = B.CreateFAbs(V: val);
1302 return B.CreateFCmpOEQ(LHS: fabsVal, RHS: ConstantFP::getInfinity(Ty: val->getType()));
1303}
1304
1305// y * log2(fabs(x))
1306static Value *emitFastExpYLnx(IRBuilder<> &B, Value *X, Value *Y) {
1307 Value *AbsX = B.CreateFAbs(V: X);
1308 Value *LogAbsX = B.CreateUnaryIntrinsic(ID: Intrinsic::log2, Op: AbsX);
1309 Value *YTimesLogX = B.CreateFMul(L: Y, R: LogAbsX);
1310 return B.CreateUnaryIntrinsic(ID: Intrinsic::exp2, Op: YTimesLogX);
1311}
1312
1313/// Emit special case management epilog code for fast pow, powr, pown, and rootn
1314/// expansions. \p x and \p y should be the arguments to the library call
1315/// (possibly with some values clamped). \p expylnx should be the result to use
1316/// in normal circumstances.
1317static Value *emitPowFixup(IRBuilder<> &B, Value *X, Value *Y, Value *ExpYLnX,
1318 PowKind Kind) {
1319 Constant *Zero = ConstantFP::getZero(Ty: X->getType());
1320 Constant *One = ConstantFP::get(Ty: X->getType(), V: 1.0);
1321 Constant *QNaN = ConstantFP::getQNaN(Ty: X->getType());
1322 Constant *PInf = ConstantFP::getInfinity(Ty: X->getType());
1323
1324 switch (Kind) {
1325 case PowKind::Pow: {
1326 // is_odd_integer(y)
1327 Value *IsOddY = emitIsOddInteger(B, Y);
1328
1329 // ret = copysign(expylnx, is_odd_y ? x : 1.0f)
1330 Value *SelSign = B.CreateSelect(C: IsOddY, True: X, False: One);
1331 Value *Ret = B.CreateCopySign(LHS: ExpYLnX, RHS: SelSign);
1332
1333 // if (x < 0 && !is_integer(y)) ret = QNAN
1334 Value *IsIntY = emitIsInteger(B, Y);
1335 Value *condNegX = B.CreateFCmpOLT(LHS: X, RHS: Zero);
1336 Value *condNotIntY = B.CreateNot(V: IsIntY);
1337 Value *condNaN = B.CreateAnd(LHS: condNegX, RHS: condNotIntY);
1338 Ret = B.CreateSelect(C: condNaN, True: QNaN, False: Ret);
1339
1340 // if (isinf(ay)) { ... }
1341
1342 // FIXME: Missing backend optimization to save on materialization cost of
1343 // mixed sign constant infinities.
1344 Value *YIsInf = emitIsInf(B, val: Y);
1345
1346 Value *AY = B.CreateFAbs(V: Y);
1347 Value *YIsNegInf = B.CreateFCmpUNE(LHS: Y, RHS: AY);
1348
1349 Value *AX = B.CreateFAbs(V: X);
1350 Value *AxEqOne = B.CreateFCmpOEQ(LHS: AX, RHS: One);
1351 Value *AxLtOne = B.CreateFCmpOLT(LHS: AX, RHS: One);
1352 Value *XorCond = B.CreateXor(LHS: AxLtOne, RHS: YIsNegInf);
1353 Value *SelInf =
1354 B.CreateSelect(C: AxEqOne, True: AX, False: B.CreateSelect(C: XorCond, True: Zero, False: AY));
1355 Ret = B.CreateSelect(C: YIsInf, True: SelInf, False: Ret);
1356
1357 // if (isinf(ax) || x == 0.0f) { ... }
1358 Value *XIsInf = emitIsInf(B, val: X);
1359 Value *XEqZero = B.CreateFCmpOEQ(LHS: X, RHS: Zero);
1360 Value *AxInfOrZero = B.CreateOr(LHS: XIsInf, RHS: XEqZero);
1361 Value *YLtZero = B.CreateFCmpOLT(LHS: Y, RHS: Zero);
1362 Value *XorZeroInf = B.CreateXor(LHS: XEqZero, RHS: YLtZero);
1363 Value *SelVal = B.CreateSelect(C: XorZeroInf, True: Zero, False: PInf);
1364 Value *SelSign2 = B.CreateSelect(C: IsOddY, True: X, False: Zero);
1365 Value *Copysign = B.CreateCopySign(LHS: SelVal, RHS: SelSign2);
1366 Ret = B.CreateSelect(C: AxInfOrZero, True: Copysign, False: Ret);
1367
1368 // if (isunordered(x, y)) ret = QNAN
1369 Value *isUnordered = B.CreateFCmpUNO(LHS: X, RHS: Y);
1370 return B.CreateSelect(C: isUnordered, True: QNaN, False: Ret);
1371 }
1372 case PowKind::PowR: {
1373 Value *YIsNeg = B.CreateFCmpOLT(LHS: Y, RHS: Zero);
1374 Value *IZ = B.CreateSelect(C: YIsNeg, True: PInf, False: Zero);
1375 Value *ZI = B.CreateSelect(C: YIsNeg, True: Zero, False: PInf);
1376
1377 Value *YEqZero = B.CreateFCmpOEQ(LHS: Y, RHS: Zero);
1378 Value *SelZeroCase = B.CreateSelect(C: YEqZero, True: QNaN, False: IZ);
1379 Value *XEqZero = B.CreateFCmpOEQ(LHS: X, RHS: Zero);
1380 Value *Ret = B.CreateSelect(C: XEqZero, True: SelZeroCase, False: ExpYLnX);
1381
1382 Value *XEqInf = B.CreateFCmpOEQ(LHS: X, RHS: PInf);
1383 Value *YNeZero = B.CreateFCmpUNE(LHS: Y, RHS: Zero);
1384 Value *CondInfCase = B.CreateAnd(LHS: XEqInf, RHS: YNeZero);
1385 Ret = B.CreateSelect(C: CondInfCase, True: ZI, False: Ret);
1386
1387 Value *IsInfY = emitIsInf(B, val: Y);
1388 Value *XNeOne = B.CreateFCmpUNE(LHS: X, RHS: One);
1389 Value *CondInfY = B.CreateAnd(LHS: IsInfY, RHS: XNeOne);
1390 Value *XLtOne = B.CreateFCmpOLT(LHS: X, RHS: One);
1391 Value *SelInfYCase = B.CreateSelect(C: XLtOne, True: IZ, False: ZI);
1392 Ret = B.CreateSelect(C: CondInfY, True: SelInfYCase, False: Ret);
1393
1394 Value *IsUnordered = B.CreateFCmpUNO(LHS: X, RHS: Y);
1395 return B.CreateSelect(C: IsUnordered, True: QNaN, False: Ret);
1396 }
1397 case PowKind::PowN: {
1398 Constant *ZeroI = ConstantInt::get(Ty: Y->getType(), V: 0);
1399
1400 // is_odd_y = (ny & 1) != 0
1401 Value *OneI = ConstantInt::get(Ty: Y->getType(), V: 1);
1402 Value *YAnd1 = B.CreateAnd(LHS: Y, RHS: OneI);
1403 Value *IsOddY = B.CreateICmpNE(LHS: YAnd1, RHS: ZeroI);
1404
1405 // ret = copysign(expylnx, is_odd_y ? x : 1.0f)
1406 Value *SelSign = B.CreateSelect(C: IsOddY, True: X, False: One);
1407 Value *Ret = B.CreateCopySign(LHS: ExpYLnX, RHS: SelSign);
1408
1409 // if (isinf(x) || x == 0.0f)
1410 Value *FabsX = B.CreateFAbs(V: X);
1411 Value *XIsInf = B.CreateFCmpOEQ(LHS: FabsX, RHS: PInf);
1412 Value *XEqZero = B.CreateFCmpOEQ(LHS: X, RHS: Zero);
1413 Value *InfOrZero = B.CreateOr(LHS: XIsInf, RHS: XEqZero);
1414
1415 // (x == 0.0f) ^ (ny < 0) ? 0.0f : +inf
1416 Value *YLtZero = B.CreateICmpSLT(LHS: Y, RHS: ZeroI);
1417 Value *XorZeroInf = B.CreateXor(LHS: XEqZero, RHS: YLtZero);
1418 Value *SelVal = B.CreateSelect(C: XorZeroInf, True: Zero, False: PInf);
1419
1420 // copysign(selVal, is_odd_y ? x : 0.0f)
1421 Value *SelSign2 = B.CreateSelect(C: IsOddY, True: X, False: Zero);
1422 Value *Copysign = B.CreateCopySign(LHS: SelVal, RHS: SelSign2);
1423
1424 return B.CreateSelect(C: InfOrZero, True: Copysign, False: Ret);
1425 }
1426 case PowKind::RootN: {
1427 Constant *ZeroI = ConstantInt::get(Ty: Y->getType(), V: 0);
1428
1429 // is_odd_y = (ny & 1) != 0
1430 Value *YAnd1 = B.CreateAnd(LHS: Y, RHS: ConstantInt::get(Ty: Y->getType(), V: 1));
1431 Value *IsOddY = B.CreateICmpNE(LHS: YAnd1, RHS: ZeroI);
1432
1433 // ret = copysign(expylnx, is_odd_y ? x : 1.0f)
1434 Value *SelSign = B.CreateSelect(C: IsOddY, True: X, False: One);
1435 Value *Ret = B.CreateCopySign(LHS: ExpYLnX, RHS: SelSign);
1436
1437 // if (isinf(x) || x == 0.0f)
1438 Value *FabsX = B.CreateFAbs(V: X);
1439 Value *IsInfX = B.CreateFCmpOEQ(LHS: FabsX, RHS: PInf);
1440 Value *XEqZero = B.CreateFCmpOEQ(LHS: X, RHS: Zero);
1441 Value *CondInfOrZero = B.CreateOr(LHS: IsInfX, RHS: XEqZero);
1442
1443 // (x == 0.0f) ^ (ny < 0) ? 0.0f : +inf
1444 Value *YLtZero = B.CreateICmpSLT(LHS: Y, RHS: ZeroI);
1445 Value *XorZeroInf = B.CreateXor(LHS: XEqZero, RHS: YLtZero);
1446 Value *SelVal = B.CreateSelect(C: XorZeroInf, True: Zero, False: PInf);
1447
1448 // copysign(selVal, is_odd_y ? x : 0.0f)
1449 Value *SelSign2 = B.CreateSelect(C: IsOddY, True: X, False: Zero);
1450 Value *Copysign = B.CreateCopySign(LHS: SelVal, RHS: SelSign2);
1451
1452 Ret = B.CreateSelect(C: CondInfOrZero, True: Copysign, False: Ret);
1453
1454 // if ((x < 0.0f && !is_odd_y) || ny == 0) ret = QNAN
1455 Value *XIsNeg = B.CreateFCmpOLT(LHS: X, RHS: Zero);
1456 Value *NotOddY = B.CreateNot(V: IsOddY);
1457 Value *CondNegAndNotOdd = B.CreateAnd(LHS: XIsNeg, RHS: NotOddY);
1458 Value *YEqZero = B.CreateICmpEQ(LHS: Y, RHS: ZeroI);
1459 Value *CondBad = B.CreateOr(LHS: CondNegAndNotOdd, RHS: YEqZero);
1460 return B.CreateSelect(C: CondBad, True: QNaN, False: Ret);
1461 }
1462 }
1463
1464 llvm_unreachable("covered switch");
1465}
1466
1467// TODO: Move the fold_pow folding to sqrt/fdiv here
1468bool AMDGPULibCalls::expandFastPow(FPMathOperator *FPOp, IRBuilder<> &B,
1469 PowKind Kind) {
1470 Type *Ty = FPOp->getType();
1471
1472 // There's currently no reason to do this for half. The correct path is
1473 // promote to float and use the fast float expansion.
1474 //
1475 // TODO: We could move this expansion to lowering to get half pow to work.
1476 if (!Ty->getScalarType()->isFloatTy())
1477 return false;
1478
1479 // TODO: Verify optimization for double and bfloat.
1480 Value *X = FPOp->getOperand(i: 0);
1481 Value *Y = FPOp->getOperand(i: 1);
1482
1483 switch (Kind) {
1484 case PowKind::Pow: {
1485 Constant *One = ConstantFP::get(Ty: X->getType(), V: 1.0);
1486
1487 // if (x == 1.0f) y = 1.0f;
1488 Value *XEqOne = B.CreateFCmpOEQ(LHS: X, RHS: One);
1489 Y = B.CreateSelect(C: XEqOne, True: One, False: Y);
1490
1491 // if (y == 0.0f) x = 1.0f;
1492 Value *YEqZero = B.CreateFCmpOEQ(LHS: Y, RHS: ConstantFP::getZero(Ty: X->getType()));
1493 X = B.CreateSelect(C: YEqZero, True: One, False: X);
1494
1495 Value *ExpYLnX = emitFastExpYLnx(B, X, Y);
1496 Value *Fixed = emitPowFixup(B, X, Y, ExpYLnX, Kind);
1497 replaceCall(I: FPOp, With: Fixed);
1498 return true;
1499 }
1500 case PowKind::PowR: {
1501 Value *NegX = B.CreateFCmpOLT(LHS: X, RHS: ConstantFP::getZero(Ty: X->getType()));
1502 X = B.CreateSelect(C: NegX, True: ConstantFP::getQNaN(Ty: X->getType()), False: X);
1503
1504 Value *ExpYLnX = emitFastExpYLnx(B, X, Y);
1505 Value *Fixed = emitPowFixup(B, X, Y, ExpYLnX, Kind);
1506 replaceCall(I: FPOp, With: Fixed);
1507 return true;
1508 }
1509 case PowKind::PowN: {
1510 // ny == 0
1511 Value *YEqZero = B.CreateICmpEQ(LHS: Y, RHS: ConstantInt::get(Ty: Y->getType(), V: 0));
1512
1513 // x = (ny == 0 ? 1.0f : x)
1514 X = B.CreateSelect(C: YEqZero, True: ConstantFP::get(Ty: X->getType(), V: 1.0), False: X);
1515
1516 Value *CastY = B.CreateSIToFP(V: Y, DestTy: X->getType());
1517 Value *ExpYLnX = emitFastExpYLnx(B, X, Y: CastY);
1518 Value *Fixed = emitPowFixup(B, X, Y, ExpYLnX, Kind);
1519 replaceCall(I: FPOp, With: Fixed);
1520 return true;
1521 }
1522 case PowKind::RootN: {
1523 Value *CastY = B.CreateSIToFP(V: Y, DestTy: X->getType());
1524
1525 // This is afn anyway, so we will turn into rcp.
1526 Value *RcpY = B.CreateFDiv(L: ConstantFP::get(Ty: X->getType(), V: 1.0), R: CastY);
1527
1528 Value *ExpYLnX = emitFastExpYLnx(B, X, Y: RcpY);
1529 Value *Fixed = emitPowFixup(B, X, Y, ExpYLnX, Kind);
1530 replaceCall(I: FPOp, With: Fixed);
1531 return true;
1532 }
1533 }
1534 llvm_unreachable("Unhandled PowKind enum");
1535}
1536
1537bool AMDGPULibCalls::tryOptimizePow(FPMathOperator *FPOp, IRBuilder<> &B,
1538 const FuncInfo &FInfo) {
1539 FastMathFlags FMF = FPOp->getFastMathFlags();
1540 CallInst *Call = cast<CallInst>(Val: FPOp);
1541 Module *M = Call->getModule();
1542
1543 FuncInfo PowrInfo;
1544 AMDGPULibFunc::EFuncId FastPowrFuncId =
1545 FMF.approxFunc() || FInfo.getId() == AMDGPULibFunc::EI_POW_FAST
1546 ? AMDGPULibFunc::EI_POWR_FAST
1547 : AMDGPULibFunc::EI_NONE;
1548 FunctionCallee PowrFunc = getFloatFastVariant(
1549 M, fInfo: FInfo, newInfo&: PowrInfo, NewFunc: AMDGPULibFunc::EI_POWR, FastVariant: FastPowrFuncId);
1550
1551 // TODO: Prefer fast pown to fast powr, but slow powr to slow pown.
1552
1553 // pow(x, y) -> powr(x, y) for x >= -0.0
1554 // TODO: Account for flags on current call
1555 if (PowrFunc && cannotBeOrderedLessThanZero(V: FPOp->getOperand(i: 0),
1556 SQ: SQ.getWithInstruction(I: Call))) {
1557 Call->setCalledFunction(PowrFunc);
1558 return fold_pow(FPOp, B, FInfo: PowrInfo) || true;
1559 }
1560
1561 // pow(x, y) -> pown(x, y) for known integral y
1562 if (isKnownIntegral(V: FPOp->getOperand(i: 1), SQ: SQ.getWithInstruction(I: Call),
1563 FMF: FPOp->getFastMathFlags())) {
1564 FunctionType *PownType = getPownType(FT: Call->getFunctionType());
1565
1566 FuncInfo PownInfo;
1567 AMDGPULibFunc::EFuncId FastPownFuncId =
1568 FMF.approxFunc() || FInfo.getId() == AMDGPULibFunc::EI_POW_FAST
1569 ? AMDGPULibFunc::EI_POWN_FAST
1570 : AMDGPULibFunc::EI_NONE;
1571 FunctionCallee PownFunc = getFloatFastVariant(
1572 M, fInfo: FInfo, newInfo&: PownInfo, NewFunc: AMDGPULibFunc::EI_POWN, FastVariant: FastPownFuncId);
1573
1574 if (PownFunc) {
1575 // TODO: If the incoming integral value is an sitofp/uitofp, it won't
1576 // fold out without a known range. We can probably take the source
1577 // value directly.
1578 Value *CastedArg =
1579 B.CreateFPToSI(V: FPOp->getOperand(i: 1), DestTy: PownType->getParamType(i: 1));
1580 // Have to drop any nofpclass attributes on the original call site.
1581 Call->removeParamAttrs(
1582 ArgNo: 1, AttrsToRemove: AttributeFuncs::typeIncompatible(Ty: CastedArg->getType(),
1583 AS: Call->getParamAttributes(ArgNo: 1)));
1584 Call->setCalledFunction(PownFunc);
1585 Call->setArgOperand(i: 1, v: CastedArg);
1586 return fold_pow(FPOp, B, FInfo: PownInfo) || true;
1587 }
1588 }
1589
1590 if (fold_pow(FPOp, B, FInfo))
1591 return true;
1592
1593 if (!FMF.approxFunc())
1594 return false;
1595
1596 if (FInfo.getId() == AMDGPULibFunc::EI_POW && FMF.approxFunc() &&
1597 getArgType(FInfo) == AMDGPULibFunc::F32) {
1598 AMDGPULibFunc PowFastInfo(AMDGPULibFunc::EI_POW_FAST, FInfo);
1599 if (FunctionCallee PowFastFunc = getFunction(M, fInfo: PowFastInfo)) {
1600 Call->setCalledFunction(PowFastFunc);
1601 return fold_pow(FPOp, B, FInfo: PowFastInfo) || true;
1602 }
1603 }
1604
1605 return expandFastPow(FPOp, B, Kind: PowKind::Pow);
1606}
1607
1608// Some library calls are just wrappers around llvm intrinsics, but compiled
1609// conservatively. Preserve the flags from the original call site by
1610// substituting them with direct calls with all the flags.
1611bool AMDGPULibCalls::shouldReplaceLibcallWithIntrinsic(const CallInst *CI,
1612 bool AllowMinSizeF32,
1613 bool AllowF64,
1614 bool AllowStrictFP) {
1615 Type *FltTy = CI->getType()->getScalarType();
1616 const bool IsF32 = FltTy->isFloatTy();
1617
1618 // f64 intrinsics aren't implemented for most operations.
1619 if (!IsF32 && !FltTy->isHalfTy() && (!AllowF64 || !FltTy->isDoubleTy()))
1620 return false;
1621
1622 // We're implicitly inlining by replacing the libcall with the intrinsic, so
1623 // don't do it for noinline call sites.
1624 if (CI->isNoInline())
1625 return false;
1626
1627 const Function *ParentF = CI->getFunction();
1628 // TODO: Handle strictfp
1629 if (!AllowStrictFP && ParentF->hasFnAttribute(Kind: Attribute::StrictFP))
1630 return false;
1631
1632 if (IsF32 && !AllowMinSizeF32 && ParentF->hasMinSize())
1633 return false;
1634 return true;
1635}
1636
1637void AMDGPULibCalls::replaceLibCallWithSimpleIntrinsic(IRBuilder<> &B,
1638 CallInst *CI,
1639 Intrinsic::ID IntrID) {
1640 if (CI->arg_size() == 2) {
1641 Value *Arg0 = CI->getArgOperand(i: 0);
1642 Value *Arg1 = CI->getArgOperand(i: 1);
1643 VectorType *Arg0VecTy = dyn_cast<VectorType>(Val: Arg0->getType());
1644 VectorType *Arg1VecTy = dyn_cast<VectorType>(Val: Arg1->getType());
1645 if (Arg0VecTy && !Arg1VecTy) {
1646 Value *SplatRHS = B.CreateVectorSplat(EC: Arg0VecTy->getElementCount(), V: Arg1);
1647 CI->setArgOperand(i: 1, v: SplatRHS);
1648 } else if (!Arg0VecTy && Arg1VecTy) {
1649 Value *SplatLHS = B.CreateVectorSplat(EC: Arg1VecTy->getElementCount(), V: Arg0);
1650 CI->setArgOperand(i: 0, v: SplatLHS);
1651 }
1652 }
1653
1654 CI->setCalledFunction(Intrinsic::getOrInsertDeclaration(
1655 M: CI->getModule(), id: IntrID, OverloadTys: {CI->getType()}));
1656 CI->setCallingConv(CallingConv::C);
1657}
1658
1659bool AMDGPULibCalls::tryReplaceLibcallWithSimpleIntrinsic(
1660 IRBuilder<> &B, CallInst *CI, Intrinsic::ID IntrID, bool AllowMinSizeF32,
1661 bool AllowF64, bool AllowStrictFP) {
1662 if (!shouldReplaceLibcallWithIntrinsic(CI, AllowMinSizeF32, AllowF64,
1663 AllowStrictFP))
1664 return false;
1665 replaceLibCallWithSimpleIntrinsic(B, CI, IntrID);
1666 return true;
1667}
1668
1669std::tuple<Value *, Value *, Value *>
1670AMDGPULibCalls::insertSinCos(Value *Arg, FastMathFlags FMF, IRBuilder<> &B,
1671 FunctionCallee Fsincos) {
1672 DebugLoc DL = B.getCurrentDebugLocation();
1673 Function *F = B.GetInsertBlock()->getParent();
1674 B.SetInsertPointPastAllocas(F);
1675
1676 AllocaInst *Alloc = B.CreateAlloca(Ty: Arg->getType(), ArraySize: nullptr, Name: "__sincos_");
1677
1678 if (Instruction *ArgInst = dyn_cast<Instruction>(Val: Arg)) {
1679 // If the argument is an instruction, it must dominate all uses so put our
1680 // sincos call there. Otherwise, right after the allocas works well enough
1681 // if it's an argument or constant.
1682
1683 B.SetInsertPoint(*ArgInst->getInsertionPointAfterDef());
1684
1685 // SetInsertPoint unwelcomely always tries to set the debug loc.
1686 B.SetCurrentDebugLocation(DL);
1687 }
1688
1689 Type *CosPtrTy = Fsincos.getFunctionType()->getParamType(i: 1);
1690
1691 // The allocaInst allocates the memory in private address space. This need
1692 // to be addrspacecasted to point to the address space of cos pointer type.
1693 // In OpenCL 2.0 this is generic, while in 1.2 that is private.
1694 Value *CastAlloc = B.CreateAddrSpaceCast(V: Alloc, DestTy: CosPtrTy);
1695
1696 CallInst *SinCos = CreateCallEx2(B, Callee: Fsincos, Arg1: Arg, Arg2: CastAlloc);
1697
1698 // TODO: Is it worth trying to preserve the location for the cos calls for the
1699 // load?
1700
1701 LoadInst *LoadCos = B.CreateLoad(Ty: Arg->getType(), Ptr: Alloc);
1702 return {SinCos, LoadCos, SinCos};
1703}
1704
1705// fold sin, cos -> sincos.
1706bool AMDGPULibCalls::fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B,
1707 const FuncInfo &fInfo) {
1708 assert(fInfo.getId() == AMDGPULibFunc::EI_SIN ||
1709 fInfo.getId() == AMDGPULibFunc::EI_COS);
1710
1711 if ((getArgType(FInfo: fInfo) != AMDGPULibFunc::F32 &&
1712 getArgType(FInfo: fInfo) != AMDGPULibFunc::F64) ||
1713 fInfo.getPrefix() != AMDGPULibFunc::NOPFX)
1714 return false;
1715
1716 bool const isSin = fInfo.getId() == AMDGPULibFunc::EI_SIN;
1717
1718 Value *CArgVal = FPOp->getOperand(i: 0);
1719
1720 // TODO: Constant fold the call
1721 if (isa<ConstantData>(Val: CArgVal))
1722 return false;
1723
1724 CallInst *CI = cast<CallInst>(Val: FPOp);
1725
1726 Function *F = B.GetInsertBlock()->getParent();
1727 Module *M = F->getParent();
1728
1729 // Merge the sin and cos. For OpenCL 2.0, there may only be a generic pointer
1730 // implementation. Prefer the private form if available.
1731 AMDGPULibFunc SinCosLibFuncPrivate(AMDGPULibFunc::EI_SINCOS, fInfo);
1732 SinCosLibFuncPrivate.getLeads()[0].PtrKind =
1733 AMDGPULibFunc::getEPtrKindFromAddrSpace(AS: AMDGPUAS::PRIVATE_ADDRESS);
1734
1735 AMDGPULibFunc SinCosLibFuncGeneric(AMDGPULibFunc::EI_SINCOS, fInfo);
1736 SinCosLibFuncGeneric.getLeads()[0].PtrKind =
1737 AMDGPULibFunc::getEPtrKindFromAddrSpace(AS: AMDGPUAS::FLAT_ADDRESS);
1738
1739 FunctionCallee FSinCosPrivate = getFunction(M, fInfo: SinCosLibFuncPrivate);
1740 FunctionCallee FSinCosGeneric = getFunction(M, fInfo: SinCosLibFuncGeneric);
1741 FunctionCallee FSinCos = FSinCosPrivate ? FSinCosPrivate : FSinCosGeneric;
1742 if (!FSinCos)
1743 return false;
1744
1745 SmallVector<CallInst *> SinCalls;
1746 SmallVector<CallInst *> CosCalls;
1747 SmallVector<CallInst *> SinCosCalls;
1748 FuncInfo PartnerInfo(isSin ? AMDGPULibFunc::EI_COS : AMDGPULibFunc::EI_SIN,
1749 fInfo);
1750 const std::string PairName = PartnerInfo.mangle(CLOpts);
1751
1752 StringRef SinName = isSin ? CI->getCalledFunction()->getName() : PairName;
1753 StringRef CosName = isSin ? PairName : CI->getCalledFunction()->getName();
1754 const std::string SinCosPrivateName = SinCosLibFuncPrivate.mangle(CLOpts);
1755 const std::string SinCosGenericName = SinCosLibFuncGeneric.mangle(CLOpts);
1756
1757 // Intersect the two sets of flags.
1758 FastMathFlags FMF = FPOp->getFastMathFlags();
1759 MDNode *FPMath = CI->getMetadata(KindID: LLVMContext::MD_fpmath);
1760
1761 SmallVector<DILocation *> MergeDbgLocs = {CI->getDebugLoc()};
1762
1763 for (User* U : CArgVal->users()) {
1764 CallInst *XI = dyn_cast<CallInst>(Val: U);
1765 if (!XI || XI->getFunction() != F || XI->isNoBuiltin())
1766 continue;
1767
1768 Function *UCallee = XI->getCalledFunction();
1769 if (!UCallee)
1770 continue;
1771
1772 bool Handled = true;
1773
1774 if (UCallee->getName() == SinName)
1775 SinCalls.push_back(Elt: XI);
1776 else if (UCallee->getName() == CosName)
1777 CosCalls.push_back(Elt: XI);
1778 else if (UCallee->getName() == SinCosPrivateName ||
1779 UCallee->getName() == SinCosGenericName)
1780 SinCosCalls.push_back(Elt: XI);
1781 else
1782 Handled = false;
1783
1784 if (Handled) {
1785 MergeDbgLocs.push_back(Elt: XI->getDebugLoc());
1786 auto *OtherOp = cast<FPMathOperator>(Val: XI);
1787 FMF &= OtherOp->getFastMathFlags();
1788 FPMath = MDNode::getMostGenericFPMath(
1789 A: FPMath, B: XI->getMetadata(KindID: LLVMContext::MD_fpmath));
1790 }
1791 }
1792
1793 if (SinCalls.empty() || CosCalls.empty())
1794 return false;
1795
1796 // insertSinCos needs an insertion point after the argument's def.
1797 if (auto *ArgInst = dyn_cast<Instruction>(Val: CArgVal);
1798 ArgInst && !ArgInst->getInsertionPointAfterDef())
1799 return false;
1800
1801 B.setFastMathFlags(FMF);
1802 B.setDefaultFPMathTag(FPMath);
1803 DILocation *DbgLoc = DILocation::getMergedLocations(Locs: MergeDbgLocs);
1804 B.SetCurrentDebugLocation(DbgLoc);
1805
1806 auto [Sin, Cos, SinCos] = insertSinCos(Arg: CArgVal, FMF, B, Fsincos: FSinCos);
1807
1808 auto replaceTrigInsts = [](ArrayRef<CallInst *> Calls, Value *Res) {
1809 for (CallInst *C : Calls)
1810 C->replaceAllUsesWith(V: Res);
1811
1812 // Leave the other dead instructions to avoid clobbering iterators.
1813 };
1814
1815 replaceTrigInsts(SinCalls, Sin);
1816 replaceTrigInsts(CosCalls, Cos);
1817 replaceTrigInsts(SinCosCalls, SinCos);
1818
1819 // It's safe to delete the original now.
1820 CI->eraseFromParent();
1821 return true;
1822}
1823
1824bool AMDGPULibCalls::evaluateScalarMathFunc(const FuncInfo &FInfo,
1825 APFloat &Res0, APFloat &Res1,
1826 Constant *copr0, Constant *copr1) {
1827 // Every function handled below reads its first operand as a floating-point
1828 // value. Refuse anything else, e.g. a poison vector lane: silently treating
1829 // it as 0.0 misfolds the whole call.
1830 ConstantFP *fpopr0 = dyn_cast_or_null<ConstantFP>(Val: copr0);
1831 if (!fpopr0)
1832 return false;
1833
1834 double opr0 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1835 ? fpopr0->getValueAPF().convertToDouble()
1836 : (double)fpopr0->getValueAPF().convertToFloat();
1837
1838 switch (FInfo.getId()) {
1839 default:
1840 return false;
1841
1842 case AMDGPULibFunc::EI_ACOS:
1843 Res0 = APFloat{acos(x: opr0)};
1844 return true;
1845
1846 case AMDGPULibFunc::EI_ACOSH:
1847 // acosh(x) == log(x + sqrt(x*x - 1))
1848 Res0 = APFloat{log(x: opr0 + sqrt(x: opr0 * opr0 - 1.0))};
1849 return true;
1850
1851 case AMDGPULibFunc::EI_ACOSPI:
1852 Res0 = APFloat{acos(x: opr0) / MATH_PI};
1853 return true;
1854
1855 case AMDGPULibFunc::EI_ASIN:
1856 Res0 = APFloat{asin(x: opr0)};
1857 return true;
1858
1859 case AMDGPULibFunc::EI_ASINH:
1860 // asinh(x) == log(x + sqrt(x*x + 1))
1861 Res0 = APFloat{log(x: opr0 + sqrt(x: opr0 * opr0 + 1.0))};
1862 return true;
1863
1864 case AMDGPULibFunc::EI_ASINPI:
1865 Res0 = APFloat{asin(x: opr0) / MATH_PI};
1866 return true;
1867
1868 case AMDGPULibFunc::EI_ATAN:
1869 Res0 = APFloat{atan(x: opr0)};
1870 return true;
1871
1872 case AMDGPULibFunc::EI_ATANH:
1873 // atanh(x) == (log(x+1) - log(x-1))/2;
1874 Res0 = APFloat{(log(x: opr0 + 1.0) - log(x: opr0 - 1.0)) / 2.0};
1875 return true;
1876
1877 case AMDGPULibFunc::EI_ATANPI:
1878 Res0 = APFloat{atan(x: opr0) / MATH_PI};
1879 return true;
1880
1881 case AMDGPULibFunc::EI_CBRT:
1882 Res0 =
1883 APFloat{(opr0 < 0.0) ? -pow(x: -opr0, y: 1.0 / 3.0) : pow(x: opr0, y: 1.0 / 3.0)};
1884 return true;
1885
1886 case AMDGPULibFunc::EI_COS:
1887 Res0 = APFloat{cos(x: opr0)};
1888 return true;
1889
1890 case AMDGPULibFunc::EI_COSH:
1891 Res0 = APFloat{cosh(x: opr0)};
1892 return true;
1893
1894 case AMDGPULibFunc::EI_COSPI:
1895 Res0 = APFloat{cos(MATH_PI * opr0)};
1896 return true;
1897
1898 case AMDGPULibFunc::EI_EXP:
1899 Res0 = APFloat{std::exp(x: opr0)};
1900 return true;
1901
1902 case AMDGPULibFunc::EI_EXP2:
1903 Res0 = APFloat{pow(x: 2.0, y: opr0)};
1904 return true;
1905
1906 case AMDGPULibFunc::EI_EXP10:
1907 Res0 = APFloat{pow(x: 10.0, y: opr0)};
1908 return true;
1909
1910 case AMDGPULibFunc::EI_LOG:
1911 Res0 = APFloat{log(x: opr0)};
1912 return true;
1913
1914 case AMDGPULibFunc::EI_LOG2:
1915 Res0 = APFloat{log(x: opr0) / log(x: 2.0)};
1916 return true;
1917
1918 case AMDGPULibFunc::EI_LOG10:
1919 Res0 = APFloat{log(x: opr0) / log(x: 10.0)};
1920 return true;
1921
1922 case AMDGPULibFunc::EI_RSQRT:
1923 Res0 = APFloat{1.0 / sqrt(x: opr0)};
1924 return true;
1925
1926 case AMDGPULibFunc::EI_SIN:
1927 Res0 = APFloat{sin(x: opr0)};
1928 return true;
1929
1930 case AMDGPULibFunc::EI_SINH:
1931 Res0 = APFloat{sinh(x: opr0)};
1932 return true;
1933
1934 case AMDGPULibFunc::EI_SINPI:
1935 Res0 = APFloat{sin(MATH_PI * opr0)};
1936 return true;
1937
1938 case AMDGPULibFunc::EI_TAN:
1939 Res0 = APFloat{tan(x: opr0)};
1940 return true;
1941
1942 case AMDGPULibFunc::EI_TANH:
1943 Res0 = APFloat{tanh(x: opr0)};
1944 return true;
1945
1946 case AMDGPULibFunc::EI_TANPI:
1947 Res0 = APFloat{tan(MATH_PI * opr0)};
1948 return true;
1949
1950 // two-arg functions
1951 case AMDGPULibFunc::EI_POW:
1952 case AMDGPULibFunc::EI_POWR: {
1953 ConstantFP *fpopr1 = dyn_cast_or_null<ConstantFP>(Val: copr1);
1954 if (!fpopr1)
1955 return false;
1956 double opr1 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1957 ? fpopr1->getValueAPF().convertToDouble()
1958 : (double)fpopr1->getValueAPF().convertToFloat();
1959 Res0 = APFloat{pow(x: opr0, y: opr1)};
1960 return true;
1961 }
1962
1963 case AMDGPULibFunc::EI_POWN: {
1964 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(Val: copr1)) {
1965 double val = (double)iopr1->getSExtValue();
1966 Res0 = APFloat{pow(x: opr0, y: val)};
1967 return true;
1968 }
1969 return false;
1970 }
1971
1972 case AMDGPULibFunc::EI_ROOTN: {
1973 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(Val: copr1)) {
1974 double val = (double)iopr1->getSExtValue();
1975 Res0 = APFloat{pow(x: opr0, y: 1.0 / val)};
1976 return true;
1977 }
1978 return false;
1979 }
1980
1981 // with ptr arg
1982 case AMDGPULibFunc::EI_SINCOS:
1983 Res0 = APFloat{sin(x: opr0)};
1984 Res1 = APFloat{cos(x: opr0)};
1985 return true;
1986 }
1987
1988 return false;
1989}
1990
1991bool AMDGPULibCalls::evaluateCall(CallInst *aCI, const FuncInfo &FInfo) {
1992 int numArgs = (int)aCI->arg_size();
1993 if (numArgs > 3)
1994 return false;
1995
1996 Constant *copr0 = nullptr;
1997 Constant *copr1 = nullptr;
1998 if (numArgs > 0) {
1999 if ((copr0 = dyn_cast<Constant>(Val: aCI->getArgOperand(i: 0))) == nullptr)
2000 return false;
2001 }
2002
2003 if (numArgs > 1) {
2004 if ((copr1 = dyn_cast<Constant>(Val: aCI->getArgOperand(i: 1))) == nullptr) {
2005 if (FInfo.getId() != AMDGPULibFunc::EI_SINCOS)
2006 return false;
2007 }
2008 }
2009
2010 // At this point, all arguments to aCI are constants.
2011
2012 // max vector size is 16, and sincos will generate two results.
2013 SmallVector<APFloat, 16> Val0, Val1;
2014 int FuncVecSize = getVecSize(FInfo);
2015 if (FuncVecSize == 1) {
2016 if (!evaluateScalarMathFunc(FInfo, Res0&: Val0.emplace_back(Args: 0.0),
2017 Res1&: Val1.emplace_back(Args: 0.0), copr0, copr1)) {
2018 return false;
2019 }
2020 } else {
2021 // An operand of a vector variant is not necessarily a vector: sincos takes
2022 // a pointer as its second operand, and fmin/fmax/ldexp accept an
2023 // implicitly splatted scalar. Only index into actual vectors.
2024 Constant *CV0 = copr0 && copr0->getType()->isVectorTy() ? copr0 : nullptr;
2025 Constant *CV1 = copr1 && copr1->getType()->isVectorTy() ? copr1 : nullptr;
2026 for (int i = 0; i < FuncVecSize; ++i) {
2027 Constant *celt0 = CV0 ? CV0->getAggregateElement(Elt: (unsigned)i) : nullptr;
2028 Constant *celt1 = CV1 ? CV1->getAggregateElement(Elt: (unsigned)i) : nullptr;
2029 if (!evaluateScalarMathFunc(FInfo, Res0&: Val0.emplace_back(Args: 0.0),
2030 Res1&: Val1.emplace_back(Args: 0.0), copr0: celt0, copr1: celt1)) {
2031 return false;
2032 }
2033 }
2034 }
2035
2036 Constant *nval0 = getConstantFloat(Values: Val0, Ty: aCI->getType());
2037
2038 // sincos
2039 if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS) {
2040 Constant *nval1 = getConstantFloat(Values: Val1, Ty: aCI->getType());
2041 new StoreInst(nval1, aCI->getArgOperand(i: 1), aCI->getIterator());
2042 }
2043
2044 replaceCall(I: aCI, With: nval0);
2045 return true;
2046}
2047
2048PreservedAnalyses AMDGPUSimplifyLibCallsPass::run(Function &F,
2049 FunctionAnalysisManager &AM) {
2050 AMDGPULibCalls Simplifier(F, AM);
2051 Simplifier.initNativeFuncs();
2052
2053 bool Changed = false;
2054
2055 LLVM_DEBUG(dbgs() << "AMDIC: process function ";
2056 F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';);
2057
2058 for (auto &BB : F) {
2059 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) {
2060 // Ignore non-calls.
2061 CallInst *CI = dyn_cast<CallInst>(Val&: I);
2062 ++I;
2063
2064 if (CI) {
2065 if (Simplifier.fold(CI))
2066 Changed = true;
2067 }
2068 }
2069 }
2070 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
2071}
2072
2073PreservedAnalyses AMDGPUUseNativeCallsPass::run(Function &F,
2074 FunctionAnalysisManager &AM) {
2075 if (UseNative.empty())
2076 return PreservedAnalyses::all();
2077
2078 AMDGPULibCalls Simplifier(F, AM);
2079 Simplifier.initNativeFuncs();
2080
2081 bool Changed = false;
2082 for (auto &BB : F) {
2083 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) {
2084 // Ignore non-calls.
2085 CallInst *CI = dyn_cast<CallInst>(Val&: I);
2086 ++I;
2087 if (CI && Simplifier.useNative(aCI: CI))
2088 Changed = true;
2089 }
2090 }
2091 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
2092}
2093