1//===- AMDGPULibCalls.cpp -------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file does AMD library function optimizations.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "AMDGPULibFunc.h"
16#include "llvm/Analysis/AssumptionCache.h"
17#include "llvm/Analysis/TargetLibraryInfo.h"
18#include "llvm/Analysis/ValueTracking.h"
19#include "llvm/IR/AttributeMask.h"
20#include "llvm/IR/Dominators.h"
21#include "llvm/IR/IRBuilder.h"
22#include "llvm/IR/MDBuilder.h"
23#include "llvm/IR/PatternMatch.h"
24#include <cmath>
25
26#define DEBUG_TYPE "amdgpu-simplifylib"
27
28using namespace llvm;
29using namespace llvm::PatternMatch;
30
31static cl::opt<bool> EnablePreLink("amdgpu-prelink",
32 cl::desc("Enable pre-link mode optimizations"),
33 cl::init(Val: false),
34 cl::Hidden);
35
36static cl::list<std::string> UseNative("amdgpu-use-native",
37 cl::desc("Comma separated list of functions to replace with native, or all"),
38 cl::CommaSeparated, cl::ValueOptional,
39 cl::Hidden);
40
41#define MATH_PI numbers::pi
42#define MATH_E numbers::e
43#define MATH_SQRT2 numbers::sqrt2
44#define MATH_SQRT1_2 numbers::inv_sqrt2
45
46enum class PowKind { Pow, PowR, PowN, RootN };
47
48namespace llvm {
49
50class AMDGPULibCalls {
51private:
52 SimplifyQuery SQ;
53
54 using FuncInfo = llvm::AMDGPULibFunc;
55
56 // -fuse-native.
57 bool AllNative = false;
58
59 bool useNativeFunc(const StringRef F) const;
60
61 // Return a pointer (pointer expr) to the function if function definition with
62 // "FuncName" exists. It may create a new function prototype in pre-link mode.
63 FunctionCallee getFunction(Module *M, const FuncInfo &fInfo);
64
65 /// Wrapper around getFunction which tries to use a faster variant if
66 /// available, and falls back to a less fast option.
67 ///
68 /// Return a replacement function for \p fInfo that has float-typed fast
69 /// variants. \p NewFunc is a base replacement function to use. \p
70 /// NewFuncFastVariant is a faster version to use if the calling context knows
71 /// it's legal. If there is no fast variant to use, \p NewFuncFastVariant
72 /// should be EI_NONE.
73 FunctionCallee getFloatFastVariant(Module *M, const FuncInfo &fInfo,
74 FuncInfo &newInfo,
75 AMDGPULibFunc::EFuncId NewFunc,
76 AMDGPULibFunc::EFuncId NewFuncFastVariant);
77
78 bool parseFunctionName(const StringRef &FMangledName, FuncInfo &FInfo);
79
80 bool TDOFold(CallInst *CI, const FuncInfo &FInfo);
81
82 /* Specialized optimizations */
83
84 // pow/powr/pown
85 bool fold_pow(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo);
86
87 /// Peform a fast math expansion of pow, powr, pown or rootn.
88 bool expandFastPow(FPMathOperator *FPOp, IRBuilder<> &B, PowKind Kind);
89
90 bool tryOptimizePow(FPMathOperator *FPOp, IRBuilder<> &B,
91 const FuncInfo &FInfo);
92
93 // rootn
94 bool fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo);
95
96 // -fuse-native for sincos
97 bool sincosUseNative(CallInst *aCI, const FuncInfo &FInfo);
98
99 // evaluate calls if calls' arguments are constants.
100 bool evaluateScalarMathFunc(const FuncInfo &FInfo, APFloat &Res0,
101 APFloat &Res1, Constant *copr0, Constant *copr1);
102 bool evaluateCall(CallInst *aCI, const FuncInfo &FInfo);
103
104 /// Insert a value to sincos function \p Fsincos. Returns (value of sin, value
105 /// of cos, sincos call).
106 std::tuple<Value *, Value *, Value *> insertSinCos(Value *Arg,
107 FastMathFlags FMF,
108 IRBuilder<> &B,
109 FunctionCallee Fsincos);
110
111 // sin/cos
112 bool fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo);
113
114 // __read_pipe/__write_pipe
115 bool fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,
116 const FuncInfo &FInfo);
117
118 // Get a scalar native builtin single argument FP function
119 FunctionCallee getNativeFunction(Module *M, const FuncInfo &FInfo);
120
121 /// Substitute a call to a known libcall with an intrinsic call. If \p
122 /// AllowMinSize is true, allow the replacement in a minsize function.
123 bool shouldReplaceLibcallWithIntrinsic(const CallInst *CI,
124 bool AllowMinSizeF32 = false,
125 bool AllowF64 = false,
126 bool AllowStrictFP = false);
127 void replaceLibCallWithSimpleIntrinsic(IRBuilder<> &B, CallInst *CI,
128 Intrinsic::ID IntrID);
129
130 bool tryReplaceLibcallWithSimpleIntrinsic(IRBuilder<> &B, CallInst *CI,
131 Intrinsic::ID IntrID,
132 bool AllowMinSizeF32 = false,
133 bool AllowF64 = false,
134 bool AllowStrictFP = false);
135
136protected:
137 bool isUnsafeFiniteOnlyMath(const FPMathOperator *FPOp) const;
138
139 bool canIncreasePrecisionOfConstantFold(const FPMathOperator *FPOp) const;
140
141 static void replaceCall(Instruction *I, Value *With) {
142 I->replaceAllUsesWith(V: With);
143 I->eraseFromParent();
144 }
145
146 static void replaceCall(FPMathOperator *I, Value *With) {
147 replaceCall(I: cast<Instruction>(Val: I), With);
148 }
149
150public:
151 AMDGPULibCalls(Function &F, FunctionAnalysisManager &FAM);
152
153 bool fold(CallInst *CI);
154
155 void initNativeFuncs();
156
157 // Replace a normal math function call with that native version
158 bool useNative(CallInst *CI);
159};
160
161} // end namespace llvm
162
163template <typename IRB>
164static CallInst *CreateCallEx(IRB &B, FunctionCallee Callee, Value *Arg,
165 const Twine &Name = "") {
166 CallInst *R = B.CreateCall(Callee, Arg, Name);
167 if (Function *F = dyn_cast<Function>(Val: Callee.getCallee()))
168 R->setCallingConv(F->getCallingConv());
169 return R;
170}
171
172template <typename IRB>
173static CallInst *CreateCallEx2(IRB &B, FunctionCallee Callee, Value *Arg1,
174 Value *Arg2, const Twine &Name = "") {
175 CallInst *R = B.CreateCall(Callee, {Arg1, Arg2}, Name);
176 if (Function *F = dyn_cast<Function>(Val: Callee.getCallee()))
177 R->setCallingConv(F->getCallingConv());
178 return R;
179}
180
181static FunctionType *getPownType(FunctionType *FT) {
182 Type *PowNExpTy = Type::getInt32Ty(C&: FT->getContext());
183 if (VectorType *VecTy = dyn_cast<VectorType>(Val: FT->getReturnType()))
184 PowNExpTy = VectorType::get(ElementType: PowNExpTy, EC: VecTy->getElementCount());
185
186 return FunctionType::get(Result: FT->getReturnType(),
187 Params: {FT->getParamType(i: 0), PowNExpTy}, isVarArg: false);
188}
189
190// Data structures for table-driven optimizations.
191// FuncTbl works for both f32 and f64 functions with 1 input argument
192
193struct TableEntry {
194 double result;
195 double input;
196};
197
198/* a list of {result, input} */
199static const TableEntry tbl_acos[] = {
200 {MATH_PI / 2.0, .input: 0.0},
201 {MATH_PI / 2.0, .input: -0.0},
202 {.result: 0.0, .input: 1.0},
203 {MATH_PI, .input: -1.0}
204};
205static const TableEntry tbl_acosh[] = {
206 {.result: 0.0, .input: 1.0}
207};
208static const TableEntry tbl_acospi[] = {
209 {.result: 0.5, .input: 0.0},
210 {.result: 0.5, .input: -0.0},
211 {.result: 0.0, .input: 1.0},
212 {.result: 1.0, .input: -1.0}
213};
214static const TableEntry tbl_asin[] = {
215 {.result: 0.0, .input: 0.0},
216 {.result: -0.0, .input: -0.0},
217 {MATH_PI / 2.0, .input: 1.0},
218 {.result: -MATH_PI / 2.0, .input: -1.0}
219};
220static const TableEntry tbl_asinh[] = {
221 {.result: 0.0, .input: 0.0},
222 {.result: -0.0, .input: -0.0}
223};
224static const TableEntry tbl_asinpi[] = {
225 {.result: 0.0, .input: 0.0},
226 {.result: -0.0, .input: -0.0},
227 {.result: 0.5, .input: 1.0},
228 {.result: -0.5, .input: -1.0}
229};
230static const TableEntry tbl_atan[] = {
231 {.result: 0.0, .input: 0.0},
232 {.result: -0.0, .input: -0.0},
233 {MATH_PI / 4.0, .input: 1.0},
234 {.result: -MATH_PI / 4.0, .input: -1.0}
235};
236static const TableEntry tbl_atanh[] = {
237 {.result: 0.0, .input: 0.0},
238 {.result: -0.0, .input: -0.0}
239};
240static const TableEntry tbl_atanpi[] = {
241 {.result: 0.0, .input: 0.0},
242 {.result: -0.0, .input: -0.0},
243 {.result: 0.25, .input: 1.0},
244 {.result: -0.25, .input: -1.0}
245};
246static const TableEntry tbl_cbrt[] = {
247 {.result: 0.0, .input: 0.0},
248 {.result: -0.0, .input: -0.0},
249 {.result: 1.0, .input: 1.0},
250 {.result: -1.0, .input: -1.0},
251};
252static const TableEntry tbl_cos[] = {
253 {.result: 1.0, .input: 0.0},
254 {.result: 1.0, .input: -0.0}
255};
256static const TableEntry tbl_cosh[] = {
257 {.result: 1.0, .input: 0.0},
258 {.result: 1.0, .input: -0.0}
259};
260static const TableEntry tbl_cospi[] = {
261 {.result: 1.0, .input: 0.0},
262 {.result: 1.0, .input: -0.0}
263};
264static const TableEntry tbl_erfc[] = {
265 {.result: 1.0, .input: 0.0},
266 {.result: 1.0, .input: -0.0}
267};
268static const TableEntry tbl_erf[] = {
269 {.result: 0.0, .input: 0.0},
270 {.result: -0.0, .input: -0.0}
271};
272static const TableEntry tbl_exp[] = {
273 {.result: 1.0, .input: 0.0},
274 {.result: 1.0, .input: -0.0},
275 {MATH_E, .input: 1.0}
276};
277static const TableEntry tbl_exp2[] = {
278 {.result: 1.0, .input: 0.0},
279 {.result: 1.0, .input: -0.0},
280 {.result: 2.0, .input: 1.0}
281};
282static const TableEntry tbl_exp10[] = {
283 {.result: 1.0, .input: 0.0},
284 {.result: 1.0, .input: -0.0},
285 {.result: 10.0, .input: 1.0}
286};
287static const TableEntry tbl_expm1[] = {
288 {.result: 0.0, .input: 0.0},
289 {.result: -0.0, .input: -0.0}
290};
291static const TableEntry tbl_log[] = {
292 {.result: 0.0, .input: 1.0},
293 {.result: 1.0, MATH_E}
294};
295static const TableEntry tbl_log2[] = {
296 {.result: 0.0, .input: 1.0},
297 {.result: 1.0, .input: 2.0}
298};
299static const TableEntry tbl_log10[] = {
300 {.result: 0.0, .input: 1.0},
301 {.result: 1.0, .input: 10.0}
302};
303static const TableEntry tbl_rsqrt[] = {
304 {.result: 1.0, .input: 1.0},
305 {MATH_SQRT1_2, .input: 2.0}
306};
307static const TableEntry tbl_sin[] = {
308 {.result: 0.0, .input: 0.0},
309 {.result: -0.0, .input: -0.0}
310};
311static const TableEntry tbl_sinh[] = {
312 {.result: 0.0, .input: 0.0},
313 {.result: -0.0, .input: -0.0}
314};
315static const TableEntry tbl_sinpi[] = {
316 {.result: 0.0, .input: 0.0},
317 {.result: -0.0, .input: -0.0}
318};
319static const TableEntry tbl_sqrt[] = {
320 {.result: 0.0, .input: 0.0},
321 {.result: 1.0, .input: 1.0},
322 {MATH_SQRT2, .input: 2.0}
323};
324static const TableEntry tbl_tan[] = {
325 {.result: 0.0, .input: 0.0},
326 {.result: -0.0, .input: -0.0}
327};
328static const TableEntry tbl_tanh[] = {
329 {.result: 0.0, .input: 0.0},
330 {.result: -0.0, .input: -0.0}
331};
332static const TableEntry tbl_tanpi[] = {
333 {.result: 0.0, .input: 0.0},
334 {.result: -0.0, .input: -0.0}
335};
336static const TableEntry tbl_tgamma[] = {
337 {.result: 1.0, .input: 1.0},
338 {.result: 1.0, .input: 2.0},
339 {.result: 2.0, .input: 3.0},
340 {.result: 6.0, .input: 4.0}
341};
342
343static bool HasNative(AMDGPULibFunc::EFuncId id) {
344 switch(id) {
345 case AMDGPULibFunc::EI_DIVIDE:
346 case AMDGPULibFunc::EI_COS:
347 case AMDGPULibFunc::EI_EXP:
348 case AMDGPULibFunc::EI_EXP2:
349 case AMDGPULibFunc::EI_EXP10:
350 case AMDGPULibFunc::EI_LOG:
351 case AMDGPULibFunc::EI_LOG2:
352 case AMDGPULibFunc::EI_LOG10:
353 case AMDGPULibFunc::EI_POWR:
354 case AMDGPULibFunc::EI_RECIP:
355 case AMDGPULibFunc::EI_RSQRT:
356 case AMDGPULibFunc::EI_SIN:
357 case AMDGPULibFunc::EI_SINCOS:
358 case AMDGPULibFunc::EI_SQRT:
359 case AMDGPULibFunc::EI_TAN:
360 return true;
361 default:;
362 }
363 return false;
364}
365
366using TableRef = ArrayRef<TableEntry>;
367
368static TableRef getOptTable(AMDGPULibFunc::EFuncId id) {
369 switch(id) {
370 case AMDGPULibFunc::EI_ACOS: return TableRef(tbl_acos);
371 case AMDGPULibFunc::EI_ACOSH: return TableRef(tbl_acosh);
372 case AMDGPULibFunc::EI_ACOSPI: return TableRef(tbl_acospi);
373 case AMDGPULibFunc::EI_ASIN: return TableRef(tbl_asin);
374 case AMDGPULibFunc::EI_ASINH: return TableRef(tbl_asinh);
375 case AMDGPULibFunc::EI_ASINPI: return TableRef(tbl_asinpi);
376 case AMDGPULibFunc::EI_ATAN: return TableRef(tbl_atan);
377 case AMDGPULibFunc::EI_ATANH: return TableRef(tbl_atanh);
378 case AMDGPULibFunc::EI_ATANPI: return TableRef(tbl_atanpi);
379 case AMDGPULibFunc::EI_CBRT: return TableRef(tbl_cbrt);
380 case AMDGPULibFunc::EI_NCOS:
381 case AMDGPULibFunc::EI_COS: return TableRef(tbl_cos);
382 case AMDGPULibFunc::EI_COSH: return TableRef(tbl_cosh);
383 case AMDGPULibFunc::EI_COSPI: return TableRef(tbl_cospi);
384 case AMDGPULibFunc::EI_ERFC: return TableRef(tbl_erfc);
385 case AMDGPULibFunc::EI_ERF: return TableRef(tbl_erf);
386 case AMDGPULibFunc::EI_EXP: return TableRef(tbl_exp);
387 case AMDGPULibFunc::EI_NEXP2:
388 case AMDGPULibFunc::EI_EXP2: return TableRef(tbl_exp2);
389 case AMDGPULibFunc::EI_EXP10: return TableRef(tbl_exp10);
390 case AMDGPULibFunc::EI_EXPM1: return TableRef(tbl_expm1);
391 case AMDGPULibFunc::EI_LOG: return TableRef(tbl_log);
392 case AMDGPULibFunc::EI_NLOG2:
393 case AMDGPULibFunc::EI_LOG2: return TableRef(tbl_log2);
394 case AMDGPULibFunc::EI_LOG10: return TableRef(tbl_log10);
395 case AMDGPULibFunc::EI_NRSQRT:
396 case AMDGPULibFunc::EI_RSQRT: return TableRef(tbl_rsqrt);
397 case AMDGPULibFunc::EI_NSIN:
398 case AMDGPULibFunc::EI_SIN: return TableRef(tbl_sin);
399 case AMDGPULibFunc::EI_SINH: return TableRef(tbl_sinh);
400 case AMDGPULibFunc::EI_SINPI: return TableRef(tbl_sinpi);
401 case AMDGPULibFunc::EI_NSQRT:
402 case AMDGPULibFunc::EI_SQRT: return TableRef(tbl_sqrt);
403 case AMDGPULibFunc::EI_TAN: return TableRef(tbl_tan);
404 case AMDGPULibFunc::EI_TANH: return TableRef(tbl_tanh);
405 case AMDGPULibFunc::EI_TANPI: return TableRef(tbl_tanpi);
406 case AMDGPULibFunc::EI_TGAMMA: return TableRef(tbl_tgamma);
407 default:;
408 }
409 return TableRef();
410}
411
412static inline int getVecSize(const AMDGPULibFunc& FInfo) {
413 return FInfo.getLeads()[0].VectorSize;
414}
415
416static inline AMDGPULibFunc::EType getArgType(const AMDGPULibFunc& FInfo) {
417 return (AMDGPULibFunc::EType)FInfo.getLeads()[0].ArgType;
418}
419
420FunctionCallee AMDGPULibCalls::getFunction(Module *M, const FuncInfo &fInfo) {
421 // If we are doing PreLinkOpt, the function is external. So it is safe to
422 // use getOrInsertFunction() at this stage.
423
424 return EnablePreLink ? AMDGPULibFunc::getOrInsertFunction(M, fInfo)
425 : AMDGPULibFunc::getFunction(M, fInfo);
426}
427
428FunctionCallee AMDGPULibCalls::getFloatFastVariant(
429 Module *M, const FuncInfo &fInfo, FuncInfo &newInfo,
430 AMDGPULibFunc::EFuncId NewFunc, AMDGPULibFunc::EFuncId FastVariant) {
431 assert(NewFunc != FastVariant);
432
433 if (FastVariant != AMDGPULibFunc::EI_NONE &&
434 getArgType(FInfo: fInfo) == AMDGPULibFunc::F32) {
435 newInfo = AMDGPULibFunc(FastVariant, fInfo);
436 if (FunctionCallee NewCallee = getFunction(M, fInfo: newInfo))
437 return NewCallee;
438 }
439
440 newInfo = AMDGPULibFunc(NewFunc, fInfo);
441 return getFunction(M, fInfo: newInfo);
442}
443
444bool AMDGPULibCalls::parseFunctionName(const StringRef &FMangledName,
445 FuncInfo &FInfo) {
446 return AMDGPULibFunc::parse(MangledName: FMangledName, Ptr&: FInfo);
447}
448
449bool AMDGPULibCalls::isUnsafeFiniteOnlyMath(const FPMathOperator *FPOp) const {
450 return FPOp->hasApproxFunc() && FPOp->hasNoNaNs() && FPOp->hasNoInfs();
451}
452
453bool AMDGPULibCalls::canIncreasePrecisionOfConstantFold(
454 const FPMathOperator *FPOp) const {
455 // TODO: Refine to approxFunc or contract
456 return FPOp->isFast();
457}
458
459AMDGPULibCalls::AMDGPULibCalls(Function &F, FunctionAnalysisManager &FAM)
460 : SQ(F.getParent()->getDataLayout(),
461 &FAM.getResult<TargetLibraryAnalysis>(IR&: F),
462 FAM.getCachedResult<DominatorTreeAnalysis>(IR&: F),
463 &FAM.getResult<AssumptionAnalysis>(IR&: F)) {}
464
465bool AMDGPULibCalls::useNativeFunc(const StringRef F) const {
466 return AllNative || llvm::is_contained(Range&: UseNative, Element: F);
467}
468
469void AMDGPULibCalls::initNativeFuncs() {
470 AllNative = useNativeFunc(F: "all") ||
471 (UseNative.getNumOccurrences() && UseNative.size() == 1 &&
472 UseNative.begin()->empty());
473}
474
475bool AMDGPULibCalls::sincosUseNative(CallInst *aCI, const FuncInfo &FInfo) {
476 bool native_sin = useNativeFunc(F: "sin");
477 bool native_cos = useNativeFunc(F: "cos");
478
479 if (native_sin && native_cos) {
480 Module *M = aCI->getModule();
481 Value *opr0 = aCI->getArgOperand(i: 0);
482
483 AMDGPULibFunc nf;
484 nf.getLeads()[0].ArgType = FInfo.getLeads()[0].ArgType;
485 nf.getLeads()[0].VectorSize = FInfo.getLeads()[0].VectorSize;
486
487 nf.setPrefix(AMDGPULibFunc::NATIVE);
488 nf.setId(AMDGPULibFunc::EI_SIN);
489 FunctionCallee sinExpr = getFunction(M, fInfo: nf);
490
491 nf.setPrefix(AMDGPULibFunc::NATIVE);
492 nf.setId(AMDGPULibFunc::EI_COS);
493 FunctionCallee cosExpr = getFunction(M, fInfo: nf);
494 if (sinExpr && cosExpr) {
495 Value *sinval =
496 CallInst::Create(Func: sinExpr, Args: opr0, NameStr: "splitsin", InsertBefore: aCI->getIterator());
497 Value *cosval =
498 CallInst::Create(Func: cosExpr, Args: opr0, NameStr: "splitcos", InsertBefore: aCI->getIterator());
499 new StoreInst(cosval, aCI->getArgOperand(i: 1), aCI->getIterator());
500
501 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI
502 << " with native version of sin/cos");
503
504 replaceCall(I: aCI, With: sinval);
505 return true;
506 }
507 }
508 return false;
509}
510
511bool AMDGPULibCalls::useNative(CallInst *aCI) {
512 Function *Callee = aCI->getCalledFunction();
513 if (!Callee || aCI->isNoBuiltin())
514 return false;
515
516 FuncInfo FInfo;
517 if (!parseFunctionName(FMangledName: Callee->getName(), FInfo) || !FInfo.isMangled() ||
518 FInfo.getPrefix() != AMDGPULibFunc::NOPFX ||
519 getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(id: FInfo.getId()) ||
520 !(AllNative || useNativeFunc(F: FInfo.getName()))) {
521 return false;
522 }
523
524 if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS)
525 return sincosUseNative(aCI, FInfo);
526
527 FInfo.setPrefix(AMDGPULibFunc::NATIVE);
528 FunctionCallee F = getFunction(M: aCI->getModule(), fInfo: FInfo);
529 if (!F)
530 return false;
531
532 aCI->setCalledFunction(F);
533 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI
534 << " with native version");
535 return true;
536}
537
538// Clang emits call of __read_pipe_2 or __read_pipe_4 for OpenCL read_pipe
539// builtin, with appended type size and alignment arguments, where 2 or 4
540// indicates the original number of arguments. The library has optimized version
541// of __read_pipe_2/__read_pipe_4 when the type size and alignment has the same
542// power of 2 value. This function transforms __read_pipe_2 to __read_pipe_2_N
543// for such cases where N is the size in bytes of the type (N = 1, 2, 4, 8, ...,
544// 128). The same for __read_pipe_4, write_pipe_2, and write_pipe_4.
545bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,
546 const FuncInfo &FInfo) {
547 auto *Callee = CI->getCalledFunction();
548 if (!Callee->isDeclaration())
549 return false;
550
551 assert(Callee->hasName() && "Invalid read_pipe/write_pipe function");
552 auto *M = Callee->getParent();
553 std::string Name = std::string(Callee->getName());
554 auto NumArg = CI->arg_size();
555 if (NumArg != 4 && NumArg != 6)
556 return false;
557 ConstantInt *PacketSize =
558 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: NumArg - 2));
559 ConstantInt *PacketAlign =
560 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: NumArg - 1));
561 if (!PacketSize || !PacketAlign)
562 return false;
563
564 unsigned Size = PacketSize->getZExtValue();
565 Align Alignment = PacketAlign->getAlignValue();
566 if (Alignment != Size)
567 return false;
568
569 unsigned PtrArgLoc = CI->arg_size() - 3;
570 Value *PtrArg = CI->getArgOperand(i: PtrArgLoc);
571 Type *PtrTy = PtrArg->getType();
572
573 SmallVector<llvm::Type *, 6> ArgTys;
574 for (unsigned I = 0; I != PtrArgLoc; ++I)
575 ArgTys.push_back(Elt: CI->getArgOperand(i: I)->getType());
576 ArgTys.push_back(Elt: PtrTy);
577
578 Name = Name + "_" + std::to_string(val: Size);
579 auto *FTy = FunctionType::get(Result: Callee->getReturnType(),
580 Params: ArrayRef<Type *>(ArgTys), isVarArg: false);
581 AMDGPULibFunc NewLibFunc(Name, FTy);
582 FunctionCallee F = AMDGPULibFunc::getOrInsertFunction(M, fInfo: NewLibFunc);
583 if (!F)
584 return false;
585
586 SmallVector<Value *, 6> Args;
587 for (unsigned I = 0; I != PtrArgLoc; ++I)
588 Args.push_back(Elt: CI->getArgOperand(i: I));
589 Args.push_back(Elt: PtrArg);
590
591 auto *NCI = B.CreateCall(Callee: F, Args);
592 NCI->setAttributes(CI->getAttributes());
593 CI->replaceAllUsesWith(V: NCI);
594 CI->dropAllReferences();
595 CI->eraseFromParent();
596
597 return true;
598}
599
600// This function returns false if no change; return true otherwise.
601bool AMDGPULibCalls::fold(CallInst *CI) {
602 Function *Callee = CI->getCalledFunction();
603 // Ignore indirect calls.
604 if (!Callee || Callee->isIntrinsic() || CI->isNoBuiltin())
605 return false;
606
607 FuncInfo FInfo;
608 if (!parseFunctionName(FMangledName: Callee->getName(), FInfo))
609 return false;
610
611 // Further check the number of arguments to see if they match.
612 // TODO: Check calling convention matches too
613 if (!FInfo.isCompatibleSignature(M: *Callee->getParent(), FuncTy: CI->getFunctionType()))
614 return false;
615
616 LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << '\n');
617
618 if (TDOFold(CI, FInfo))
619 return true;
620
621 IRBuilder<> B(CI);
622 if (CI->isStrictFP())
623 B.setIsFPConstrained(true);
624
625 if (FPMathOperator *FPOp = dyn_cast<FPMathOperator>(Val: CI)) {
626 // Under unsafe-math, evaluate calls if possible.
627 // According to Brian Sumner, we can do this for all f32 function calls
628 // using host's double function calls.
629 if (canIncreasePrecisionOfConstantFold(FPOp) && evaluateCall(aCI: CI, FInfo))
630 return true;
631
632 // Copy fast flags from the original call.
633 FastMathFlags FMF = FPOp->getFastMathFlags();
634 B.setFastMathFlags(FMF);
635
636 // Specialized optimizations for each function call.
637 //
638 // TODO: Handle native functions
639 switch (FInfo.getId()) {
640 case AMDGPULibFunc::EI_EXP:
641 if (FMF.none())
642 return false;
643 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::exp,
644 AllowMinSizeF32: FMF.approxFunc());
645 case AMDGPULibFunc::EI_EXP2:
646 if (FMF.none())
647 return false;
648 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::exp2,
649 AllowMinSizeF32: FMF.approxFunc());
650 case AMDGPULibFunc::EI_LOG:
651 if (FMF.none())
652 return false;
653 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::log,
654 AllowMinSizeF32: FMF.approxFunc());
655 case AMDGPULibFunc::EI_LOG2:
656 if (FMF.none())
657 return false;
658 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::log2,
659 AllowMinSizeF32: FMF.approxFunc());
660 case AMDGPULibFunc::EI_LOG10:
661 if (FMF.none())
662 return false;
663 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::log10,
664 AllowMinSizeF32: FMF.approxFunc());
665 case AMDGPULibFunc::EI_FMIN:
666 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::minnum,
667 AllowMinSizeF32: true, AllowF64: true);
668 case AMDGPULibFunc::EI_FMAX:
669 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::maxnum,
670 AllowMinSizeF32: true, AllowF64: true);
671 case AMDGPULibFunc::EI_FMA:
672 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::fma, AllowMinSizeF32: true,
673 AllowF64: true);
674 case AMDGPULibFunc::EI_MAD:
675 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::fmuladd,
676 AllowMinSizeF32: true, AllowF64: true);
677 case AMDGPULibFunc::EI_FABS:
678 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::fabs, AllowMinSizeF32: true,
679 AllowF64: true, AllowStrictFP: true);
680 case AMDGPULibFunc::EI_COPYSIGN:
681 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::copysign,
682 AllowMinSizeF32: true, AllowF64: true, AllowStrictFP: true);
683 case AMDGPULibFunc::EI_FLOOR:
684 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::floor, AllowMinSizeF32: true,
685 AllowF64: true);
686 case AMDGPULibFunc::EI_CEIL:
687 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::ceil, AllowMinSizeF32: true,
688 AllowF64: true);
689 case AMDGPULibFunc::EI_TRUNC:
690 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::trunc, AllowMinSizeF32: true,
691 AllowF64: true);
692 case AMDGPULibFunc::EI_RINT:
693 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::rint, AllowMinSizeF32: true,
694 AllowF64: true);
695 case AMDGPULibFunc::EI_ROUND:
696 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::round, AllowMinSizeF32: true,
697 AllowF64: true);
698 case AMDGPULibFunc::EI_LDEXP: {
699 if (!shouldReplaceLibcallWithIntrinsic(CI, AllowMinSizeF32: true, AllowF64: true))
700 return false;
701
702 Value *Arg1 = CI->getArgOperand(i: 1);
703 if (VectorType *VecTy = dyn_cast<VectorType>(Val: CI->getType());
704 VecTy && !isa<VectorType>(Val: Arg1->getType())) {
705 Value *SplatArg1 = B.CreateVectorSplat(EC: VecTy->getElementCount(), V: Arg1);
706 CI->setArgOperand(i: 1, v: SplatArg1);
707 }
708
709 CI->setCalledFunction(Intrinsic::getOrInsertDeclaration(
710 M: CI->getModule(), id: Intrinsic::ldexp,
711 OverloadTys: {CI->getType(), CI->getArgOperand(i: 1)->getType()}));
712 CI->setCallingConv(CallingConv::C);
713 return true;
714 }
715 case AMDGPULibFunc::EI_POW:
716 case AMDGPULibFunc::EI_POW_FAST:
717 return tryOptimizePow(FPOp, B, FInfo);
718 case AMDGPULibFunc::EI_POWR:
719 case AMDGPULibFunc::EI_POWR_FAST: {
720 if (fold_pow(FPOp, B, FInfo))
721 return true;
722 if (!FMF.approxFunc())
723 return false;
724
725 if (FInfo.getId() == AMDGPULibFunc::EI_POWR && FMF.approxFunc() &&
726 getArgType(FInfo) == AMDGPULibFunc::F32) {
727 Module *M = Callee->getParent();
728 AMDGPULibFunc PowrFastInfo(AMDGPULibFunc::EI_POWR_FAST, FInfo);
729 if (FunctionCallee PowrFastFunc = getFunction(M, fInfo: PowrFastInfo)) {
730 CI->setCalledFunction(PowrFastFunc);
731 return true;
732 }
733 }
734
735 if (!shouldReplaceLibcallWithIntrinsic(CI))
736 return false;
737 return expandFastPow(FPOp, B, Kind: PowKind::PowR);
738 }
739 case AMDGPULibFunc::EI_POWN:
740 case AMDGPULibFunc::EI_POWN_FAST: {
741 if (fold_pow(FPOp, B, FInfo))
742 return true;
743 if (!FMF.approxFunc())
744 return false;
745
746 if (FInfo.getId() == AMDGPULibFunc::EI_POWN &&
747 getArgType(FInfo) == AMDGPULibFunc::F32) {
748 Module *M = Callee->getParent();
749 AMDGPULibFunc PownFastInfo(AMDGPULibFunc::EI_POWN_FAST, FInfo);
750 if (FunctionCallee PownFastFunc = getFunction(M, fInfo: PownFastInfo)) {
751 CI->setCalledFunction(PownFastFunc);
752 return true;
753 }
754 }
755
756 if (!shouldReplaceLibcallWithIntrinsic(CI))
757 return false;
758 return expandFastPow(FPOp, B, Kind: PowKind::PowN);
759 }
760 case AMDGPULibFunc::EI_ROOTN:
761 case AMDGPULibFunc::EI_ROOTN_FAST: {
762 if (fold_rootn(FPOp, B, FInfo))
763 return true;
764 if (!FMF.approxFunc())
765 return false;
766
767 if (getArgType(FInfo) == AMDGPULibFunc::F32) {
768 Module *M = Callee->getParent();
769 AMDGPULibFunc RootnFastInfo(AMDGPULibFunc::EI_ROOTN_FAST, FInfo);
770 if (FunctionCallee RootnFastFunc = getFunction(M, fInfo: RootnFastInfo)) {
771 CI->setCalledFunction(RootnFastFunc);
772 return true;
773 }
774 }
775
776 return expandFastPow(FPOp, B, Kind: PowKind::RootN);
777 }
778 case AMDGPULibFunc::EI_SQRT:
779 // TODO: Allow with strictfp + constrained intrinsic
780 return tryReplaceLibcallWithSimpleIntrinsic(
781 B, CI, IntrID: Intrinsic::sqrt, AllowMinSizeF32: true, AllowF64: true, /*AllowStrictFP=*/false);
782 case AMDGPULibFunc::EI_COS:
783 case AMDGPULibFunc::EI_SIN:
784 return fold_sincos(FPOp, B, FInfo);
785 default:
786 break;
787 }
788 } else {
789 // Specialized optimizations for each function call
790 switch (FInfo.getId()) {
791 case AMDGPULibFunc::EI_READ_PIPE_2:
792 case AMDGPULibFunc::EI_READ_PIPE_4:
793 case AMDGPULibFunc::EI_WRITE_PIPE_2:
794 case AMDGPULibFunc::EI_WRITE_PIPE_4:
795 return fold_read_write_pipe(CI, B, FInfo);
796 default:
797 break;
798 }
799 }
800
801 return false;
802}
803
804static Constant *getConstantFloat(const ArrayRef<APFloat> Values,
805 const Type *Ty) {
806
807 assert(Ty->isSingleValueType() &&
808 "Type must either be a scalar or a vector.");
809 assert((!Ty->isVectorTy() || Ty->isScalableTy() ||
810 Values.size() == cast<FixedVectorType>(Ty)->getNumElements()) &&
811 "Unexpected number of constant values.");
812 assert((Ty->isVectorTy() || Values.size() == 1) &&
813 "Expected exactly one constant value");
814
815 Type *ElemTy = Ty->getScalarType();
816 const fltSemantics &FltSem = ElemTy->getFltSemantics();
817
818 SmallVector<Constant *, 4> ConstValues;
819 ConstValues.reserve(N: Values.size());
820 for (APFloat APF : Values) {
821 bool Unused;
822 APF.convert(ToSemantics: FltSem, RM: APFloat::rmNearestTiesToEven, losesInfo: &Unused);
823 ConstValues.push_back(Elt: ConstantFP::get(Ty: ElemTy, V: APF));
824 }
825
826 return Ty->isVectorTy() ? ConstantVector::get(V: ConstValues) : ConstValues[0];
827}
828
829bool AMDGPULibCalls::TDOFold(CallInst *CI, const FuncInfo &FInfo) {
830 // Table-Driven optimization
831 const TableRef tr = getOptTable(id: FInfo.getId());
832 if (tr.empty())
833 return false;
834
835 int const sz = (int)tr.size();
836 Value *opr0 = CI->getArgOperand(i: 0);
837
838 int vecSize = getVecSize(FInfo);
839 if (vecSize > 1) {
840 // Vector version
841 Constant *CV = dyn_cast<Constant>(Val: opr0);
842 if (CV && CV->getType()->isVectorTy()) {
843 SmallVector<APFloat, 4> Values;
844 Values.reserve(N: vecSize);
845 for (int eltNo = 0; eltNo < vecSize; ++eltNo) {
846 // A lane may be undef or poison, in which case there is nothing to
847 // look up in the table.
848 ConstantFP *eltval = dyn_cast_or_null<ConstantFP>(
849 Val: CV->getAggregateElement(Elt: (unsigned)eltNo));
850 if (!eltval)
851 return false;
852 auto MatchingRow = llvm::find_if(Range: tr, P: [eltval](const TableEntry &entry) {
853 return eltval->isExactlyValue(V: entry.input);
854 });
855 if (MatchingRow == tr.end())
856 return false;
857 Values.push_back(Elt: APFloat(MatchingRow->result));
858 }
859 Constant *NewValues = getConstantFloat(Values, Ty: CI->getType());
860 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *NewValues << "\n");
861 replaceCall(I: CI, With: NewValues);
862 return true;
863 }
864 } else {
865 // Scalar version
866 if (ConstantFP *CF = dyn_cast<ConstantFP>(Val: opr0)) {
867 for (int i = 0; i < sz; ++i) {
868 if (CF->isExactlyValue(V: tr[i].input)) {
869 Value *nval = ConstantFP::get(Ty: CF->getType(), V: tr[i].result);
870 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");
871 replaceCall(I: CI, With: nval);
872 return true;
873 }
874 }
875 }
876 }
877
878 return false;
879}
880
881namespace llvm {
882static double log2(double V) {
883#if _XOPEN_SOURCE >= 600 || defined(_ISOC99_SOURCE) || _POSIX_C_SOURCE >= 200112L
884 return ::log2(x: V);
885#else
886 return log(V) / numbers::ln2;
887#endif
888}
889} // namespace llvm
890
891bool AMDGPULibCalls::fold_pow(FPMathOperator *FPOp, IRBuilder<> &B,
892 const FuncInfo &FInfo) {
893 assert((FInfo.getId() == AMDGPULibFunc::EI_POW ||
894 FInfo.getId() == AMDGPULibFunc::EI_POW_FAST ||
895 FInfo.getId() == AMDGPULibFunc::EI_POWR ||
896 FInfo.getId() == AMDGPULibFunc::EI_POWR_FAST ||
897 FInfo.getId() == AMDGPULibFunc::EI_POWN ||
898 FInfo.getId() == AMDGPULibFunc::EI_POWN_FAST) &&
899 "fold_pow: encounter a wrong function call");
900
901 Module *M = B.GetInsertBlock()->getModule();
902 Type *eltType = FPOp->getType()->getScalarType();
903 Value *opr0 = FPOp->getOperand(i: 0);
904 Value *opr1 = FPOp->getOperand(i: 1);
905
906 const APFloat *CF = nullptr;
907 const APInt *CINT = nullptr;
908 if (!match(V: opr1, P: m_APFloatAllowPoison(Res&: CF)))
909 match(V: opr1, P: m_APIntAllowPoison(Res&: CINT));
910
911 // 0x1111111 means that we don't do anything for this call.
912 int ci_opr1 = (CINT ? (int)CINT->getSExtValue() : 0x1111111);
913
914 // OpenCL powr(x<0, y) = NaN, but the folds below would turn it into a
915 // finite number. Skip them unless NaNs are ignored or the base is known
916 // non-negative.
917 bool IsPowr = FInfo.getId() == AMDGPULibFunc::EI_POWR ||
918 FInfo.getId() == AMDGPULibFunc::EI_POWR_FAST;
919 bool SkipConstantFolds =
920 IsPowr && !FPOp->hasNoNaNs() &&
921 !cannotBeOrderedLessThanZero(
922 V: opr0, SQ: SQ.getWithInstruction(I: cast<Instruction>(Val: FPOp)));
923
924 if (CF && (CF->isExactlyValue(V: 0.5) || CF->isExactlyValue(V: -0.5))) {
925 // pow[r](x, [-]0.5) = sqrt(x) / rsqrt(x)
926 //
927 // sqrt/rsqrt and pow disagree on two negative inputs:
928 // pow(-Inf, 0.5) == +Inf but sqrt(-Inf) == NaN (ninf case)
929 // pow(-0.0, 0.5) == +0.0 but sqrt(-0.0) == -0.0 (nsz case)
930 // powr requires x >= 0 by the OpenCL spec, so -Inf is undefined behaviour
931 // and the ninf check can be skipped for powr/powr_fast. -0.0 is a valid
932 // input for powr since -0.0 >= 0 by IEEE comparison, so nsz is still
933 // required for all variants. sqrt/rsqrt already return NaN for a
934 // negative base like powr does, so this fold skips the base-sign check.
935 if (FPOp->hasNoSignedZeros() && (IsPowr || FPOp->hasNoInfs())) {
936 bool issqrt = CF->isExactlyValue(V: 0.5);
937 if (FunctionCallee FPExpr =
938 getFunction(M, fInfo: AMDGPULibFunc(issqrt ? AMDGPULibFunc::EI_SQRT
939 : AMDGPULibFunc::EI_RSQRT,
940 FInfo))) {
941 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << FInfo.getName()
942 << '(' << *opr0 << ")\n");
943 Value *nval = CreateCallEx(B, Callee: FPExpr, Arg: opr0,
944 Name: issqrt ? "__pow2sqrt" : "__pow2rsqrt");
945 replaceCall(I: FPOp, With: nval);
946 return true;
947 }
948 }
949 }
950
951 if (!SkipConstantFolds) {
952 if ((CF && CF->isZero()) || (CINT && ci_opr1 == 0)) {
953 // pow/powr/pown(x, 0) == 1
954 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1\n");
955 Constant *cnval = ConstantFP::get(Ty: eltType, V: 1.0);
956 if (getVecSize(FInfo) > 1) {
957 cnval = ConstantDataVector::getSplat(NumElts: getVecSize(FInfo), Elt: cnval);
958 }
959 replaceCall(I: FPOp, With: cnval);
960 return true;
961 }
962 if ((CF && CF->isOne()) || (CINT && ci_opr1 == 1)) {
963 // pow/powr/pown(x, 1.0) = x
964 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << "\n");
965 replaceCall(I: FPOp, With: opr0);
966 return true;
967 }
968 if ((CF && CF->isExactlyValue(V: 2.0)) || (CINT && ci_opr1 == 2)) {
969 // pow/powr/pown(x, 2.0) = x*x
970 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << " * "
971 << *opr0 << "\n");
972 Value *nval = B.CreateFMul(L: opr0, R: opr0, Name: "__pow2");
973 replaceCall(I: FPOp, With: nval);
974 return true;
975 }
976 if ((CF && CF->isMinusOne()) || (CINT && ci_opr1 == -1)) {
977 // pow/powr/pown(x, -1.0) = 1.0/x
978 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1 / " << *opr0 << "\n");
979 Constant *cnval = ConstantFP::get(Ty: eltType, V: 1.0);
980 if (getVecSize(FInfo) > 1) {
981 cnval = ConstantDataVector::getSplat(NumElts: getVecSize(FInfo), Elt: cnval);
982 }
983 Value *nval = B.CreateFDiv(L: cnval, R: opr0, Name: "__powrecip");
984 replaceCall(I: FPOp, With: nval);
985 return true;
986 }
987 }
988
989 if (!isUnsafeFiniteOnlyMath(FPOp))
990 return false;
991
992 // Unsafe Math optimization
993
994 // Remember that ci_opr1 is set if opr1 is integral
995 if (CF) {
996 double dval = (getArgType(FInfo) == AMDGPULibFunc::F32)
997 ? (double)CF->convertToFloat()
998 : CF->convertToDouble();
999 int ival = (int)dval;
1000 if ((double)ival == dval) {
1001 ci_opr1 = ival;
1002 } else
1003 ci_opr1 = 0x11111111;
1004 }
1005
1006 // pow/powr/pown(x, c) = [1/](x*x*..x); where
1007 // trunc(c) == c && the number of x == c && |c| <= 12
1008 unsigned abs_opr1 = (ci_opr1 < 0) ? -ci_opr1 : ci_opr1;
1009 if (abs_opr1 <= 12) {
1010 Constant *cnval;
1011 Value *nval;
1012 if (abs_opr1 == 0) {
1013 cnval = ConstantFP::get(Ty: eltType, V: 1.0);
1014 if (getVecSize(FInfo) > 1) {
1015 cnval = ConstantDataVector::getSplat(NumElts: getVecSize(FInfo), Elt: cnval);
1016 }
1017 nval = cnval;
1018 } else {
1019 Value *valx2 = nullptr;
1020 nval = nullptr;
1021 while (abs_opr1 > 0) {
1022 valx2 = valx2 ? B.CreateFMul(L: valx2, R: valx2, Name: "__powx2") : opr0;
1023 if (abs_opr1 & 1) {
1024 nval = nval ? B.CreateFMul(L: nval, R: valx2, Name: "__powprod") : valx2;
1025 }
1026 abs_opr1 >>= 1;
1027 }
1028 }
1029
1030 if (ci_opr1 < 0) {
1031 cnval = ConstantFP::get(Ty: eltType, V: 1.0);
1032 if (getVecSize(FInfo) > 1) {
1033 cnval = ConstantDataVector::getSplat(NumElts: getVecSize(FInfo), Elt: cnval);
1034 }
1035 nval = B.CreateFDiv(L: cnval, R: nval, Name: "__1powprod");
1036 }
1037 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> "
1038 << ((ci_opr1 < 0) ? "1/prod(" : "prod(") << *opr0
1039 << ")\n");
1040 replaceCall(I: FPOp, With: nval);
1041 return true;
1042 }
1043
1044 // If we should use the generic intrinsic instead of emitting a libcall
1045 const bool ShouldUseIntrinsic = eltType->isFloatTy() || eltType->isHalfTy();
1046
1047 // powr ---> exp2(y * log2(x))
1048 // pown/pow ---> powr(fabs(x), y) | (x & ((int)y << 31))
1049 FunctionCallee ExpExpr;
1050 if (ShouldUseIntrinsic)
1051 ExpExpr = Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::exp2,
1052 OverloadTys: {FPOp->getType()});
1053 else {
1054 ExpExpr = getFunction(M, fInfo: AMDGPULibFunc(AMDGPULibFunc::EI_EXP2, FInfo));
1055 if (!ExpExpr)
1056 return false;
1057 }
1058
1059 bool needlog = false;
1060 bool needabs = false;
1061 bool needcopysign = false;
1062 Constant *cnval = nullptr;
1063 if (getVecSize(FInfo) == 1) {
1064 CF = nullptr;
1065 match(V: opr0, P: m_APFloatAllowPoison(Res&: CF));
1066
1067 if (CF) {
1068 double V = (getArgType(FInfo) == AMDGPULibFunc::F32)
1069 ? (double)CF->convertToFloat()
1070 : CF->convertToDouble();
1071
1072 V = log2(V: std::abs(x: V));
1073 cnval = ConstantFP::get(Ty: eltType, V);
1074 needcopysign = (FInfo.getId() != AMDGPULibFunc::EI_POWR &&
1075 FInfo.getId() != AMDGPULibFunc::EI_POWR_FAST) &&
1076 CF->isNegative();
1077 } else {
1078 needlog = true;
1079 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR &&
1080 FInfo.getId() != AMDGPULibFunc::EI_POWR_FAST;
1081 }
1082 } else {
1083 ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(Val: opr0);
1084
1085 if (!CDV) {
1086 needlog = true;
1087 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR &&
1088 FInfo.getId() != AMDGPULibFunc::EI_POWR_FAST;
1089 } else {
1090 assert ((int)CDV->getNumElements() == getVecSize(FInfo) &&
1091 "Wrong vector size detected");
1092
1093 SmallVector<double, 0> DVal;
1094 for (int i=0; i < getVecSize(FInfo); ++i) {
1095 double V = CDV->getElementAsAPFloat(i).convertToDouble();
1096 if (V < 0.0) needcopysign = true;
1097 V = log2(V: std::abs(x: V));
1098 DVal.push_back(Elt: V);
1099 }
1100 if (getArgType(FInfo) == AMDGPULibFunc::F32) {
1101 SmallVector<float, 0> FVal;
1102 for (double D : DVal)
1103 FVal.push_back(Elt: (float)D);
1104 ArrayRef<float> tmp(FVal);
1105 cnval = ConstantDataVector::get(Context&: M->getContext(), Elts: tmp);
1106 } else {
1107 ArrayRef<double> tmp(DVal);
1108 cnval = ConstantDataVector::get(Context&: M->getContext(), Elts: tmp);
1109 }
1110 }
1111 }
1112
1113 if (needcopysign && (FInfo.getId() == AMDGPULibFunc::EI_POW ||
1114 FInfo.getId() == AMDGPULibFunc::EI_POW_FAST)) {
1115 // We cannot handle corner cases for a general pow() function, give up
1116 // unless y is a constant integral value. Then proceed as if it were pown.
1117 if (!isKnownIntegral(V: opr1, SQ: SQ.getWithInstruction(I: cast<Instruction>(Val: FPOp)),
1118 FMF: FPOp->getFastMathFlags()))
1119 return false;
1120 }
1121
1122 Value *nval;
1123 if (needabs) {
1124 nval = B.CreateFAbs(V: opr0, FMFSource: nullptr, Name: "__fabs");
1125 } else {
1126 nval = cnval ? cnval : opr0;
1127 }
1128 if (needlog) {
1129 FunctionCallee LogExpr;
1130 if (ShouldUseIntrinsic) {
1131 LogExpr = Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::log2,
1132 OverloadTys: {FPOp->getType()});
1133 } else {
1134 LogExpr = getFunction(M, fInfo: AMDGPULibFunc(AMDGPULibFunc::EI_LOG2, FInfo));
1135 if (!LogExpr)
1136 return false;
1137 }
1138
1139 nval = CreateCallEx(B,Callee: LogExpr, Arg: nval, Name: "__log2");
1140 }
1141
1142 if (FInfo.getId() == AMDGPULibFunc::EI_POWN ||
1143 FInfo.getId() == AMDGPULibFunc::EI_POWN_FAST) {
1144 // convert int(32) to fp(f32 or f64)
1145 opr1 = B.CreateSIToFP(V: opr1, DestTy: nval->getType(), Name: "pownI2F");
1146 }
1147 nval = B.CreateFMul(L: opr1, R: nval, Name: "__ylogx");
1148
1149 CallInst *Exp2Call = CreateCallEx(B, Callee: ExpExpr, Arg: nval, Name: "__exp2");
1150
1151 // TODO: Generalized fpclass logic for pow
1152 FPClassTest KnownNot = FPClassTest::fcNegative;
1153 if (FPOp->hasNoNaNs())
1154 KnownNot |= FPClassTest::fcNan;
1155
1156 Exp2Call->addRetAttr(
1157 Attr: Attribute::getWithNoFPClass(Context&: Exp2Call->getContext(), Mask: KnownNot));
1158 nval = Exp2Call;
1159
1160 if (needcopysign) {
1161 Type* nTyS = B.getIntNTy(N: eltType->getPrimitiveSizeInBits());
1162 Type *nTy = FPOp->getType()->getWithNewType(EltTy: nTyS);
1163 Value *opr_n = FPOp->getOperand(i: 1);
1164 if (opr_n->getType()->getScalarType()->isIntegerTy())
1165 opr_n = B.CreateZExtOrTrunc(V: opr_n, DestTy: nTy, Name: "__ytou");
1166 else
1167 opr_n = B.CreateFPToSI(V: opr1, DestTy: nTy, Name: "__ytou");
1168
1169 unsigned size = nTy->getScalarSizeInBits();
1170 Value *sign = B.CreateShl(LHS: opr_n, RHS: size-1, Name: "__yeven");
1171 sign = B.CreateAnd(LHS: B.CreateBitCast(V: opr0, DestTy: nTy), RHS: sign, Name: "__pow_sign");
1172
1173 nval = B.CreateCopySign(LHS: nval, RHS: B.CreateBitCast(V: sign, DestTy: nval->getType()),
1174 FMFSource: nullptr, Name: "__pow_sign");
1175 }
1176
1177 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> "
1178 << "exp2(" << *opr1 << " * log2(" << *opr0 << "))\n");
1179 replaceCall(I: FPOp, With: nval);
1180
1181 return true;
1182}
1183
1184bool AMDGPULibCalls::fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B,
1185 const FuncInfo &FInfo) {
1186 Value *opr0 = FPOp->getOperand(i: 0);
1187 Value *opr1 = FPOp->getOperand(i: 1);
1188
1189 const APInt *CINT = nullptr;
1190 if (!match(V: opr1, P: m_APIntAllowPoison(Res&: CINT)))
1191 return false;
1192
1193 Function *Parent = B.GetInsertBlock()->getParent();
1194
1195 int ci_opr1 = (int)CINT->getSExtValue();
1196 if (ci_opr1 == 1 && !Parent->hasFnAttribute(Kind: Attribute::StrictFP)) {
1197 // rootn(x, 1) = x
1198 //
1199 // TODO: Insert constrained canonicalize for strictfp case.
1200 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << '\n');
1201 replaceCall(I: FPOp, With: opr0);
1202 return true;
1203 }
1204
1205 Module *M = B.GetInsertBlock()->getModule();
1206
1207 CallInst *CI = cast<CallInst>(Val: FPOp);
1208
1209 // rootn and sqrt disagree on signed-zero / -Inf inputs (e.g. rootn(-0.0, 2)
1210 // is +0.0, sqrt(-0.0) is -0.0), so require nsz/ninf.
1211 bool FMFOkForSqrt = FPOp->hasNoSignedZeros() && FPOp->hasNoInfs();
1212
1213 if (ci_opr1 == 2 && FMFOkForSqrt &&
1214 shouldReplaceLibcallWithIntrinsic(CI,
1215 /*AllowMinSizeF32=*/true,
1216 /*AllowF64=*/true)) {
1217 // rootn(x, 2) = sqrt(x)
1218 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> sqrt(" << *opr0 << ")\n");
1219
1220 Value *NewCall = B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, Op: opr0, FMFSource: CI);
1221 NewCall->takeName(V: CI);
1222
1223 // OpenCL rootn has a looser ulp of 2 requirement than sqrt, so add some
1224 // metadata.
1225 MDBuilder MDHelper(M->getContext());
1226 MDNode *FPMD = MDHelper.createFPMath(Accuracy: std::max(a: FPOp->getFPAccuracy(), b: 2.0f));
1227 if (auto *NewCallI = dyn_cast<Instruction>(Val: NewCall))
1228 NewCallI->setMetadata(KindID: LLVMContext::MD_fpmath, Node: FPMD);
1229
1230 replaceCall(I: CI, With: NewCall);
1231 return true;
1232 }
1233
1234 if (ci_opr1 == 3) { // rootn(x, 3) = cbrt(x)
1235 if (FunctionCallee FPExpr =
1236 getFunction(M, fInfo: AMDGPULibFunc(AMDGPULibFunc::EI_CBRT, FInfo))) {
1237 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> cbrt(" << *opr0
1238 << ")\n");
1239 Value *nval = CreateCallEx(B,Callee: FPExpr, Arg: opr0, Name: "__rootn2cbrt");
1240 replaceCall(I: FPOp, With: nval);
1241 return true;
1242 }
1243 } else if (ci_opr1 == -1) { // rootn(x, -1) = 1.0/x
1244 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1.0 / " << *opr0 << "\n");
1245 Value *nval = B.CreateFDiv(L: ConstantFP::get(Ty: opr0->getType(), V: 1.0),
1246 R: opr0,
1247 Name: "__rootn2div");
1248 replaceCall(I: FPOp, With: nval);
1249 return true;
1250 }
1251
1252 if (ci_opr1 == -2 && FMFOkForSqrt &&
1253 shouldReplaceLibcallWithIntrinsic(CI,
1254 /*AllowMinSizeF32=*/true,
1255 /*AllowF64=*/true)) {
1256 // rootn(x, -2) = rsqrt(x)
1257
1258 // The original rootn had looser ulp requirements than the resultant sqrt
1259 // and fdiv.
1260 MDBuilder MDHelper(M->getContext());
1261 MDNode *FPMD = MDHelper.createFPMath(Accuracy: std::max(a: FPOp->getFPAccuracy(), b: 2.0f));
1262
1263 // TODO: Could handle strictfp but need to fix strict sqrt emission
1264 FastMathFlags FMF = FPOp->getFastMathFlags();
1265 FMF.setAllowContract(true);
1266
1267 Value *Sqrt = B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, Op: opr0, FMFSource: CI);
1268 Instruction *RSqrt = cast<Instruction>(
1269 Val: B.CreateFDiv(L: ConstantFP::get(Ty: opr0->getType(), V: 1.0), R: Sqrt));
1270 if (auto *SqrtI = dyn_cast<Instruction>(Val: Sqrt))
1271 SqrtI->setFastMathFlags(FMF);
1272 RSqrt->setFastMathFlags(FMF);
1273 RSqrt->setMetadata(KindID: LLVMContext::MD_fpmath, Node: FPMD);
1274
1275 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> rsqrt(" << *opr0
1276 << ")\n");
1277 replaceCall(I: CI, With: RSqrt);
1278 return true;
1279 }
1280
1281 return false;
1282}
1283
1284// is_integer(y) => trunc(y) == y
1285static Value *emitIsInteger(IRBuilder<> &B, Value *Y) {
1286 Value *TruncY = B.CreateUnaryIntrinsic(ID: Intrinsic::trunc, Op: Y);
1287 return B.CreateFCmpOEQ(LHS: TruncY, RHS: Y);
1288}
1289
1290static Value *emitIsEvenInteger(IRBuilder<> &B, Value *Y) {
1291 // Even integers are still integers after division by 2.
1292 auto *HalfY = B.CreateFMul(L: Y, R: ConstantFP::get(Ty: Y->getType(), V: 0.5));
1293 return emitIsInteger(B, Y: HalfY);
1294}
1295
1296// is_odd_integer(y) => is_integer(y) && !is_even_integer(y)
1297static Value *emitIsOddInteger(IRBuilder<> &B, Value *Y) {
1298 Value *IsIntY = emitIsInteger(B, Y);
1299 Value *IsEvenY = emitIsEvenInteger(B, Y);
1300 Value *NotEvenY = B.CreateNot(V: IsEvenY);
1301 return B.CreateAnd(LHS: IsIntY, RHS: NotEvenY);
1302}
1303
1304// isinf(val) => fabs(val) == +inf
1305static Value *emitIsInf(IRBuilder<> &B, Value *val) {
1306 auto *fabsVal = B.CreateFAbs(V: val);
1307 return B.CreateFCmpOEQ(LHS: fabsVal, RHS: ConstantFP::getInfinity(Ty: val->getType()));
1308}
1309
1310// y * log2(fabs(x))
1311static Value *emitFastExpYLnx(IRBuilder<> &B, Value *X, Value *Y) {
1312 Value *AbsX = B.CreateFAbs(V: X);
1313 Value *LogAbsX = B.CreateUnaryIntrinsic(ID: Intrinsic::log2, Op: AbsX);
1314 Value *YTimesLogX = B.CreateFMul(L: Y, R: LogAbsX);
1315 return B.CreateUnaryIntrinsic(ID: Intrinsic::exp2, Op: YTimesLogX);
1316}
1317
1318/// Emit special case management epilog code for fast pow, powr, pown, and rootn
1319/// expansions. \p x and \p y should be the arguments to the library call
1320/// (possibly with some values clamped). \p expylnx should be the result to use
1321/// in normal circumstances.
1322static Value *emitPowFixup(IRBuilder<> &B, Value *X, Value *Y, Value *ExpYLnX,
1323 PowKind Kind) {
1324 Constant *Zero = ConstantFP::getZero(Ty: X->getType());
1325 Constant *One = ConstantFP::get(Ty: X->getType(), V: 1.0);
1326 Constant *QNaN = ConstantFP::getQNaN(Ty: X->getType());
1327 Constant *PInf = ConstantFP::getInfinity(Ty: X->getType());
1328
1329 switch (Kind) {
1330 case PowKind::Pow: {
1331 // is_odd_integer(y)
1332 Value *IsOddY = emitIsOddInteger(B, Y);
1333
1334 // ret = copysign(expylnx, is_odd_y ? x : 1.0f)
1335 Value *SelSign = B.CreateSelect(C: IsOddY, True: X, False: One);
1336 Value *Ret = B.CreateCopySign(LHS: ExpYLnX, RHS: SelSign);
1337
1338 // if (x < 0 && !is_integer(y)) ret = QNAN
1339 Value *IsIntY = emitIsInteger(B, Y);
1340 Value *condNegX = B.CreateFCmpOLT(LHS: X, RHS: Zero);
1341 Value *condNotIntY = B.CreateNot(V: IsIntY);
1342 Value *condNaN = B.CreateAnd(LHS: condNegX, RHS: condNotIntY);
1343 Ret = B.CreateSelect(C: condNaN, True: QNaN, False: Ret);
1344
1345 // if (isinf(ay)) { ... }
1346
1347 // FIXME: Missing backend optimization to save on materialization cost of
1348 // mixed sign constant infinities.
1349 Value *YIsInf = emitIsInf(B, val: Y);
1350
1351 Value *AY = B.CreateFAbs(V: Y);
1352 Value *YIsNegInf = B.CreateFCmpUNE(LHS: Y, RHS: AY);
1353
1354 Value *AX = B.CreateFAbs(V: X);
1355 Value *AxEqOne = B.CreateFCmpOEQ(LHS: AX, RHS: One);
1356 Value *AxLtOne = B.CreateFCmpOLT(LHS: AX, RHS: One);
1357 Value *XorCond = B.CreateXor(LHS: AxLtOne, RHS: YIsNegInf);
1358 Value *SelInf =
1359 B.CreateSelect(C: AxEqOne, True: AX, False: B.CreateSelect(C: XorCond, True: Zero, False: AY));
1360 Ret = B.CreateSelect(C: YIsInf, True: SelInf, False: Ret);
1361
1362 // if (isinf(ax) || x == 0.0f) { ... }
1363 Value *XIsInf = emitIsInf(B, val: X);
1364 Value *XEqZero = B.CreateFCmpOEQ(LHS: X, RHS: Zero);
1365 Value *AxInfOrZero = B.CreateOr(LHS: XIsInf, RHS: XEqZero);
1366 Value *YLtZero = B.CreateFCmpOLT(LHS: Y, RHS: Zero);
1367 Value *XorZeroInf = B.CreateXor(LHS: XEqZero, RHS: YLtZero);
1368 Value *SelVal = B.CreateSelect(C: XorZeroInf, True: Zero, False: PInf);
1369 Value *SelSign2 = B.CreateSelect(C: IsOddY, True: X, False: Zero);
1370 Value *Copysign = B.CreateCopySign(LHS: SelVal, RHS: SelSign2);
1371 Ret = B.CreateSelect(C: AxInfOrZero, True: Copysign, False: Ret);
1372
1373 // if (isunordered(x, y)) ret = QNAN
1374 Value *isUnordered = B.CreateFCmpUNO(LHS: X, RHS: Y);
1375 return B.CreateSelect(C: isUnordered, True: QNaN, False: Ret);
1376 }
1377 case PowKind::PowR: {
1378 Value *YIsNeg = B.CreateFCmpOLT(LHS: Y, RHS: Zero);
1379 Value *IZ = B.CreateSelect(C: YIsNeg, True: PInf, False: Zero);
1380 Value *ZI = B.CreateSelect(C: YIsNeg, True: Zero, False: PInf);
1381
1382 Value *YEqZero = B.CreateFCmpOEQ(LHS: Y, RHS: Zero);
1383 Value *SelZeroCase = B.CreateSelect(C: YEqZero, True: QNaN, False: IZ);
1384 Value *XEqZero = B.CreateFCmpOEQ(LHS: X, RHS: Zero);
1385 Value *Ret = B.CreateSelect(C: XEqZero, True: SelZeroCase, False: ExpYLnX);
1386
1387 Value *XEqInf = B.CreateFCmpOEQ(LHS: X, RHS: PInf);
1388 Value *YNeZero = B.CreateFCmpUNE(LHS: Y, RHS: Zero);
1389 Value *CondInfCase = B.CreateAnd(LHS: XEqInf, RHS: YNeZero);
1390 Ret = B.CreateSelect(C: CondInfCase, True: ZI, False: Ret);
1391
1392 Value *IsInfY = emitIsInf(B, val: Y);
1393 Value *XNeOne = B.CreateFCmpUNE(LHS: X, RHS: One);
1394 Value *CondInfY = B.CreateAnd(LHS: IsInfY, RHS: XNeOne);
1395 Value *XLtOne = B.CreateFCmpOLT(LHS: X, RHS: One);
1396 Value *SelInfYCase = B.CreateSelect(C: XLtOne, True: IZ, False: ZI);
1397 Ret = B.CreateSelect(C: CondInfY, True: SelInfYCase, False: Ret);
1398
1399 Value *IsUnordered = B.CreateFCmpUNO(LHS: X, RHS: Y);
1400 return B.CreateSelect(C: IsUnordered, True: QNaN, False: Ret);
1401 }
1402 case PowKind::PowN: {
1403 Constant *ZeroI = ConstantInt::get(Ty: Y->getType(), V: 0);
1404
1405 // is_odd_y = (ny & 1) != 0
1406 Value *OneI = ConstantInt::get(Ty: Y->getType(), V: 1);
1407 Value *YAnd1 = B.CreateAnd(LHS: Y, RHS: OneI);
1408 Value *IsOddY = B.CreateICmpNE(LHS: YAnd1, RHS: ZeroI);
1409
1410 // ret = copysign(expylnx, is_odd_y ? x : 1.0f)
1411 Value *SelSign = B.CreateSelect(C: IsOddY, True: X, False: One);
1412 Value *Ret = B.CreateCopySign(LHS: ExpYLnX, RHS: SelSign);
1413
1414 // if (isinf(x) || x == 0.0f)
1415 Value *FabsX = B.CreateFAbs(V: X);
1416 Value *XIsInf = B.CreateFCmpOEQ(LHS: FabsX, RHS: PInf);
1417 Value *XEqZero = B.CreateFCmpOEQ(LHS: X, RHS: Zero);
1418 Value *InfOrZero = B.CreateOr(LHS: XIsInf, RHS: XEqZero);
1419
1420 // (x == 0.0f) ^ (ny < 0) ? 0.0f : +inf
1421 Value *YLtZero = B.CreateICmpSLT(LHS: Y, RHS: ZeroI);
1422 Value *XorZeroInf = B.CreateXor(LHS: XEqZero, RHS: YLtZero);
1423 Value *SelVal = B.CreateSelect(C: XorZeroInf, True: Zero, False: PInf);
1424
1425 // copysign(selVal, is_odd_y ? x : 0.0f)
1426 Value *SelSign2 = B.CreateSelect(C: IsOddY, True: X, False: Zero);
1427 Value *Copysign = B.CreateCopySign(LHS: SelVal, RHS: SelSign2);
1428
1429 return B.CreateSelect(C: InfOrZero, True: Copysign, False: Ret);
1430 }
1431 case PowKind::RootN: {
1432 Constant *ZeroI = ConstantInt::get(Ty: Y->getType(), V: 0);
1433
1434 // is_odd_y = (ny & 1) != 0
1435 Value *YAnd1 = B.CreateAnd(LHS: Y, RHS: ConstantInt::get(Ty: Y->getType(), V: 1));
1436 Value *IsOddY = B.CreateICmpNE(LHS: YAnd1, RHS: ZeroI);
1437
1438 // ret = copysign(expylnx, is_odd_y ? x : 1.0f)
1439 Value *SelSign = B.CreateSelect(C: IsOddY, True: X, False: One);
1440 Value *Ret = B.CreateCopySign(LHS: ExpYLnX, RHS: SelSign);
1441
1442 // if (isinf(x) || x == 0.0f)
1443 Value *FabsX = B.CreateFAbs(V: X);
1444 Value *IsInfX = B.CreateFCmpOEQ(LHS: FabsX, RHS: PInf);
1445 Value *XEqZero = B.CreateFCmpOEQ(LHS: X, RHS: Zero);
1446 Value *CondInfOrZero = B.CreateOr(LHS: IsInfX, RHS: XEqZero);
1447
1448 // (x == 0.0f) ^ (ny < 0) ? 0.0f : +inf
1449 Value *YLtZero = B.CreateICmpSLT(LHS: Y, RHS: ZeroI);
1450 Value *XorZeroInf = B.CreateXor(LHS: XEqZero, RHS: YLtZero);
1451 Value *SelVal = B.CreateSelect(C: XorZeroInf, True: Zero, False: PInf);
1452
1453 // copysign(selVal, is_odd_y ? x : 0.0f)
1454 Value *SelSign2 = B.CreateSelect(C: IsOddY, True: X, False: Zero);
1455 Value *Copysign = B.CreateCopySign(LHS: SelVal, RHS: SelSign2);
1456
1457 Ret = B.CreateSelect(C: CondInfOrZero, True: Copysign, False: Ret);
1458
1459 // if ((x < 0.0f && !is_odd_y) || ny == 0) ret = QNAN
1460 Value *XIsNeg = B.CreateFCmpOLT(LHS: X, RHS: Zero);
1461 Value *NotOddY = B.CreateNot(V: IsOddY);
1462 Value *CondNegAndNotOdd = B.CreateAnd(LHS: XIsNeg, RHS: NotOddY);
1463 Value *YEqZero = B.CreateICmpEQ(LHS: Y, RHS: ZeroI);
1464 Value *CondBad = B.CreateOr(LHS: CondNegAndNotOdd, RHS: YEqZero);
1465 return B.CreateSelect(C: CondBad, True: QNaN, False: Ret);
1466 }
1467 }
1468
1469 llvm_unreachable("covered switch");
1470}
1471
1472// TODO: Move the fold_pow folding to sqrt/fdiv here
1473bool AMDGPULibCalls::expandFastPow(FPMathOperator *FPOp, IRBuilder<> &B,
1474 PowKind Kind) {
1475 Type *Ty = FPOp->getType();
1476
1477 // There's currently no reason to do this for half. The correct path is
1478 // promote to float and use the fast float expansion.
1479 //
1480 // TODO: We could move this expansion to lowering to get half pow to work.
1481 if (!Ty->getScalarType()->isFloatTy())
1482 return false;
1483
1484 // TODO: Verify optimization for double and bfloat.
1485 Value *X = FPOp->getOperand(i: 0);
1486 Value *Y = FPOp->getOperand(i: 1);
1487
1488 switch (Kind) {
1489 case PowKind::Pow: {
1490 Constant *One = ConstantFP::get(Ty: X->getType(), V: 1.0);
1491
1492 // if (x == 1.0f) y = 1.0f;
1493 Value *XEqOne = B.CreateFCmpOEQ(LHS: X, RHS: One);
1494 Y = B.CreateSelect(C: XEqOne, True: One, False: Y);
1495
1496 // if (y == 0.0f) x = 1.0f;
1497 Value *YEqZero = B.CreateFCmpOEQ(LHS: Y, RHS: ConstantFP::getZero(Ty: X->getType()));
1498 X = B.CreateSelect(C: YEqZero, True: One, False: X);
1499
1500 Value *ExpYLnX = emitFastExpYLnx(B, X, Y);
1501 Value *Fixed = emitPowFixup(B, X, Y, ExpYLnX, Kind);
1502 replaceCall(I: FPOp, With: Fixed);
1503 return true;
1504 }
1505 case PowKind::PowR: {
1506 Value *NegX = B.CreateFCmpOLT(LHS: X, RHS: ConstantFP::getZero(Ty: X->getType()));
1507 X = B.CreateSelect(C: NegX, True: ConstantFP::getQNaN(Ty: X->getType()), False: X);
1508
1509 Value *ExpYLnX = emitFastExpYLnx(B, X, Y);
1510 Value *Fixed = emitPowFixup(B, X, Y, ExpYLnX, Kind);
1511 replaceCall(I: FPOp, With: Fixed);
1512 return true;
1513 }
1514 case PowKind::PowN: {
1515 // ny == 0
1516 Value *YEqZero = B.CreateICmpEQ(LHS: Y, RHS: ConstantInt::get(Ty: Y->getType(), V: 0));
1517
1518 // x = (ny == 0 ? 1.0f : x)
1519 X = B.CreateSelect(C: YEqZero, True: ConstantFP::get(Ty: X->getType(), V: 1.0), False: X);
1520
1521 Value *CastY = B.CreateSIToFP(V: Y, DestTy: X->getType());
1522 Value *ExpYLnX = emitFastExpYLnx(B, X, Y: CastY);
1523 Value *Fixed = emitPowFixup(B, X, Y, ExpYLnX, Kind);
1524 replaceCall(I: FPOp, With: Fixed);
1525 return true;
1526 }
1527 case PowKind::RootN: {
1528 Value *CastY = B.CreateSIToFP(V: Y, DestTy: X->getType());
1529
1530 // This is afn anyway, so we will turn into rcp.
1531 Value *RcpY = B.CreateFDiv(L: ConstantFP::get(Ty: X->getType(), V: 1.0), R: CastY);
1532
1533 Value *ExpYLnX = emitFastExpYLnx(B, X, Y: RcpY);
1534 Value *Fixed = emitPowFixup(B, X, Y, ExpYLnX, Kind);
1535 replaceCall(I: FPOp, With: Fixed);
1536 return true;
1537 }
1538 }
1539 llvm_unreachable("Unhandled PowKind enum");
1540}
1541
1542bool AMDGPULibCalls::tryOptimizePow(FPMathOperator *FPOp, IRBuilder<> &B,
1543 const FuncInfo &FInfo) {
1544 FastMathFlags FMF = FPOp->getFastMathFlags();
1545 CallInst *Call = cast<CallInst>(Val: FPOp);
1546 Module *M = Call->getModule();
1547
1548 FuncInfo PowrInfo;
1549 AMDGPULibFunc::EFuncId FastPowrFuncId =
1550 FMF.approxFunc() || FInfo.getId() == AMDGPULibFunc::EI_POW_FAST
1551 ? AMDGPULibFunc::EI_POWR_FAST
1552 : AMDGPULibFunc::EI_NONE;
1553 FunctionCallee PowrFunc = getFloatFastVariant(
1554 M, fInfo: FInfo, newInfo&: PowrInfo, NewFunc: AMDGPULibFunc::EI_POWR, FastVariant: FastPowrFuncId);
1555
1556 // TODO: Prefer fast pown to fast powr, but slow powr to slow pown.
1557
1558 // pow(x, y) -> powr(x, y) for x >= -0.0
1559 // TODO: Account for flags on current call
1560 if (PowrFunc && cannotBeOrderedLessThanZero(V: FPOp->getOperand(i: 0),
1561 SQ: SQ.getWithInstruction(I: Call))) {
1562 Call->setCalledFunction(PowrFunc);
1563 return fold_pow(FPOp, B, FInfo: PowrInfo) || true;
1564 }
1565
1566 // pow(x, y) -> pown(x, y) for known integral y
1567 if (isKnownIntegral(V: FPOp->getOperand(i: 1), SQ: SQ.getWithInstruction(I: Call),
1568 FMF: FPOp->getFastMathFlags())) {
1569 FunctionType *PownType = getPownType(FT: Call->getFunctionType());
1570
1571 FuncInfo PownInfo;
1572 AMDGPULibFunc::EFuncId FastPownFuncId =
1573 FMF.approxFunc() || FInfo.getId() == AMDGPULibFunc::EI_POW_FAST
1574 ? AMDGPULibFunc::EI_POWN_FAST
1575 : AMDGPULibFunc::EI_NONE;
1576 FunctionCallee PownFunc = getFloatFastVariant(
1577 M, fInfo: FInfo, newInfo&: PownInfo, NewFunc: AMDGPULibFunc::EI_POWN, FastVariant: FastPownFuncId);
1578
1579 if (PownFunc) {
1580 // TODO: If the incoming integral value is an sitofp/uitofp, it won't
1581 // fold out without a known range. We can probably take the source
1582 // value directly.
1583 Value *CastedArg =
1584 B.CreateFPToSI(V: FPOp->getOperand(i: 1), DestTy: PownType->getParamType(i: 1));
1585 // Have to drop any nofpclass attributes on the original call site.
1586 Call->removeParamAttrs(
1587 ArgNo: 1, AttrsToRemove: AttributeFuncs::typeIncompatible(Ty: CastedArg->getType(),
1588 AS: Call->getParamAttributes(ArgNo: 1)));
1589 Call->setCalledFunction(PownFunc);
1590 Call->setArgOperand(i: 1, v: CastedArg);
1591 return fold_pow(FPOp, B, FInfo: PownInfo) || true;
1592 }
1593 }
1594
1595 if (fold_pow(FPOp, B, FInfo))
1596 return true;
1597
1598 if (!FMF.approxFunc())
1599 return false;
1600
1601 if (FInfo.getId() == AMDGPULibFunc::EI_POW && FMF.approxFunc() &&
1602 getArgType(FInfo) == AMDGPULibFunc::F32) {
1603 AMDGPULibFunc PowFastInfo(AMDGPULibFunc::EI_POW_FAST, FInfo);
1604 if (FunctionCallee PowFastFunc = getFunction(M, fInfo: PowFastInfo)) {
1605 Call->setCalledFunction(PowFastFunc);
1606 return fold_pow(FPOp, B, FInfo: PowFastInfo) || true;
1607 }
1608 }
1609
1610 return expandFastPow(FPOp, B, Kind: PowKind::Pow);
1611}
1612
1613// Get a scalar native builtin single argument FP function
1614FunctionCallee AMDGPULibCalls::getNativeFunction(Module *M,
1615 const FuncInfo &FInfo) {
1616 if (getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(id: FInfo.getId()))
1617 return nullptr;
1618 FuncInfo nf = FInfo;
1619 nf.setPrefix(AMDGPULibFunc::NATIVE);
1620 return getFunction(M, fInfo: nf);
1621}
1622
1623// Some library calls are just wrappers around llvm intrinsics, but compiled
1624// conservatively. Preserve the flags from the original call site by
1625// substituting them with direct calls with all the flags.
1626bool AMDGPULibCalls::shouldReplaceLibcallWithIntrinsic(const CallInst *CI,
1627 bool AllowMinSizeF32,
1628 bool AllowF64,
1629 bool AllowStrictFP) {
1630 Type *FltTy = CI->getType()->getScalarType();
1631 const bool IsF32 = FltTy->isFloatTy();
1632
1633 // f64 intrinsics aren't implemented for most operations.
1634 if (!IsF32 && !FltTy->isHalfTy() && (!AllowF64 || !FltTy->isDoubleTy()))
1635 return false;
1636
1637 // We're implicitly inlining by replacing the libcall with the intrinsic, so
1638 // don't do it for noinline call sites.
1639 if (CI->isNoInline())
1640 return false;
1641
1642 const Function *ParentF = CI->getFunction();
1643 // TODO: Handle strictfp
1644 if (!AllowStrictFP && ParentF->hasFnAttribute(Kind: Attribute::StrictFP))
1645 return false;
1646
1647 if (IsF32 && !AllowMinSizeF32 && ParentF->hasMinSize())
1648 return false;
1649 return true;
1650}
1651
1652void AMDGPULibCalls::replaceLibCallWithSimpleIntrinsic(IRBuilder<> &B,
1653 CallInst *CI,
1654 Intrinsic::ID IntrID) {
1655 if (CI->arg_size() == 2) {
1656 Value *Arg0 = CI->getArgOperand(i: 0);
1657 Value *Arg1 = CI->getArgOperand(i: 1);
1658 VectorType *Arg0VecTy = dyn_cast<VectorType>(Val: Arg0->getType());
1659 VectorType *Arg1VecTy = dyn_cast<VectorType>(Val: Arg1->getType());
1660 if (Arg0VecTy && !Arg1VecTy) {
1661 Value *SplatRHS = B.CreateVectorSplat(EC: Arg0VecTy->getElementCount(), V: Arg1);
1662 CI->setArgOperand(i: 1, v: SplatRHS);
1663 } else if (!Arg0VecTy && Arg1VecTy) {
1664 Value *SplatLHS = B.CreateVectorSplat(EC: Arg1VecTy->getElementCount(), V: Arg0);
1665 CI->setArgOperand(i: 0, v: SplatLHS);
1666 }
1667 }
1668
1669 CI->setCalledFunction(Intrinsic::getOrInsertDeclaration(
1670 M: CI->getModule(), id: IntrID, OverloadTys: {CI->getType()}));
1671 CI->setCallingConv(CallingConv::C);
1672}
1673
1674bool AMDGPULibCalls::tryReplaceLibcallWithSimpleIntrinsic(
1675 IRBuilder<> &B, CallInst *CI, Intrinsic::ID IntrID, bool AllowMinSizeF32,
1676 bool AllowF64, bool AllowStrictFP) {
1677 if (!shouldReplaceLibcallWithIntrinsic(CI, AllowMinSizeF32, AllowF64,
1678 AllowStrictFP))
1679 return false;
1680 replaceLibCallWithSimpleIntrinsic(B, CI, IntrID);
1681 return true;
1682}
1683
1684std::tuple<Value *, Value *, Value *>
1685AMDGPULibCalls::insertSinCos(Value *Arg, FastMathFlags FMF, IRBuilder<> &B,
1686 FunctionCallee Fsincos) {
1687 DebugLoc DL = B.getCurrentDebugLocation();
1688 Function *F = B.GetInsertBlock()->getParent();
1689 B.SetInsertPointPastAllocas(F);
1690
1691 AllocaInst *Alloc = B.CreateAlloca(Ty: Arg->getType(), ArraySize: nullptr, Name: "__sincos_");
1692
1693 if (Instruction *ArgInst = dyn_cast<Instruction>(Val: Arg)) {
1694 // If the argument is an instruction, it must dominate all uses so put our
1695 // sincos call there. Otherwise, right after the allocas works well enough
1696 // if it's an argument or constant.
1697
1698 B.SetInsertPoint(*ArgInst->getInsertionPointAfterDef());
1699
1700 // SetInsertPoint unwelcomely always tries to set the debug loc.
1701 B.SetCurrentDebugLocation(DL);
1702 }
1703
1704 Type *CosPtrTy = Fsincos.getFunctionType()->getParamType(i: 1);
1705
1706 // The allocaInst allocates the memory in private address space. This need
1707 // to be addrspacecasted to point to the address space of cos pointer type.
1708 // In OpenCL 2.0 this is generic, while in 1.2 that is private.
1709 Value *CastAlloc = B.CreateAddrSpaceCast(V: Alloc, DestTy: CosPtrTy);
1710
1711 CallInst *SinCos = CreateCallEx2(B, Callee: Fsincos, Arg1: Arg, Arg2: CastAlloc);
1712
1713 // TODO: Is it worth trying to preserve the location for the cos calls for the
1714 // load?
1715
1716 LoadInst *LoadCos = B.CreateLoad(Ty: Arg->getType(), Ptr: Alloc);
1717 return {SinCos, LoadCos, SinCos};
1718}
1719
1720// fold sin, cos -> sincos.
1721bool AMDGPULibCalls::fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B,
1722 const FuncInfo &fInfo) {
1723 assert(fInfo.getId() == AMDGPULibFunc::EI_SIN ||
1724 fInfo.getId() == AMDGPULibFunc::EI_COS);
1725
1726 if ((getArgType(FInfo: fInfo) != AMDGPULibFunc::F32 &&
1727 getArgType(FInfo: fInfo) != AMDGPULibFunc::F64) ||
1728 fInfo.getPrefix() != AMDGPULibFunc::NOPFX)
1729 return false;
1730
1731 bool const isSin = fInfo.getId() == AMDGPULibFunc::EI_SIN;
1732
1733 Value *CArgVal = FPOp->getOperand(i: 0);
1734
1735 // TODO: Constant fold the call
1736 if (isa<ConstantData>(Val: CArgVal))
1737 return false;
1738
1739 CallInst *CI = cast<CallInst>(Val: FPOp);
1740
1741 Function *F = B.GetInsertBlock()->getParent();
1742 Module *M = F->getParent();
1743
1744 // Merge the sin and cos. For OpenCL 2.0, there may only be a generic pointer
1745 // implementation. Prefer the private form if available.
1746 AMDGPULibFunc SinCosLibFuncPrivate(AMDGPULibFunc::EI_SINCOS, fInfo);
1747 SinCosLibFuncPrivate.getLeads()[0].PtrKind =
1748 AMDGPULibFunc::getEPtrKindFromAddrSpace(AS: AMDGPUAS::PRIVATE_ADDRESS);
1749
1750 AMDGPULibFunc SinCosLibFuncGeneric(AMDGPULibFunc::EI_SINCOS, fInfo);
1751 SinCosLibFuncGeneric.getLeads()[0].PtrKind =
1752 AMDGPULibFunc::getEPtrKindFromAddrSpace(AS: AMDGPUAS::FLAT_ADDRESS);
1753
1754 FunctionCallee FSinCosPrivate = getFunction(M, fInfo: SinCosLibFuncPrivate);
1755 FunctionCallee FSinCosGeneric = getFunction(M, fInfo: SinCosLibFuncGeneric);
1756 FunctionCallee FSinCos = FSinCosPrivate ? FSinCosPrivate : FSinCosGeneric;
1757 if (!FSinCos)
1758 return false;
1759
1760 SmallVector<CallInst *> SinCalls;
1761 SmallVector<CallInst *> CosCalls;
1762 SmallVector<CallInst *> SinCosCalls;
1763 FuncInfo PartnerInfo(isSin ? AMDGPULibFunc::EI_COS : AMDGPULibFunc::EI_SIN,
1764 fInfo);
1765 const std::string PairName = PartnerInfo.mangle();
1766
1767 StringRef SinName = isSin ? CI->getCalledFunction()->getName() : PairName;
1768 StringRef CosName = isSin ? PairName : CI->getCalledFunction()->getName();
1769 const std::string SinCosPrivateName = SinCosLibFuncPrivate.mangle();
1770 const std::string SinCosGenericName = SinCosLibFuncGeneric.mangle();
1771
1772 // Intersect the two sets of flags.
1773 FastMathFlags FMF = FPOp->getFastMathFlags();
1774 MDNode *FPMath = CI->getMetadata(KindID: LLVMContext::MD_fpmath);
1775
1776 SmallVector<DILocation *> MergeDbgLocs = {CI->getDebugLoc()};
1777
1778 for (User* U : CArgVal->users()) {
1779 CallInst *XI = dyn_cast<CallInst>(Val: U);
1780 if (!XI || XI->getFunction() != F || XI->isNoBuiltin())
1781 continue;
1782
1783 Function *UCallee = XI->getCalledFunction();
1784 if (!UCallee)
1785 continue;
1786
1787 bool Handled = true;
1788
1789 if (UCallee->getName() == SinName)
1790 SinCalls.push_back(Elt: XI);
1791 else if (UCallee->getName() == CosName)
1792 CosCalls.push_back(Elt: XI);
1793 else if (UCallee->getName() == SinCosPrivateName ||
1794 UCallee->getName() == SinCosGenericName)
1795 SinCosCalls.push_back(Elt: XI);
1796 else
1797 Handled = false;
1798
1799 if (Handled) {
1800 MergeDbgLocs.push_back(Elt: XI->getDebugLoc());
1801 auto *OtherOp = cast<FPMathOperator>(Val: XI);
1802 FMF &= OtherOp->getFastMathFlags();
1803 FPMath = MDNode::getMostGenericFPMath(
1804 A: FPMath, B: XI->getMetadata(KindID: LLVMContext::MD_fpmath));
1805 }
1806 }
1807
1808 if (SinCalls.empty() || CosCalls.empty())
1809 return false;
1810
1811 // insertSinCos needs an insertion point after the argument's def.
1812 if (auto *ArgInst = dyn_cast<Instruction>(Val: CArgVal);
1813 ArgInst && !ArgInst->getInsertionPointAfterDef())
1814 return false;
1815
1816 B.setFastMathFlags(FMF);
1817 B.setDefaultFPMathTag(FPMath);
1818 DILocation *DbgLoc = DILocation::getMergedLocations(Locs: MergeDbgLocs);
1819 B.SetCurrentDebugLocation(DbgLoc);
1820
1821 auto [Sin, Cos, SinCos] = insertSinCos(Arg: CArgVal, FMF, B, Fsincos: FSinCos);
1822
1823 auto replaceTrigInsts = [](ArrayRef<CallInst *> Calls, Value *Res) {
1824 for (CallInst *C : Calls)
1825 C->replaceAllUsesWith(V: Res);
1826
1827 // Leave the other dead instructions to avoid clobbering iterators.
1828 };
1829
1830 replaceTrigInsts(SinCalls, Sin);
1831 replaceTrigInsts(CosCalls, Cos);
1832 replaceTrigInsts(SinCosCalls, SinCos);
1833
1834 // It's safe to delete the original now.
1835 CI->eraseFromParent();
1836 return true;
1837}
1838
1839bool AMDGPULibCalls::evaluateScalarMathFunc(const FuncInfo &FInfo,
1840 APFloat &Res0, APFloat &Res1,
1841 Constant *copr0, Constant *copr1) {
1842 // Every function handled below reads its first operand as a floating-point
1843 // value. Refuse anything else, e.g. a poison vector lane: silently treating
1844 // it as 0.0 misfolds the whole call.
1845 ConstantFP *fpopr0 = dyn_cast_or_null<ConstantFP>(Val: copr0);
1846 if (!fpopr0)
1847 return false;
1848
1849 double opr0 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1850 ? fpopr0->getValueAPF().convertToDouble()
1851 : (double)fpopr0->getValueAPF().convertToFloat();
1852
1853 switch (FInfo.getId()) {
1854 default:
1855 return false;
1856
1857 case AMDGPULibFunc::EI_ACOS:
1858 Res0 = APFloat{acos(x: opr0)};
1859 return true;
1860
1861 case AMDGPULibFunc::EI_ACOSH:
1862 // acosh(x) == log(x + sqrt(x*x - 1))
1863 Res0 = APFloat{log(x: opr0 + sqrt(x: opr0 * opr0 - 1.0))};
1864 return true;
1865
1866 case AMDGPULibFunc::EI_ACOSPI:
1867 Res0 = APFloat{acos(x: opr0) / MATH_PI};
1868 return true;
1869
1870 case AMDGPULibFunc::EI_ASIN:
1871 Res0 = APFloat{asin(x: opr0)};
1872 return true;
1873
1874 case AMDGPULibFunc::EI_ASINH:
1875 // asinh(x) == log(x + sqrt(x*x + 1))
1876 Res0 = APFloat{log(x: opr0 + sqrt(x: opr0 * opr0 + 1.0))};
1877 return true;
1878
1879 case AMDGPULibFunc::EI_ASINPI:
1880 Res0 = APFloat{asin(x: opr0) / MATH_PI};
1881 return true;
1882
1883 case AMDGPULibFunc::EI_ATAN:
1884 Res0 = APFloat{atan(x: opr0)};
1885 return true;
1886
1887 case AMDGPULibFunc::EI_ATANH:
1888 // atanh(x) == (log(x+1) - log(x-1))/2;
1889 Res0 = APFloat{(log(x: opr0 + 1.0) - log(x: opr0 - 1.0)) / 2.0};
1890 return true;
1891
1892 case AMDGPULibFunc::EI_ATANPI:
1893 Res0 = APFloat{atan(x: opr0) / MATH_PI};
1894 return true;
1895
1896 case AMDGPULibFunc::EI_CBRT:
1897 Res0 =
1898 APFloat{(opr0 < 0.0) ? -pow(x: -opr0, y: 1.0 / 3.0) : pow(x: opr0, y: 1.0 / 3.0)};
1899 return true;
1900
1901 case AMDGPULibFunc::EI_COS:
1902 Res0 = APFloat{cos(x: opr0)};
1903 return true;
1904
1905 case AMDGPULibFunc::EI_COSH:
1906 Res0 = APFloat{cosh(x: opr0)};
1907 return true;
1908
1909 case AMDGPULibFunc::EI_COSPI:
1910 Res0 = APFloat{cos(MATH_PI * opr0)};
1911 return true;
1912
1913 case AMDGPULibFunc::EI_EXP:
1914 Res0 = APFloat{std::exp(x: opr0)};
1915 return true;
1916
1917 case AMDGPULibFunc::EI_EXP2:
1918 Res0 = APFloat{pow(x: 2.0, y: opr0)};
1919 return true;
1920
1921 case AMDGPULibFunc::EI_EXP10:
1922 Res0 = APFloat{pow(x: 10.0, y: opr0)};
1923 return true;
1924
1925 case AMDGPULibFunc::EI_LOG:
1926 Res0 = APFloat{log(x: opr0)};
1927 return true;
1928
1929 case AMDGPULibFunc::EI_LOG2:
1930 Res0 = APFloat{log(x: opr0) / log(x: 2.0)};
1931 return true;
1932
1933 case AMDGPULibFunc::EI_LOG10:
1934 Res0 = APFloat{log(x: opr0) / log(x: 10.0)};
1935 return true;
1936
1937 case AMDGPULibFunc::EI_RSQRT:
1938 Res0 = APFloat{1.0 / sqrt(x: opr0)};
1939 return true;
1940
1941 case AMDGPULibFunc::EI_SIN:
1942 Res0 = APFloat{sin(x: opr0)};
1943 return true;
1944
1945 case AMDGPULibFunc::EI_SINH:
1946 Res0 = APFloat{sinh(x: opr0)};
1947 return true;
1948
1949 case AMDGPULibFunc::EI_SINPI:
1950 Res0 = APFloat{sin(MATH_PI * opr0)};
1951 return true;
1952
1953 case AMDGPULibFunc::EI_TAN:
1954 Res0 = APFloat{tan(x: opr0)};
1955 return true;
1956
1957 case AMDGPULibFunc::EI_TANH:
1958 Res0 = APFloat{tanh(x: opr0)};
1959 return true;
1960
1961 case AMDGPULibFunc::EI_TANPI:
1962 Res0 = APFloat{tan(MATH_PI * opr0)};
1963 return true;
1964
1965 // two-arg functions
1966 case AMDGPULibFunc::EI_POW:
1967 case AMDGPULibFunc::EI_POWR: {
1968 ConstantFP *fpopr1 = dyn_cast_or_null<ConstantFP>(Val: copr1);
1969 if (!fpopr1)
1970 return false;
1971 double opr1 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1972 ? fpopr1->getValueAPF().convertToDouble()
1973 : (double)fpopr1->getValueAPF().convertToFloat();
1974 Res0 = APFloat{pow(x: opr0, y: opr1)};
1975 return true;
1976 }
1977
1978 case AMDGPULibFunc::EI_POWN: {
1979 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(Val: copr1)) {
1980 double val = (double)iopr1->getSExtValue();
1981 Res0 = APFloat{pow(x: opr0, y: val)};
1982 return true;
1983 }
1984 return false;
1985 }
1986
1987 case AMDGPULibFunc::EI_ROOTN: {
1988 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(Val: copr1)) {
1989 double val = (double)iopr1->getSExtValue();
1990 Res0 = APFloat{pow(x: opr0, y: 1.0 / val)};
1991 return true;
1992 }
1993 return false;
1994 }
1995
1996 // with ptr arg
1997 case AMDGPULibFunc::EI_SINCOS:
1998 Res0 = APFloat{sin(x: opr0)};
1999 Res1 = APFloat{cos(x: opr0)};
2000 return true;
2001 }
2002
2003 return false;
2004}
2005
2006bool AMDGPULibCalls::evaluateCall(CallInst *aCI, const FuncInfo &FInfo) {
2007 int numArgs = (int)aCI->arg_size();
2008 if (numArgs > 3)
2009 return false;
2010
2011 Constant *copr0 = nullptr;
2012 Constant *copr1 = nullptr;
2013 if (numArgs > 0) {
2014 if ((copr0 = dyn_cast<Constant>(Val: aCI->getArgOperand(i: 0))) == nullptr)
2015 return false;
2016 }
2017
2018 if (numArgs > 1) {
2019 if ((copr1 = dyn_cast<Constant>(Val: aCI->getArgOperand(i: 1))) == nullptr) {
2020 if (FInfo.getId() != AMDGPULibFunc::EI_SINCOS)
2021 return false;
2022 }
2023 }
2024
2025 // At this point, all arguments to aCI are constants.
2026
2027 // max vector size is 16, and sincos will generate two results.
2028 SmallVector<APFloat, 16> Val0, Val1;
2029 int FuncVecSize = getVecSize(FInfo);
2030 if (FuncVecSize == 1) {
2031 if (!evaluateScalarMathFunc(FInfo, Res0&: Val0.emplace_back(Args: 0.0),
2032 Res1&: Val1.emplace_back(Args: 0.0), copr0, copr1)) {
2033 return false;
2034 }
2035 } else {
2036 // An operand of a vector variant is not necessarily a vector: sincos takes
2037 // a pointer as its second operand, and fmin/fmax/ldexp accept an
2038 // implicitly splatted scalar. Only index into actual vectors.
2039 Constant *CV0 = copr0 && copr0->getType()->isVectorTy() ? copr0 : nullptr;
2040 Constant *CV1 = copr1 && copr1->getType()->isVectorTy() ? copr1 : nullptr;
2041 for (int i = 0; i < FuncVecSize; ++i) {
2042 Constant *celt0 = CV0 ? CV0->getAggregateElement(Elt: (unsigned)i) : nullptr;
2043 Constant *celt1 = CV1 ? CV1->getAggregateElement(Elt: (unsigned)i) : nullptr;
2044 if (!evaluateScalarMathFunc(FInfo, Res0&: Val0.emplace_back(Args: 0.0),
2045 Res1&: Val1.emplace_back(Args: 0.0), copr0: celt0, copr1: celt1)) {
2046 return false;
2047 }
2048 }
2049 }
2050
2051 Constant *nval0 = getConstantFloat(Values: Val0, Ty: aCI->getType());
2052
2053 // sincos
2054 if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS) {
2055 Constant *nval1 = getConstantFloat(Values: Val1, Ty: aCI->getType());
2056 new StoreInst(nval1, aCI->getArgOperand(i: 1), aCI->getIterator());
2057 }
2058
2059 replaceCall(I: aCI, With: nval0);
2060 return true;
2061}
2062
2063PreservedAnalyses AMDGPUSimplifyLibCallsPass::run(Function &F,
2064 FunctionAnalysisManager &AM) {
2065 AMDGPULibCalls Simplifier(F, AM);
2066 Simplifier.initNativeFuncs();
2067
2068 bool Changed = false;
2069
2070 LLVM_DEBUG(dbgs() << "AMDIC: process function ";
2071 F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';);
2072
2073 for (auto &BB : F) {
2074 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) {
2075 // Ignore non-calls.
2076 CallInst *CI = dyn_cast<CallInst>(Val&: I);
2077 ++I;
2078
2079 if (CI) {
2080 if (Simplifier.fold(CI))
2081 Changed = true;
2082 }
2083 }
2084 }
2085 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
2086}
2087
2088PreservedAnalyses AMDGPUUseNativeCallsPass::run(Function &F,
2089 FunctionAnalysisManager &AM) {
2090 if (UseNative.empty())
2091 return PreservedAnalyses::all();
2092
2093 AMDGPULibCalls Simplifier(F, AM);
2094 Simplifier.initNativeFuncs();
2095
2096 bool Changed = false;
2097 for (auto &BB : F) {
2098 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) {
2099 // Ignore non-calls.
2100 CallInst *CI = dyn_cast<CallInst>(Val&: I);
2101 ++I;
2102 if (CI && Simplifier.useNative(aCI: CI))
2103 Changed = true;
2104 }
2105 }
2106 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
2107}
2108