1//===- AMDGPULibCalls.cpp -------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file does AMD library function optimizations.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "AMDGPULibFunc.h"
16#include "llvm/Analysis/AssumptionCache.h"
17#include "llvm/Analysis/TargetLibraryInfo.h"
18#include "llvm/Analysis/ValueTracking.h"
19#include "llvm/IR/AttributeMask.h"
20#include "llvm/IR/Dominators.h"
21#include "llvm/IR/IRBuilder.h"
22#include "llvm/IR/IntrinsicsAMDGPU.h"
23#include "llvm/IR/MDBuilder.h"
24#include "llvm/IR/PatternMatch.h"
25#include <cmath>
26
27#define DEBUG_TYPE "amdgpu-simplifylib"
28
29using namespace llvm;
30using namespace llvm::PatternMatch;
31
32static cl::opt<bool> EnablePreLink("amdgpu-prelink",
33 cl::desc("Enable pre-link mode optimizations"),
34 cl::init(Val: false),
35 cl::Hidden);
36
37static cl::list<std::string> UseNative("amdgpu-use-native",
38 cl::desc("Comma separated list of functions to replace with native, or all"),
39 cl::CommaSeparated, cl::ValueOptional,
40 cl::Hidden);
41
42#define MATH_PI numbers::pi
43#define MATH_E numbers::e
44#define MATH_SQRT2 numbers::sqrt2
45#define MATH_SQRT1_2 numbers::inv_sqrt2
46
47enum class PowKind { Pow, PowR, PowN, RootN };
48
49namespace llvm {
50
51class AMDGPULibCalls {
52private:
53 SimplifyQuery SQ;
54
55 using FuncInfo = llvm::AMDGPULibFunc;
56
57 // -fuse-native.
58 bool AllNative = false;
59
60 bool useNativeFunc(const StringRef F) const;
61
62 // Return a pointer (pointer expr) to the function if function definition with
63 // "FuncName" exists. It may create a new function prototype in pre-link mode.
64 FunctionCallee getFunction(Module *M, const FuncInfo &fInfo);
65
66 /// Wrapper around getFunction which tries to use a faster variant if
67 /// available, and falls back to a less fast option.
68 ///
69 /// Return a replacement function for \p fInfo that has float-typed fast
70 /// variants. \p NewFunc is a base replacement function to use. \p
71 /// NewFuncFastVariant is a faster version to use if the calling context knows
72 /// it's legal. If there is no fast variant to use, \p NewFuncFastVariant
73 /// should be EI_NONE.
74 FunctionCallee getFloatFastVariant(Module *M, const FuncInfo &fInfo,
75 FuncInfo &newInfo,
76 AMDGPULibFunc::EFuncId NewFunc,
77 AMDGPULibFunc::EFuncId NewFuncFastVariant);
78
79 bool parseFunctionName(const StringRef &FMangledName, FuncInfo &FInfo);
80
81 bool TDOFold(CallInst *CI, const FuncInfo &FInfo);
82
83 /* Specialized optimizations */
84
85 // pow/powr/pown
86 bool fold_pow(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo);
87
88 /// Peform a fast math expansion of pow, powr, pown or rootn.
89 bool expandFastPow(FPMathOperator *FPOp, IRBuilder<> &B, PowKind Kind);
90
91 bool tryOptimizePow(FPMathOperator *FPOp, IRBuilder<> &B,
92 const FuncInfo &FInfo);
93
94 // rootn
95 bool fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo);
96
97 // -fuse-native for sincos
98 bool sincosUseNative(CallInst *aCI, const FuncInfo &FInfo);
99
100 // evaluate calls if calls' arguments are constants.
101 bool evaluateScalarMathFunc(const FuncInfo &FInfo, APFloat &Res0,
102 APFloat &Res1, Constant *copr0, Constant *copr1);
103 bool evaluateCall(CallInst *aCI, const FuncInfo &FInfo);
104
105 /// Insert a value to sincos function \p Fsincos. Returns (value of sin, value
106 /// of cos, sincos call).
107 std::tuple<Value *, Value *, Value *> insertSinCos(Value *Arg,
108 FastMathFlags FMF,
109 IRBuilder<> &B,
110 FunctionCallee Fsincos);
111
112 // sin/cos
113 bool fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo);
114
115 // __read_pipe/__write_pipe
116 bool fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,
117 const FuncInfo &FInfo);
118
119 // Get a scalar native builtin single argument FP function
120 FunctionCallee getNativeFunction(Module *M, const FuncInfo &FInfo);
121
122 /// Substitute a call to a known libcall with an intrinsic call. If \p
123 /// AllowMinSize is true, allow the replacement in a minsize function.
124 bool shouldReplaceLibcallWithIntrinsic(const CallInst *CI,
125 bool AllowMinSizeF32 = false,
126 bool AllowF64 = false,
127 bool AllowStrictFP = false);
128 void replaceLibCallWithSimpleIntrinsic(IRBuilder<> &B, CallInst *CI,
129 Intrinsic::ID IntrID);
130
131 bool tryReplaceLibcallWithSimpleIntrinsic(IRBuilder<> &B, CallInst *CI,
132 Intrinsic::ID IntrID,
133 bool AllowMinSizeF32 = false,
134 bool AllowF64 = false,
135 bool AllowStrictFP = false);
136
137protected:
138 bool isUnsafeFiniteOnlyMath(const FPMathOperator *FPOp) const;
139
140 bool canIncreasePrecisionOfConstantFold(const FPMathOperator *FPOp) const;
141
142 static void replaceCall(Instruction *I, Value *With) {
143 I->replaceAllUsesWith(V: With);
144 I->eraseFromParent();
145 }
146
147 static void replaceCall(FPMathOperator *I, Value *With) {
148 replaceCall(I: cast<Instruction>(Val: I), With);
149 }
150
151public:
152 AMDGPULibCalls(Function &F, FunctionAnalysisManager &FAM);
153
154 bool fold(CallInst *CI);
155
156 void initNativeFuncs();
157
158 // Replace a normal math function call with that native version
159 bool useNative(CallInst *CI);
160};
161
162} // end namespace llvm
163
164template <typename IRB>
165static CallInst *CreateCallEx(IRB &B, FunctionCallee Callee, Value *Arg,
166 const Twine &Name = "") {
167 CallInst *R = B.CreateCall(Callee, Arg, Name);
168 if (Function *F = dyn_cast<Function>(Val: Callee.getCallee()))
169 R->setCallingConv(F->getCallingConv());
170 return R;
171}
172
173template <typename IRB>
174static CallInst *CreateCallEx2(IRB &B, FunctionCallee Callee, Value *Arg1,
175 Value *Arg2, const Twine &Name = "") {
176 CallInst *R = B.CreateCall(Callee, {Arg1, Arg2}, Name);
177 if (Function *F = dyn_cast<Function>(Val: Callee.getCallee()))
178 R->setCallingConv(F->getCallingConv());
179 return R;
180}
181
182static FunctionType *getPownType(FunctionType *FT) {
183 Type *PowNExpTy = Type::getInt32Ty(C&: FT->getContext());
184 if (VectorType *VecTy = dyn_cast<VectorType>(Val: FT->getReturnType()))
185 PowNExpTy = VectorType::get(ElementType: PowNExpTy, EC: VecTy->getElementCount());
186
187 return FunctionType::get(Result: FT->getReturnType(),
188 Params: {FT->getParamType(i: 0), PowNExpTy}, isVarArg: false);
189}
190
191// Data structures for table-driven optimizations.
192// FuncTbl works for both f32 and f64 functions with 1 input argument
193
194struct TableEntry {
195 double result;
196 double input;
197};
198
199/* a list of {result, input} */
200static const TableEntry tbl_acos[] = {
201 {MATH_PI / 2.0, .input: 0.0},
202 {MATH_PI / 2.0, .input: -0.0},
203 {.result: 0.0, .input: 1.0},
204 {MATH_PI, .input: -1.0}
205};
206static const TableEntry tbl_acosh[] = {
207 {.result: 0.0, .input: 1.0}
208};
209static const TableEntry tbl_acospi[] = {
210 {.result: 0.5, .input: 0.0},
211 {.result: 0.5, .input: -0.0},
212 {.result: 0.0, .input: 1.0},
213 {.result: 1.0, .input: -1.0}
214};
215static const TableEntry tbl_asin[] = {
216 {.result: 0.0, .input: 0.0},
217 {.result: -0.0, .input: -0.0},
218 {MATH_PI / 2.0, .input: 1.0},
219 {.result: -MATH_PI / 2.0, .input: -1.0}
220};
221static const TableEntry tbl_asinh[] = {
222 {.result: 0.0, .input: 0.0},
223 {.result: -0.0, .input: -0.0}
224};
225static const TableEntry tbl_asinpi[] = {
226 {.result: 0.0, .input: 0.0},
227 {.result: -0.0, .input: -0.0},
228 {.result: 0.5, .input: 1.0},
229 {.result: -0.5, .input: -1.0}
230};
231static const TableEntry tbl_atan[] = {
232 {.result: 0.0, .input: 0.0},
233 {.result: -0.0, .input: -0.0},
234 {MATH_PI / 4.0, .input: 1.0},
235 {.result: -MATH_PI / 4.0, .input: -1.0}
236};
237static const TableEntry tbl_atanh[] = {
238 {.result: 0.0, .input: 0.0},
239 {.result: -0.0, .input: -0.0}
240};
241static const TableEntry tbl_atanpi[] = {
242 {.result: 0.0, .input: 0.0},
243 {.result: -0.0, .input: -0.0},
244 {.result: 0.25, .input: 1.0},
245 {.result: -0.25, .input: -1.0}
246};
247static const TableEntry tbl_cbrt[] = {
248 {.result: 0.0, .input: 0.0},
249 {.result: -0.0, .input: -0.0},
250 {.result: 1.0, .input: 1.0},
251 {.result: -1.0, .input: -1.0},
252};
253static const TableEntry tbl_cos[] = {
254 {.result: 1.0, .input: 0.0},
255 {.result: 1.0, .input: -0.0}
256};
257static const TableEntry tbl_cosh[] = {
258 {.result: 1.0, .input: 0.0},
259 {.result: 1.0, .input: -0.0}
260};
261static const TableEntry tbl_cospi[] = {
262 {.result: 1.0, .input: 0.0},
263 {.result: 1.0, .input: -0.0}
264};
265static const TableEntry tbl_erfc[] = {
266 {.result: 1.0, .input: 0.0},
267 {.result: 1.0, .input: -0.0}
268};
269static const TableEntry tbl_erf[] = {
270 {.result: 0.0, .input: 0.0},
271 {.result: -0.0, .input: -0.0}
272};
273static const TableEntry tbl_exp[] = {
274 {.result: 1.0, .input: 0.0},
275 {.result: 1.0, .input: -0.0},
276 {MATH_E, .input: 1.0}
277};
278static const TableEntry tbl_exp2[] = {
279 {.result: 1.0, .input: 0.0},
280 {.result: 1.0, .input: -0.0},
281 {.result: 2.0, .input: 1.0}
282};
283static const TableEntry tbl_exp10[] = {
284 {.result: 1.0, .input: 0.0},
285 {.result: 1.0, .input: -0.0},
286 {.result: 10.0, .input: 1.0}
287};
288static const TableEntry tbl_expm1[] = {
289 {.result: 0.0, .input: 0.0},
290 {.result: -0.0, .input: -0.0}
291};
292static const TableEntry tbl_log[] = {
293 {.result: 0.0, .input: 1.0},
294 {.result: 1.0, MATH_E}
295};
296static const TableEntry tbl_log2[] = {
297 {.result: 0.0, .input: 1.0},
298 {.result: 1.0, .input: 2.0}
299};
300static const TableEntry tbl_log10[] = {
301 {.result: 0.0, .input: 1.0},
302 {.result: 1.0, .input: 10.0}
303};
304static const TableEntry tbl_rsqrt[] = {
305 {.result: 1.0, .input: 1.0},
306 {MATH_SQRT1_2, .input: 2.0}
307};
308static const TableEntry tbl_sin[] = {
309 {.result: 0.0, .input: 0.0},
310 {.result: -0.0, .input: -0.0}
311};
312static const TableEntry tbl_sinh[] = {
313 {.result: 0.0, .input: 0.0},
314 {.result: -0.0, .input: -0.0}
315};
316static const TableEntry tbl_sinpi[] = {
317 {.result: 0.0, .input: 0.0},
318 {.result: -0.0, .input: -0.0}
319};
320static const TableEntry tbl_sqrt[] = {
321 {.result: 0.0, .input: 0.0},
322 {.result: 1.0, .input: 1.0},
323 {MATH_SQRT2, .input: 2.0}
324};
325static const TableEntry tbl_tan[] = {
326 {.result: 0.0, .input: 0.0},
327 {.result: -0.0, .input: -0.0}
328};
329static const TableEntry tbl_tanh[] = {
330 {.result: 0.0, .input: 0.0},
331 {.result: -0.0, .input: -0.0}
332};
333static const TableEntry tbl_tanpi[] = {
334 {.result: 0.0, .input: 0.0},
335 {.result: -0.0, .input: -0.0}
336};
337static const TableEntry tbl_tgamma[] = {
338 {.result: 1.0, .input: 1.0},
339 {.result: 1.0, .input: 2.0},
340 {.result: 2.0, .input: 3.0},
341 {.result: 6.0, .input: 4.0}
342};
343
344static bool HasNative(AMDGPULibFunc::EFuncId id) {
345 switch(id) {
346 case AMDGPULibFunc::EI_DIVIDE:
347 case AMDGPULibFunc::EI_COS:
348 case AMDGPULibFunc::EI_EXP:
349 case AMDGPULibFunc::EI_EXP2:
350 case AMDGPULibFunc::EI_EXP10:
351 case AMDGPULibFunc::EI_LOG:
352 case AMDGPULibFunc::EI_LOG2:
353 case AMDGPULibFunc::EI_LOG10:
354 case AMDGPULibFunc::EI_POWR:
355 case AMDGPULibFunc::EI_RECIP:
356 case AMDGPULibFunc::EI_RSQRT:
357 case AMDGPULibFunc::EI_SIN:
358 case AMDGPULibFunc::EI_SINCOS:
359 case AMDGPULibFunc::EI_SQRT:
360 case AMDGPULibFunc::EI_TAN:
361 return true;
362 default:;
363 }
364 return false;
365}
366
367using TableRef = ArrayRef<TableEntry>;
368
369static TableRef getOptTable(AMDGPULibFunc::EFuncId id) {
370 switch(id) {
371 case AMDGPULibFunc::EI_ACOS: return TableRef(tbl_acos);
372 case AMDGPULibFunc::EI_ACOSH: return TableRef(tbl_acosh);
373 case AMDGPULibFunc::EI_ACOSPI: return TableRef(tbl_acospi);
374 case AMDGPULibFunc::EI_ASIN: return TableRef(tbl_asin);
375 case AMDGPULibFunc::EI_ASINH: return TableRef(tbl_asinh);
376 case AMDGPULibFunc::EI_ASINPI: return TableRef(tbl_asinpi);
377 case AMDGPULibFunc::EI_ATAN: return TableRef(tbl_atan);
378 case AMDGPULibFunc::EI_ATANH: return TableRef(tbl_atanh);
379 case AMDGPULibFunc::EI_ATANPI: return TableRef(tbl_atanpi);
380 case AMDGPULibFunc::EI_CBRT: return TableRef(tbl_cbrt);
381 case AMDGPULibFunc::EI_NCOS:
382 case AMDGPULibFunc::EI_COS: return TableRef(tbl_cos);
383 case AMDGPULibFunc::EI_COSH: return TableRef(tbl_cosh);
384 case AMDGPULibFunc::EI_COSPI: return TableRef(tbl_cospi);
385 case AMDGPULibFunc::EI_ERFC: return TableRef(tbl_erfc);
386 case AMDGPULibFunc::EI_ERF: return TableRef(tbl_erf);
387 case AMDGPULibFunc::EI_EXP: return TableRef(tbl_exp);
388 case AMDGPULibFunc::EI_NEXP2:
389 case AMDGPULibFunc::EI_EXP2: return TableRef(tbl_exp2);
390 case AMDGPULibFunc::EI_EXP10: return TableRef(tbl_exp10);
391 case AMDGPULibFunc::EI_EXPM1: return TableRef(tbl_expm1);
392 case AMDGPULibFunc::EI_LOG: return TableRef(tbl_log);
393 case AMDGPULibFunc::EI_NLOG2:
394 case AMDGPULibFunc::EI_LOG2: return TableRef(tbl_log2);
395 case AMDGPULibFunc::EI_LOG10: return TableRef(tbl_log10);
396 case AMDGPULibFunc::EI_NRSQRT:
397 case AMDGPULibFunc::EI_RSQRT: return TableRef(tbl_rsqrt);
398 case AMDGPULibFunc::EI_NSIN:
399 case AMDGPULibFunc::EI_SIN: return TableRef(tbl_sin);
400 case AMDGPULibFunc::EI_SINH: return TableRef(tbl_sinh);
401 case AMDGPULibFunc::EI_SINPI: return TableRef(tbl_sinpi);
402 case AMDGPULibFunc::EI_NSQRT:
403 case AMDGPULibFunc::EI_SQRT: return TableRef(tbl_sqrt);
404 case AMDGPULibFunc::EI_TAN: return TableRef(tbl_tan);
405 case AMDGPULibFunc::EI_TANH: return TableRef(tbl_tanh);
406 case AMDGPULibFunc::EI_TANPI: return TableRef(tbl_tanpi);
407 case AMDGPULibFunc::EI_TGAMMA: return TableRef(tbl_tgamma);
408 default:;
409 }
410 return TableRef();
411}
412
413static inline int getVecSize(const AMDGPULibFunc& FInfo) {
414 return FInfo.getLeads()[0].VectorSize;
415}
416
417static inline AMDGPULibFunc::EType getArgType(const AMDGPULibFunc& FInfo) {
418 return (AMDGPULibFunc::EType)FInfo.getLeads()[0].ArgType;
419}
420
421FunctionCallee AMDGPULibCalls::getFunction(Module *M, const FuncInfo &fInfo) {
422 // If we are doing PreLinkOpt, the function is external. So it is safe to
423 // use getOrInsertFunction() at this stage.
424
425 return EnablePreLink ? AMDGPULibFunc::getOrInsertFunction(M, fInfo)
426 : AMDGPULibFunc::getFunction(M, fInfo);
427}
428
429FunctionCallee AMDGPULibCalls::getFloatFastVariant(
430 Module *M, const FuncInfo &fInfo, FuncInfo &newInfo,
431 AMDGPULibFunc::EFuncId NewFunc, AMDGPULibFunc::EFuncId FastVariant) {
432 assert(NewFunc != FastVariant);
433
434 if (FastVariant != AMDGPULibFunc::EI_NONE &&
435 getArgType(FInfo: fInfo) == AMDGPULibFunc::F32) {
436 newInfo = AMDGPULibFunc(FastVariant, fInfo);
437 if (FunctionCallee NewCallee = getFunction(M, fInfo: newInfo))
438 return NewCallee;
439 }
440
441 newInfo = AMDGPULibFunc(NewFunc, fInfo);
442 return getFunction(M, fInfo: newInfo);
443}
444
445bool AMDGPULibCalls::parseFunctionName(const StringRef &FMangledName,
446 FuncInfo &FInfo) {
447 return AMDGPULibFunc::parse(MangledName: FMangledName, Ptr&: FInfo);
448}
449
450bool AMDGPULibCalls::isUnsafeFiniteOnlyMath(const FPMathOperator *FPOp) const {
451 return FPOp->hasApproxFunc() && FPOp->hasNoNaNs() && FPOp->hasNoInfs();
452}
453
454bool AMDGPULibCalls::canIncreasePrecisionOfConstantFold(
455 const FPMathOperator *FPOp) const {
456 // TODO: Refine to approxFunc or contract
457 return FPOp->isFast();
458}
459
460AMDGPULibCalls::AMDGPULibCalls(Function &F, FunctionAnalysisManager &FAM)
461 : SQ(F.getParent()->getDataLayout(),
462 &FAM.getResult<TargetLibraryAnalysis>(IR&: F),
463 FAM.getCachedResult<DominatorTreeAnalysis>(IR&: F),
464 &FAM.getResult<AssumptionAnalysis>(IR&: F)) {}
465
466bool AMDGPULibCalls::useNativeFunc(const StringRef F) const {
467 return AllNative || llvm::is_contained(Range&: UseNative, Element: F);
468}
469
470void AMDGPULibCalls::initNativeFuncs() {
471 AllNative = useNativeFunc(F: "all") ||
472 (UseNative.getNumOccurrences() && UseNative.size() == 1 &&
473 UseNative.begin()->empty());
474}
475
476bool AMDGPULibCalls::sincosUseNative(CallInst *aCI, const FuncInfo &FInfo) {
477 bool native_sin = useNativeFunc(F: "sin");
478 bool native_cos = useNativeFunc(F: "cos");
479
480 if (native_sin && native_cos) {
481 Module *M = aCI->getModule();
482 Value *opr0 = aCI->getArgOperand(i: 0);
483
484 AMDGPULibFunc nf;
485 nf.getLeads()[0].ArgType = FInfo.getLeads()[0].ArgType;
486 nf.getLeads()[0].VectorSize = FInfo.getLeads()[0].VectorSize;
487
488 nf.setPrefix(AMDGPULibFunc::NATIVE);
489 nf.setId(AMDGPULibFunc::EI_SIN);
490 FunctionCallee sinExpr = getFunction(M, fInfo: nf);
491
492 nf.setPrefix(AMDGPULibFunc::NATIVE);
493 nf.setId(AMDGPULibFunc::EI_COS);
494 FunctionCallee cosExpr = getFunction(M, fInfo: nf);
495 if (sinExpr && cosExpr) {
496 Value *sinval =
497 CallInst::Create(Func: sinExpr, Args: opr0, NameStr: "splitsin", InsertBefore: aCI->getIterator());
498 Value *cosval =
499 CallInst::Create(Func: cosExpr, Args: opr0, NameStr: "splitcos", InsertBefore: aCI->getIterator());
500 new StoreInst(cosval, aCI->getArgOperand(i: 1), aCI->getIterator());
501
502 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI
503 << " with native version of sin/cos");
504
505 replaceCall(I: aCI, With: sinval);
506 return true;
507 }
508 }
509 return false;
510}
511
512bool AMDGPULibCalls::useNative(CallInst *aCI) {
513 Function *Callee = aCI->getCalledFunction();
514 if (!Callee || aCI->isNoBuiltin())
515 return false;
516
517 FuncInfo FInfo;
518 if (!parseFunctionName(FMangledName: Callee->getName(), FInfo) || !FInfo.isMangled() ||
519 FInfo.getPrefix() != AMDGPULibFunc::NOPFX ||
520 getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(id: FInfo.getId()) ||
521 !(AllNative || useNativeFunc(F: FInfo.getName()))) {
522 return false;
523 }
524
525 if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS)
526 return sincosUseNative(aCI, FInfo);
527
528 FInfo.setPrefix(AMDGPULibFunc::NATIVE);
529 FunctionCallee F = getFunction(M: aCI->getModule(), fInfo: FInfo);
530 if (!F)
531 return false;
532
533 aCI->setCalledFunction(F);
534 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI
535 << " with native version");
536 return true;
537}
538
539// Clang emits call of __read_pipe_2 or __read_pipe_4 for OpenCL read_pipe
540// builtin, with appended type size and alignment arguments, where 2 or 4
541// indicates the original number of arguments. The library has optimized version
542// of __read_pipe_2/__read_pipe_4 when the type size and alignment has the same
543// power of 2 value. This function transforms __read_pipe_2 to __read_pipe_2_N
544// for such cases where N is the size in bytes of the type (N = 1, 2, 4, 8, ...,
545// 128). The same for __read_pipe_4, write_pipe_2, and write_pipe_4.
546bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,
547 const FuncInfo &FInfo) {
548 auto *Callee = CI->getCalledFunction();
549 if (!Callee->isDeclaration())
550 return false;
551
552 assert(Callee->hasName() && "Invalid read_pipe/write_pipe function");
553 auto *M = Callee->getParent();
554 std::string Name = std::string(Callee->getName());
555 auto NumArg = CI->arg_size();
556 if (NumArg != 4 && NumArg != 6)
557 return false;
558 ConstantInt *PacketSize =
559 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: NumArg - 2));
560 ConstantInt *PacketAlign =
561 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: NumArg - 1));
562 if (!PacketSize || !PacketAlign)
563 return false;
564
565 unsigned Size = PacketSize->getZExtValue();
566 Align Alignment = PacketAlign->getAlignValue();
567 if (Alignment != Size)
568 return false;
569
570 unsigned PtrArgLoc = CI->arg_size() - 3;
571 Value *PtrArg = CI->getArgOperand(i: PtrArgLoc);
572 Type *PtrTy = PtrArg->getType();
573
574 SmallVector<llvm::Type *, 6> ArgTys;
575 for (unsigned I = 0; I != PtrArgLoc; ++I)
576 ArgTys.push_back(Elt: CI->getArgOperand(i: I)->getType());
577 ArgTys.push_back(Elt: PtrTy);
578
579 Name = Name + "_" + std::to_string(val: Size);
580 auto *FTy = FunctionType::get(Result: Callee->getReturnType(),
581 Params: ArrayRef<Type *>(ArgTys), isVarArg: false);
582 AMDGPULibFunc NewLibFunc(Name, FTy);
583 FunctionCallee F = AMDGPULibFunc::getOrInsertFunction(M, fInfo: NewLibFunc);
584 if (!F)
585 return false;
586
587 SmallVector<Value *, 6> Args;
588 for (unsigned I = 0; I != PtrArgLoc; ++I)
589 Args.push_back(Elt: CI->getArgOperand(i: I));
590 Args.push_back(Elt: PtrArg);
591
592 auto *NCI = B.CreateCall(Callee: F, Args);
593 NCI->setAttributes(CI->getAttributes());
594 CI->replaceAllUsesWith(V: NCI);
595 CI->dropAllReferences();
596 CI->eraseFromParent();
597
598 return true;
599}
600
601// This function returns false if no change; return true otherwise.
602bool AMDGPULibCalls::fold(CallInst *CI) {
603 Function *Callee = CI->getCalledFunction();
604 // Ignore indirect calls.
605 if (!Callee || Callee->isIntrinsic() || CI->isNoBuiltin())
606 return false;
607
608 FuncInfo FInfo;
609 if (!parseFunctionName(FMangledName: Callee->getName(), FInfo))
610 return false;
611
612 // Further check the number of arguments to see if they match.
613 // TODO: Check calling convention matches too
614 if (!FInfo.isCompatibleSignature(M: *Callee->getParent(), FuncTy: CI->getFunctionType()))
615 return false;
616
617 LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << '\n');
618
619 if (TDOFold(CI, FInfo))
620 return true;
621
622 IRBuilder<> B(CI);
623 if (CI->isStrictFP())
624 B.setIsFPConstrained(true);
625
626 if (FPMathOperator *FPOp = dyn_cast<FPMathOperator>(Val: CI)) {
627 // Under unsafe-math, evaluate calls if possible.
628 // According to Brian Sumner, we can do this for all f32 function calls
629 // using host's double function calls.
630 if (canIncreasePrecisionOfConstantFold(FPOp) && evaluateCall(aCI: CI, FInfo))
631 return true;
632
633 // Copy fast flags from the original call.
634 FastMathFlags FMF = FPOp->getFastMathFlags();
635 B.setFastMathFlags(FMF);
636
637 // Specialized optimizations for each function call.
638 //
639 // TODO: Handle native functions
640 switch (FInfo.getId()) {
641 case AMDGPULibFunc::EI_EXP:
642 if (FMF.none())
643 return false;
644 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::exp,
645 AllowMinSizeF32: FMF.approxFunc());
646 case AMDGPULibFunc::EI_EXP2:
647 if (FMF.none())
648 return false;
649 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::exp2,
650 AllowMinSizeF32: FMF.approxFunc());
651 case AMDGPULibFunc::EI_LOG:
652 if (FMF.none())
653 return false;
654 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::log,
655 AllowMinSizeF32: FMF.approxFunc());
656 case AMDGPULibFunc::EI_LOG2:
657 if (FMF.none())
658 return false;
659 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::log2,
660 AllowMinSizeF32: FMF.approxFunc());
661 case AMDGPULibFunc::EI_LOG10:
662 if (FMF.none())
663 return false;
664 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::log10,
665 AllowMinSizeF32: FMF.approxFunc());
666 case AMDGPULibFunc::EI_FMIN:
667 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::minnum,
668 AllowMinSizeF32: true, AllowF64: true);
669 case AMDGPULibFunc::EI_FMAX:
670 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::maxnum,
671 AllowMinSizeF32: true, AllowF64: true);
672 case AMDGPULibFunc::EI_FMA:
673 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::fma, AllowMinSizeF32: true,
674 AllowF64: true);
675 case AMDGPULibFunc::EI_MAD:
676 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::fmuladd,
677 AllowMinSizeF32: true, AllowF64: true);
678 case AMDGPULibFunc::EI_FABS:
679 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::fabs, AllowMinSizeF32: true,
680 AllowF64: true, AllowStrictFP: true);
681 case AMDGPULibFunc::EI_COPYSIGN:
682 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::copysign,
683 AllowMinSizeF32: true, AllowF64: true, AllowStrictFP: true);
684 case AMDGPULibFunc::EI_FLOOR:
685 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::floor, AllowMinSizeF32: true,
686 AllowF64: true);
687 case AMDGPULibFunc::EI_CEIL:
688 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::ceil, AllowMinSizeF32: true,
689 AllowF64: true);
690 case AMDGPULibFunc::EI_TRUNC:
691 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::trunc, AllowMinSizeF32: true,
692 AllowF64: true);
693 case AMDGPULibFunc::EI_RINT:
694 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::rint, AllowMinSizeF32: true,
695 AllowF64: true);
696 case AMDGPULibFunc::EI_ROUND:
697 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, IntrID: Intrinsic::round, AllowMinSizeF32: true,
698 AllowF64: true);
699 case AMDGPULibFunc::EI_LDEXP: {
700 if (!shouldReplaceLibcallWithIntrinsic(CI, AllowMinSizeF32: true, AllowF64: true))
701 return false;
702
703 Value *Arg1 = CI->getArgOperand(i: 1);
704 if (VectorType *VecTy = dyn_cast<VectorType>(Val: CI->getType());
705 VecTy && !isa<VectorType>(Val: Arg1->getType())) {
706 Value *SplatArg1 = B.CreateVectorSplat(EC: VecTy->getElementCount(), V: Arg1);
707 CI->setArgOperand(i: 1, v: SplatArg1);
708 }
709
710 CI->setCalledFunction(Intrinsic::getOrInsertDeclaration(
711 M: CI->getModule(), id: Intrinsic::ldexp,
712 OverloadTys: {CI->getType(), CI->getArgOperand(i: 1)->getType()}));
713 CI->setCallingConv(CallingConv::C);
714 return true;
715 }
716 case AMDGPULibFunc::EI_POW:
717 case AMDGPULibFunc::EI_POW_FAST:
718 return tryOptimizePow(FPOp, B, FInfo);
719 case AMDGPULibFunc::EI_POWR:
720 case AMDGPULibFunc::EI_POWR_FAST: {
721 if (fold_pow(FPOp, B, FInfo))
722 return true;
723 if (!FMF.approxFunc())
724 return false;
725
726 if (FInfo.getId() == AMDGPULibFunc::EI_POWR && FMF.approxFunc() &&
727 getArgType(FInfo) == AMDGPULibFunc::F32) {
728 Module *M = Callee->getParent();
729 AMDGPULibFunc PowrFastInfo(AMDGPULibFunc::EI_POWR_FAST, FInfo);
730 if (FunctionCallee PowrFastFunc = getFunction(M, fInfo: PowrFastInfo)) {
731 CI->setCalledFunction(PowrFastFunc);
732 return true;
733 }
734 }
735
736 if (!shouldReplaceLibcallWithIntrinsic(CI))
737 return false;
738 return expandFastPow(FPOp, B, Kind: PowKind::PowR);
739 }
740 case AMDGPULibFunc::EI_POWN:
741 case AMDGPULibFunc::EI_POWN_FAST: {
742 if (fold_pow(FPOp, B, FInfo))
743 return true;
744 if (!FMF.approxFunc())
745 return false;
746
747 if (FInfo.getId() == AMDGPULibFunc::EI_POWN &&
748 getArgType(FInfo) == AMDGPULibFunc::F32) {
749 Module *M = Callee->getParent();
750 AMDGPULibFunc PownFastInfo(AMDGPULibFunc::EI_POWN_FAST, FInfo);
751 if (FunctionCallee PownFastFunc = getFunction(M, fInfo: PownFastInfo)) {
752 CI->setCalledFunction(PownFastFunc);
753 return true;
754 }
755 }
756
757 if (!shouldReplaceLibcallWithIntrinsic(CI))
758 return false;
759 return expandFastPow(FPOp, B, Kind: PowKind::PowN);
760 }
761 case AMDGPULibFunc::EI_ROOTN:
762 case AMDGPULibFunc::EI_ROOTN_FAST: {
763 if (fold_rootn(FPOp, B, FInfo))
764 return true;
765 if (!FMF.approxFunc())
766 return false;
767
768 if (getArgType(FInfo) == AMDGPULibFunc::F32) {
769 Module *M = Callee->getParent();
770 AMDGPULibFunc RootnFastInfo(AMDGPULibFunc::EI_ROOTN_FAST, FInfo);
771 if (FunctionCallee RootnFastFunc = getFunction(M, fInfo: RootnFastInfo)) {
772 CI->setCalledFunction(RootnFastFunc);
773 return true;
774 }
775 }
776
777 return expandFastPow(FPOp, B, Kind: PowKind::RootN);
778 }
779 case AMDGPULibFunc::EI_SQRT:
780 // TODO: Allow with strictfp + constrained intrinsic
781 return tryReplaceLibcallWithSimpleIntrinsic(
782 B, CI, IntrID: Intrinsic::sqrt, AllowMinSizeF32: true, AllowF64: true, /*AllowStrictFP=*/false);
783 case AMDGPULibFunc::EI_COS:
784 case AMDGPULibFunc::EI_SIN:
785 return fold_sincos(FPOp, B, FInfo);
786 default:
787 break;
788 }
789 } else {
790 // Specialized optimizations for each function call
791 switch (FInfo.getId()) {
792 case AMDGPULibFunc::EI_READ_PIPE_2:
793 case AMDGPULibFunc::EI_READ_PIPE_4:
794 case AMDGPULibFunc::EI_WRITE_PIPE_2:
795 case AMDGPULibFunc::EI_WRITE_PIPE_4:
796 return fold_read_write_pipe(CI, B, FInfo);
797 default:
798 break;
799 }
800 }
801
802 return false;
803}
804
805static Constant *getConstantFloat(const ArrayRef<APFloat> Values,
806 const Type *Ty) {
807
808 assert(Ty->isSingleValueType() &&
809 "Type must either be a scalar or a vector.");
810 assert((!Ty->isVectorTy() || Ty->isScalableTy() ||
811 Values.size() == cast<FixedVectorType>(Ty)->getNumElements()) &&
812 "Unexpected number of constant values.");
813 assert((Ty->isVectorTy() || Values.size() == 1) &&
814 "Expected exactly one constant value");
815
816 Type *ElemTy = Ty->getScalarType();
817 const fltSemantics &FltSem = ElemTy->getFltSemantics();
818
819 SmallVector<Constant *, 4> ConstValues;
820 ConstValues.reserve(N: Values.size());
821 for (APFloat APF : Values) {
822 bool Unused;
823 APF.convert(ToSemantics: FltSem, RM: APFloat::rmNearestTiesToEven, losesInfo: &Unused);
824 ConstValues.push_back(Elt: ConstantFP::get(Ty: ElemTy, V: APF));
825 }
826
827 return Ty->isVectorTy() ? ConstantVector::get(V: ConstValues) : ConstValues[0];
828}
829
830bool AMDGPULibCalls::TDOFold(CallInst *CI, const FuncInfo &FInfo) {
831 // Table-Driven optimization
832 const TableRef tr = getOptTable(id: FInfo.getId());
833 if (tr.empty())
834 return false;
835
836 int const sz = (int)tr.size();
837 Value *opr0 = CI->getArgOperand(i: 0);
838
839 int vecSize = getVecSize(FInfo);
840 if (vecSize > 1) {
841 // Vector version
842 Constant *CV = dyn_cast<Constant>(Val: opr0);
843 if (CV && CV->getType()->isVectorTy()) {
844 SmallVector<APFloat, 4> Values;
845 Values.reserve(N: vecSize);
846 for (int eltNo = 0; eltNo < vecSize; ++eltNo) {
847 // A lane may be undef or poison, in which case there is nothing to
848 // look up in the table.
849 ConstantFP *eltval = dyn_cast_or_null<ConstantFP>(
850 Val: CV->getAggregateElement(Elt: (unsigned)eltNo));
851 if (!eltval)
852 return false;
853 auto MatchingRow = llvm::find_if(Range: tr, P: [eltval](const TableEntry &entry) {
854 return eltval->isExactlyValue(V: entry.input);
855 });
856 if (MatchingRow == tr.end())
857 return false;
858 Values.push_back(Elt: APFloat(MatchingRow->result));
859 }
860 Constant *NewValues = getConstantFloat(Values, Ty: CI->getType());
861 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *NewValues << "\n");
862 replaceCall(I: CI, With: NewValues);
863 return true;
864 }
865 } else {
866 // Scalar version
867 if (ConstantFP *CF = dyn_cast<ConstantFP>(Val: opr0)) {
868 for (int i = 0; i < sz; ++i) {
869 if (CF->isExactlyValue(V: tr[i].input)) {
870 Value *nval = ConstantFP::get(Ty: CF->getType(), V: tr[i].result);
871 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");
872 replaceCall(I: CI, With: nval);
873 return true;
874 }
875 }
876 }
877 }
878
879 return false;
880}
881
882namespace llvm {
883static double log2(double V) {
884#if _XOPEN_SOURCE >= 600 || defined(_ISOC99_SOURCE) || _POSIX_C_SOURCE >= 200112L
885 return ::log2(x: V);
886#else
887 return log(V) / numbers::ln2;
888#endif
889}
890} // namespace llvm
891
892bool AMDGPULibCalls::fold_pow(FPMathOperator *FPOp, IRBuilder<> &B,
893 const FuncInfo &FInfo) {
894 assert((FInfo.getId() == AMDGPULibFunc::EI_POW ||
895 FInfo.getId() == AMDGPULibFunc::EI_POW_FAST ||
896 FInfo.getId() == AMDGPULibFunc::EI_POWR ||
897 FInfo.getId() == AMDGPULibFunc::EI_POWR_FAST ||
898 FInfo.getId() == AMDGPULibFunc::EI_POWN ||
899 FInfo.getId() == AMDGPULibFunc::EI_POWN_FAST) &&
900 "fold_pow: encounter a wrong function call");
901
902 Module *M = B.GetInsertBlock()->getModule();
903 Type *eltType = FPOp->getType()->getScalarType();
904 Value *opr0 = FPOp->getOperand(i: 0);
905 Value *opr1 = FPOp->getOperand(i: 1);
906
907 const APFloat *CF = nullptr;
908 const APInt *CINT = nullptr;
909 if (!match(V: opr1, P: m_APFloatAllowPoison(Res&: CF)))
910 match(V: opr1, P: m_APIntAllowPoison(Res&: CINT));
911
912 // 0x1111111 means that we don't do anything for this call.
913 int ci_opr1 = (CINT ? (int)CINT->getSExtValue() : 0x1111111);
914
915 // OpenCL powr(x<0, y) = NaN, but the folds below would turn it into a
916 // finite number. Skip them unless NaNs are ignored or the base is known
917 // non-negative.
918 bool IsPowr = FInfo.getId() == AMDGPULibFunc::EI_POWR ||
919 FInfo.getId() == AMDGPULibFunc::EI_POWR_FAST;
920 bool SkipConstantFolds =
921 IsPowr && !FPOp->hasNoNaNs() &&
922 !cannotBeOrderedLessThanZero(
923 V: opr0, SQ: SQ.getWithInstruction(I: cast<Instruction>(Val: FPOp)));
924
925 if (CF && (CF->isExactlyValue(V: 0.5) || CF->isExactlyValue(V: -0.5))) {
926 // pow[r](x, [-]0.5) = sqrt(x) / rsqrt(x)
927 //
928 // sqrt/rsqrt and pow disagree on two negative inputs:
929 // pow(-Inf, 0.5) == +Inf but sqrt(-Inf) == NaN (ninf case)
930 // pow(-0.0, 0.5) == +0.0 but sqrt(-0.0) == -0.0 (nsz case)
931 // powr requires x >= 0 by the OpenCL spec, so -Inf is undefined behaviour
932 // and the ninf check can be skipped for powr/powr_fast. -0.0 is a valid
933 // input for powr since -0.0 >= 0 by IEEE comparison, so nsz is still
934 // required for all variants. sqrt/rsqrt already return NaN for a
935 // negative base like powr does, so this fold skips the base-sign check.
936 if (FPOp->hasNoSignedZeros() && (IsPowr || FPOp->hasNoInfs())) {
937 bool issqrt = CF->isExactlyValue(V: 0.5);
938 if (FunctionCallee FPExpr =
939 getFunction(M, fInfo: AMDGPULibFunc(issqrt ? AMDGPULibFunc::EI_SQRT
940 : AMDGPULibFunc::EI_RSQRT,
941 FInfo))) {
942 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << FInfo.getName()
943 << '(' << *opr0 << ")\n");
944 Value *nval = CreateCallEx(B, Callee: FPExpr, Arg: opr0,
945 Name: issqrt ? "__pow2sqrt" : "__pow2rsqrt");
946 replaceCall(I: FPOp, With: nval);
947 return true;
948 }
949 }
950 }
951
952 if (!SkipConstantFolds) {
953 if ((CF && CF->isZero()) || (CINT && ci_opr1 == 0)) {
954 // pow/powr/pown(x, 0) == 1
955 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1\n");
956 Constant *cnval = ConstantFP::get(Ty: eltType, V: 1.0);
957 if (getVecSize(FInfo) > 1) {
958 cnval = ConstantDataVector::getSplat(NumElts: getVecSize(FInfo), Elt: cnval);
959 }
960 replaceCall(I: FPOp, With: cnval);
961 return true;
962 }
963 if ((CF && CF->isOne()) || (CINT && ci_opr1 == 1)) {
964 // pow/powr/pown(x, 1.0) = x
965 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << "\n");
966 replaceCall(I: FPOp, With: opr0);
967 return true;
968 }
969 if ((CF && CF->isExactlyValue(V: 2.0)) || (CINT && ci_opr1 == 2)) {
970 // pow/powr/pown(x, 2.0) = x*x
971 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << " * "
972 << *opr0 << "\n");
973 Value *nval = B.CreateFMul(L: opr0, R: opr0, Name: "__pow2");
974 replaceCall(I: FPOp, With: nval);
975 return true;
976 }
977 if ((CF && CF->isMinusOne()) || (CINT && ci_opr1 == -1)) {
978 // pow/powr/pown(x, -1.0) = 1.0/x
979 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1 / " << *opr0 << "\n");
980 Constant *cnval = ConstantFP::get(Ty: eltType, V: 1.0);
981 if (getVecSize(FInfo) > 1) {
982 cnval = ConstantDataVector::getSplat(NumElts: getVecSize(FInfo), Elt: cnval);
983 }
984 Value *nval = B.CreateFDiv(L: cnval, R: opr0, Name: "__powrecip");
985 replaceCall(I: FPOp, With: nval);
986 return true;
987 }
988 }
989
990 if (!isUnsafeFiniteOnlyMath(FPOp))
991 return false;
992
993 // Unsafe Math optimization
994
995 // Remember that ci_opr1 is set if opr1 is integral
996 if (CF) {
997 double dval = (getArgType(FInfo) == AMDGPULibFunc::F32)
998 ? (double)CF->convertToFloat()
999 : CF->convertToDouble();
1000 int ival = (int)dval;
1001 if ((double)ival == dval) {
1002 ci_opr1 = ival;
1003 } else
1004 ci_opr1 = 0x11111111;
1005 }
1006
1007 // pow/powr/pown(x, c) = [1/](x*x*..x); where
1008 // trunc(c) == c && the number of x == c && |c| <= 12
1009 unsigned abs_opr1 = (ci_opr1 < 0) ? -ci_opr1 : ci_opr1;
1010 if (abs_opr1 <= 12) {
1011 Constant *cnval;
1012 Value *nval;
1013 if (abs_opr1 == 0) {
1014 cnval = ConstantFP::get(Ty: eltType, V: 1.0);
1015 if (getVecSize(FInfo) > 1) {
1016 cnval = ConstantDataVector::getSplat(NumElts: getVecSize(FInfo), Elt: cnval);
1017 }
1018 nval = cnval;
1019 } else {
1020 Value *valx2 = nullptr;
1021 nval = nullptr;
1022 while (abs_opr1 > 0) {
1023 valx2 = valx2 ? B.CreateFMul(L: valx2, R: valx2, Name: "__powx2") : opr0;
1024 if (abs_opr1 & 1) {
1025 nval = nval ? B.CreateFMul(L: nval, R: valx2, Name: "__powprod") : valx2;
1026 }
1027 abs_opr1 >>= 1;
1028 }
1029 }
1030
1031 if (ci_opr1 < 0) {
1032 cnval = ConstantFP::get(Ty: eltType, V: 1.0);
1033 if (getVecSize(FInfo) > 1) {
1034 cnval = ConstantDataVector::getSplat(NumElts: getVecSize(FInfo), Elt: cnval);
1035 }
1036 nval = B.CreateFDiv(L: cnval, R: nval, Name: "__1powprod");
1037 }
1038 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> "
1039 << ((ci_opr1 < 0) ? "1/prod(" : "prod(") << *opr0
1040 << ")\n");
1041 replaceCall(I: FPOp, With: nval);
1042 return true;
1043 }
1044
1045 // If we should use the generic intrinsic instead of emitting a libcall
1046 const bool ShouldUseIntrinsic = eltType->isFloatTy() || eltType->isHalfTy();
1047
1048 // powr ---> exp2(y * log2(x))
1049 // pown/pow ---> powr(fabs(x), y) | (x & ((int)y << 31))
1050 FunctionCallee ExpExpr;
1051 if (ShouldUseIntrinsic)
1052 ExpExpr = Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::exp2,
1053 OverloadTys: {FPOp->getType()});
1054 else {
1055 ExpExpr = getFunction(M, fInfo: AMDGPULibFunc(AMDGPULibFunc::EI_EXP2, FInfo));
1056 if (!ExpExpr)
1057 return false;
1058 }
1059
1060 bool needlog = false;
1061 bool needabs = false;
1062 bool needcopysign = false;
1063 Constant *cnval = nullptr;
1064 if (getVecSize(FInfo) == 1) {
1065 CF = nullptr;
1066 match(V: opr0, P: m_APFloatAllowPoison(Res&: CF));
1067
1068 if (CF) {
1069 double V = (getArgType(FInfo) == AMDGPULibFunc::F32)
1070 ? (double)CF->convertToFloat()
1071 : CF->convertToDouble();
1072
1073 V = log2(V: std::abs(x: V));
1074 cnval = ConstantFP::get(Ty: eltType, V);
1075 needcopysign = (FInfo.getId() != AMDGPULibFunc::EI_POWR &&
1076 FInfo.getId() != AMDGPULibFunc::EI_POWR_FAST) &&
1077 CF->isNegative();
1078 } else {
1079 needlog = true;
1080 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR &&
1081 FInfo.getId() != AMDGPULibFunc::EI_POWR_FAST;
1082 }
1083 } else {
1084 ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(Val: opr0);
1085
1086 if (!CDV) {
1087 needlog = true;
1088 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR &&
1089 FInfo.getId() != AMDGPULibFunc::EI_POWR_FAST;
1090 } else {
1091 assert ((int)CDV->getNumElements() == getVecSize(FInfo) &&
1092 "Wrong vector size detected");
1093
1094 SmallVector<double, 0> DVal;
1095 for (int i=0; i < getVecSize(FInfo); ++i) {
1096 double V = CDV->getElementAsAPFloat(i).convertToDouble();
1097 if (V < 0.0) needcopysign = true;
1098 V = log2(V: std::abs(x: V));
1099 DVal.push_back(Elt: V);
1100 }
1101 if (getArgType(FInfo) == AMDGPULibFunc::F32) {
1102 SmallVector<float, 0> FVal;
1103 for (double D : DVal)
1104 FVal.push_back(Elt: (float)D);
1105 ArrayRef<float> tmp(FVal);
1106 cnval = ConstantDataVector::get(Context&: M->getContext(), Elts: tmp);
1107 } else {
1108 ArrayRef<double> tmp(DVal);
1109 cnval = ConstantDataVector::get(Context&: M->getContext(), Elts: tmp);
1110 }
1111 }
1112 }
1113
1114 if (needcopysign && (FInfo.getId() == AMDGPULibFunc::EI_POW ||
1115 FInfo.getId() == AMDGPULibFunc::EI_POW_FAST)) {
1116 // We cannot handle corner cases for a general pow() function, give up
1117 // unless y is a constant integral value. Then proceed as if it were pown.
1118 if (!isKnownIntegral(V: opr1, SQ: SQ.getWithInstruction(I: cast<Instruction>(Val: FPOp)),
1119 FMF: FPOp->getFastMathFlags()))
1120 return false;
1121 }
1122
1123 Value *nval;
1124 if (needabs) {
1125 nval = B.CreateFAbs(V: opr0, FMFSource: nullptr, Name: "__fabs");
1126 } else {
1127 nval = cnval ? cnval : opr0;
1128 }
1129 if (needlog) {
1130 FunctionCallee LogExpr;
1131 if (ShouldUseIntrinsic) {
1132 LogExpr = Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::log2,
1133 OverloadTys: {FPOp->getType()});
1134 } else {
1135 LogExpr = getFunction(M, fInfo: AMDGPULibFunc(AMDGPULibFunc::EI_LOG2, FInfo));
1136 if (!LogExpr)
1137 return false;
1138 }
1139
1140 nval = CreateCallEx(B,Callee: LogExpr, Arg: nval, Name: "__log2");
1141 }
1142
1143 if (FInfo.getId() == AMDGPULibFunc::EI_POWN ||
1144 FInfo.getId() == AMDGPULibFunc::EI_POWN_FAST) {
1145 // convert int(32) to fp(f32 or f64)
1146 opr1 = B.CreateSIToFP(V: opr1, DestTy: nval->getType(), Name: "pownI2F");
1147 }
1148 nval = B.CreateFMul(L: opr1, R: nval, Name: "__ylogx");
1149
1150 CallInst *Exp2Call = CreateCallEx(B, Callee: ExpExpr, Arg: nval, Name: "__exp2");
1151
1152 // TODO: Generalized fpclass logic for pow
1153 FPClassTest KnownNot = FPClassTest::fcNegative;
1154 if (FPOp->hasNoNaNs())
1155 KnownNot |= FPClassTest::fcNan;
1156
1157 Exp2Call->addRetAttr(
1158 Attr: Attribute::getWithNoFPClass(Context&: Exp2Call->getContext(), Mask: KnownNot));
1159 nval = Exp2Call;
1160
1161 if (needcopysign) {
1162 Type* nTyS = B.getIntNTy(N: eltType->getPrimitiveSizeInBits());
1163 Type *nTy = FPOp->getType()->getWithNewType(EltTy: nTyS);
1164 Value *opr_n = FPOp->getOperand(i: 1);
1165 if (opr_n->getType()->getScalarType()->isIntegerTy())
1166 opr_n = B.CreateZExtOrTrunc(V: opr_n, DestTy: nTy, Name: "__ytou");
1167 else
1168 opr_n = B.CreateFPToSI(V: opr1, DestTy: nTy, Name: "__ytou");
1169
1170 unsigned size = nTy->getScalarSizeInBits();
1171 Value *sign = B.CreateShl(LHS: opr_n, RHS: size-1, Name: "__yeven");
1172 sign = B.CreateAnd(LHS: B.CreateBitCast(V: opr0, DestTy: nTy), RHS: sign, Name: "__pow_sign");
1173
1174 nval = B.CreateCopySign(LHS: nval, RHS: B.CreateBitCast(V: sign, DestTy: nval->getType()),
1175 FMFSource: nullptr, Name: "__pow_sign");
1176 }
1177
1178 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> "
1179 << "exp2(" << *opr1 << " * log2(" << *opr0 << "))\n");
1180 replaceCall(I: FPOp, With: nval);
1181
1182 return true;
1183}
1184
1185bool AMDGPULibCalls::fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B,
1186 const FuncInfo &FInfo) {
1187 Value *opr0 = FPOp->getOperand(i: 0);
1188 Value *opr1 = FPOp->getOperand(i: 1);
1189
1190 const APInt *CINT = nullptr;
1191 if (!match(V: opr1, P: m_APIntAllowPoison(Res&: CINT)))
1192 return false;
1193
1194 Function *Parent = B.GetInsertBlock()->getParent();
1195
1196 int ci_opr1 = (int)CINT->getSExtValue();
1197 if (ci_opr1 == 1 && !Parent->hasFnAttribute(Kind: Attribute::StrictFP)) {
1198 // rootn(x, 1) = x
1199 //
1200 // TODO: Insert constrained canonicalize for strictfp case.
1201 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << '\n');
1202 replaceCall(I: FPOp, With: opr0);
1203 return true;
1204 }
1205
1206 Module *M = B.GetInsertBlock()->getModule();
1207
1208 CallInst *CI = cast<CallInst>(Val: FPOp);
1209
1210 // rootn and sqrt disagree on signed-zero / -Inf inputs (e.g. rootn(-0.0, 2)
1211 // is +0.0, sqrt(-0.0) is -0.0), so require nsz/ninf.
1212 bool FMFOkForSqrt = FPOp->hasNoSignedZeros() && FPOp->hasNoInfs();
1213
1214 if (ci_opr1 == 2 && FMFOkForSqrt &&
1215 shouldReplaceLibcallWithIntrinsic(CI,
1216 /*AllowMinSizeF32=*/true,
1217 /*AllowF64=*/true)) {
1218 // rootn(x, 2) = sqrt(x)
1219 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> sqrt(" << *opr0 << ")\n");
1220
1221 Value *NewCall = B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, Op: opr0, FMFSource: CI);
1222 NewCall->takeName(V: CI);
1223
1224 // OpenCL rootn has a looser ulp of 2 requirement than sqrt, so add some
1225 // metadata.
1226 MDBuilder MDHelper(M->getContext());
1227 MDNode *FPMD = MDHelper.createFPMath(Accuracy: std::max(a: FPOp->getFPAccuracy(), b: 2.0f));
1228 if (auto *NewCallI = dyn_cast<Instruction>(Val: NewCall))
1229 NewCallI->setMetadata(KindID: LLVMContext::MD_fpmath, Node: FPMD);
1230
1231 replaceCall(I: CI, With: NewCall);
1232 return true;
1233 }
1234
1235 if (ci_opr1 == 3) { // rootn(x, 3) = cbrt(x)
1236 if (FunctionCallee FPExpr =
1237 getFunction(M, fInfo: AMDGPULibFunc(AMDGPULibFunc::EI_CBRT, FInfo))) {
1238 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> cbrt(" << *opr0
1239 << ")\n");
1240 Value *nval = CreateCallEx(B,Callee: FPExpr, Arg: opr0, Name: "__rootn2cbrt");
1241 replaceCall(I: FPOp, With: nval);
1242 return true;
1243 }
1244 } else if (ci_opr1 == -1) { // rootn(x, -1) = 1.0/x
1245 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1.0 / " << *opr0 << "\n");
1246 Value *nval = B.CreateFDiv(L: ConstantFP::get(Ty: opr0->getType(), V: 1.0),
1247 R: opr0,
1248 Name: "__rootn2div");
1249 replaceCall(I: FPOp, With: nval);
1250 return true;
1251 }
1252
1253 if (ci_opr1 == -2 && FMFOkForSqrt &&
1254 shouldReplaceLibcallWithIntrinsic(CI,
1255 /*AllowMinSizeF32=*/true,
1256 /*AllowF64=*/true)) {
1257 // rootn(x, -2) = rsqrt(x)
1258
1259 // The original rootn had looser ulp requirements than the resultant sqrt
1260 // and fdiv.
1261 MDBuilder MDHelper(M->getContext());
1262 MDNode *FPMD = MDHelper.createFPMath(Accuracy: std::max(a: FPOp->getFPAccuracy(), b: 2.0f));
1263
1264 // TODO: Could handle strictfp but need to fix strict sqrt emission
1265 FastMathFlags FMF = FPOp->getFastMathFlags();
1266 FMF.setAllowContract(true);
1267
1268 Value *Sqrt = B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, Op: opr0, FMFSource: CI);
1269 Instruction *RSqrt = cast<Instruction>(
1270 Val: B.CreateFDiv(L: ConstantFP::get(Ty: opr0->getType(), V: 1.0), R: Sqrt));
1271 if (auto *SqrtI = dyn_cast<Instruction>(Val: Sqrt))
1272 SqrtI->setFastMathFlags(FMF);
1273 RSqrt->setFastMathFlags(FMF);
1274 RSqrt->setMetadata(KindID: LLVMContext::MD_fpmath, Node: FPMD);
1275
1276 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> rsqrt(" << *opr0
1277 << ")\n");
1278 replaceCall(I: CI, With: RSqrt);
1279 return true;
1280 }
1281
1282 return false;
1283}
1284
1285// is_integer(y) => trunc(y) == y
1286static Value *emitIsInteger(IRBuilder<> &B, Value *Y) {
1287 Value *TruncY = B.CreateUnaryIntrinsic(ID: Intrinsic::trunc, Op: Y);
1288 return B.CreateFCmpOEQ(LHS: TruncY, RHS: Y);
1289}
1290
1291static Value *emitIsEvenInteger(IRBuilder<> &B, Value *Y) {
1292 // Even integers are still integers after division by 2.
1293 auto *HalfY = B.CreateFMul(L: Y, R: ConstantFP::get(Ty: Y->getType(), V: 0.5));
1294 return emitIsInteger(B, Y: HalfY);
1295}
1296
1297// is_odd_integer(y) => is_integer(y) && !is_even_integer(y)
1298static Value *emitIsOddInteger(IRBuilder<> &B, Value *Y) {
1299 Value *IsIntY = emitIsInteger(B, Y);
1300 Value *IsEvenY = emitIsEvenInteger(B, Y);
1301 Value *NotEvenY = B.CreateNot(V: IsEvenY);
1302 return B.CreateAnd(LHS: IsIntY, RHS: NotEvenY);
1303}
1304
1305// isinf(val) => fabs(val) == +inf
1306static Value *emitIsInf(IRBuilder<> &B, Value *val) {
1307 auto *fabsVal = B.CreateFAbs(V: val);
1308 return B.CreateFCmpOEQ(LHS: fabsVal, RHS: ConstantFP::getInfinity(Ty: val->getType()));
1309}
1310
1311// y * log2(fabs(x))
1312static Value *emitFastExpYLnx(IRBuilder<> &B, Value *X, Value *Y) {
1313 Value *AbsX = B.CreateFAbs(V: X);
1314 Value *LogAbsX = B.CreateUnaryIntrinsic(ID: Intrinsic::log2, Op: AbsX);
1315 Value *YTimesLogX = B.CreateFMul(L: Y, R: LogAbsX);
1316 return B.CreateUnaryIntrinsic(ID: Intrinsic::exp2, Op: YTimesLogX);
1317}
1318
1319/// Emit special case management epilog code for fast pow, powr, pown, and rootn
1320/// expansions. \p x and \p y should be the arguments to the library call
1321/// (possibly with some values clamped). \p expylnx should be the result to use
1322/// in normal circumstances.
1323static Value *emitPowFixup(IRBuilder<> &B, Value *X, Value *Y, Value *ExpYLnX,
1324 PowKind Kind) {
1325 Constant *Zero = ConstantFP::getZero(Ty: X->getType());
1326 Constant *One = ConstantFP::get(Ty: X->getType(), V: 1.0);
1327 Constant *QNaN = ConstantFP::getQNaN(Ty: X->getType());
1328 Constant *PInf = ConstantFP::getInfinity(Ty: X->getType());
1329
1330 switch (Kind) {
1331 case PowKind::Pow: {
1332 // is_odd_integer(y)
1333 Value *IsOddY = emitIsOddInteger(B, Y);
1334
1335 // ret = copysign(expylnx, is_odd_y ? x : 1.0f)
1336 Value *SelSign = B.CreateSelect(C: IsOddY, True: X, False: One);
1337 Value *Ret = B.CreateCopySign(LHS: ExpYLnX, RHS: SelSign);
1338
1339 // if (x < 0 && !is_integer(y)) ret = QNAN
1340 Value *IsIntY = emitIsInteger(B, Y);
1341 Value *condNegX = B.CreateFCmpOLT(LHS: X, RHS: Zero);
1342 Value *condNotIntY = B.CreateNot(V: IsIntY);
1343 Value *condNaN = B.CreateAnd(LHS: condNegX, RHS: condNotIntY);
1344 Ret = B.CreateSelect(C: condNaN, True: QNaN, False: Ret);
1345
1346 // if (isinf(ay)) { ... }
1347
1348 // FIXME: Missing backend optimization to save on materialization cost of
1349 // mixed sign constant infinities.
1350 Value *YIsInf = emitIsInf(B, val: Y);
1351
1352 Value *AY = B.CreateFAbs(V: Y);
1353 Value *YIsNegInf = B.CreateFCmpUNE(LHS: Y, RHS: AY);
1354
1355 Value *AX = B.CreateFAbs(V: X);
1356 Value *AxEqOne = B.CreateFCmpOEQ(LHS: AX, RHS: One);
1357 Value *AxLtOne = B.CreateFCmpOLT(LHS: AX, RHS: One);
1358 Value *XorCond = B.CreateXor(LHS: AxLtOne, RHS: YIsNegInf);
1359 Value *SelInf =
1360 B.CreateSelect(C: AxEqOne, True: AX, False: B.CreateSelect(C: XorCond, True: Zero, False: AY));
1361 Ret = B.CreateSelect(C: YIsInf, True: SelInf, False: Ret);
1362
1363 // if (isinf(ax) || x == 0.0f) { ... }
1364 Value *XIsInf = emitIsInf(B, val: X);
1365 Value *XEqZero = B.CreateFCmpOEQ(LHS: X, RHS: Zero);
1366 Value *AxInfOrZero = B.CreateOr(LHS: XIsInf, RHS: XEqZero);
1367 Value *YLtZero = B.CreateFCmpOLT(LHS: Y, RHS: Zero);
1368 Value *XorZeroInf = B.CreateXor(LHS: XEqZero, RHS: YLtZero);
1369 Value *SelVal = B.CreateSelect(C: XorZeroInf, True: Zero, False: PInf);
1370 Value *SelSign2 = B.CreateSelect(C: IsOddY, True: X, False: Zero);
1371 Value *Copysign = B.CreateCopySign(LHS: SelVal, RHS: SelSign2);
1372 Ret = B.CreateSelect(C: AxInfOrZero, True: Copysign, False: Ret);
1373
1374 // if (isunordered(x, y)) ret = QNAN
1375 Value *isUnordered = B.CreateFCmpUNO(LHS: X, RHS: Y);
1376 return B.CreateSelect(C: isUnordered, True: QNaN, False: Ret);
1377 }
1378 case PowKind::PowR: {
1379 Value *YIsNeg = B.CreateFCmpOLT(LHS: Y, RHS: Zero);
1380 Value *IZ = B.CreateSelect(C: YIsNeg, True: PInf, False: Zero);
1381 Value *ZI = B.CreateSelect(C: YIsNeg, True: Zero, False: PInf);
1382
1383 Value *YEqZero = B.CreateFCmpOEQ(LHS: Y, RHS: Zero);
1384 Value *SelZeroCase = B.CreateSelect(C: YEqZero, True: QNaN, False: IZ);
1385 Value *XEqZero = B.CreateFCmpOEQ(LHS: X, RHS: Zero);
1386 Value *Ret = B.CreateSelect(C: XEqZero, True: SelZeroCase, False: ExpYLnX);
1387
1388 Value *XEqInf = B.CreateFCmpOEQ(LHS: X, RHS: PInf);
1389 Value *YNeZero = B.CreateFCmpUNE(LHS: Y, RHS: Zero);
1390 Value *CondInfCase = B.CreateAnd(LHS: XEqInf, RHS: YNeZero);
1391 Ret = B.CreateSelect(C: CondInfCase, True: ZI, False: Ret);
1392
1393 Value *IsInfY = emitIsInf(B, val: Y);
1394 Value *XNeOne = B.CreateFCmpUNE(LHS: X, RHS: One);
1395 Value *CondInfY = B.CreateAnd(LHS: IsInfY, RHS: XNeOne);
1396 Value *XLtOne = B.CreateFCmpOLT(LHS: X, RHS: One);
1397 Value *SelInfYCase = B.CreateSelect(C: XLtOne, True: IZ, False: ZI);
1398 Ret = B.CreateSelect(C: CondInfY, True: SelInfYCase, False: Ret);
1399
1400 Value *IsUnordered = B.CreateFCmpUNO(LHS: X, RHS: Y);
1401 return B.CreateSelect(C: IsUnordered, True: QNaN, False: Ret);
1402 }
1403 case PowKind::PowN: {
1404 Constant *ZeroI = ConstantInt::get(Ty: Y->getType(), V: 0);
1405
1406 // is_odd_y = (ny & 1) != 0
1407 Value *OneI = ConstantInt::get(Ty: Y->getType(), V: 1);
1408 Value *YAnd1 = B.CreateAnd(LHS: Y, RHS: OneI);
1409 Value *IsOddY = B.CreateICmpNE(LHS: YAnd1, RHS: ZeroI);
1410
1411 // ret = copysign(expylnx, is_odd_y ? x : 1.0f)
1412 Value *SelSign = B.CreateSelect(C: IsOddY, True: X, False: One);
1413 Value *Ret = B.CreateCopySign(LHS: ExpYLnX, RHS: SelSign);
1414
1415 // if (isinf(x) || x == 0.0f)
1416 Value *FabsX = B.CreateFAbs(V: X);
1417 Value *XIsInf = B.CreateFCmpOEQ(LHS: FabsX, RHS: PInf);
1418 Value *XEqZero = B.CreateFCmpOEQ(LHS: X, RHS: Zero);
1419 Value *InfOrZero = B.CreateOr(LHS: XIsInf, RHS: XEqZero);
1420
1421 // (x == 0.0f) ^ (ny < 0) ? 0.0f : +inf
1422 Value *YLtZero = B.CreateICmpSLT(LHS: Y, RHS: ZeroI);
1423 Value *XorZeroInf = B.CreateXor(LHS: XEqZero, RHS: YLtZero);
1424 Value *SelVal = B.CreateSelect(C: XorZeroInf, True: Zero, False: PInf);
1425
1426 // copysign(selVal, is_odd_y ? x : 0.0f)
1427 Value *SelSign2 = B.CreateSelect(C: IsOddY, True: X, False: Zero);
1428 Value *Copysign = B.CreateCopySign(LHS: SelVal, RHS: SelSign2);
1429
1430 return B.CreateSelect(C: InfOrZero, True: Copysign, False: Ret);
1431 }
1432 case PowKind::RootN: {
1433 Constant *ZeroI = ConstantInt::get(Ty: Y->getType(), V: 0);
1434
1435 // is_odd_y = (ny & 1) != 0
1436 Value *YAnd1 = B.CreateAnd(LHS: Y, RHS: ConstantInt::get(Ty: Y->getType(), V: 1));
1437 Value *IsOddY = B.CreateICmpNE(LHS: YAnd1, RHS: ZeroI);
1438
1439 // ret = copysign(expylnx, is_odd_y ? x : 1.0f)
1440 Value *SelSign = B.CreateSelect(C: IsOddY, True: X, False: One);
1441 Value *Ret = B.CreateCopySign(LHS: ExpYLnX, RHS: SelSign);
1442
1443 // if (isinf(x) || x == 0.0f)
1444 Value *FabsX = B.CreateFAbs(V: X);
1445 Value *IsInfX = B.CreateFCmpOEQ(LHS: FabsX, RHS: PInf);
1446 Value *XEqZero = B.CreateFCmpOEQ(LHS: X, RHS: Zero);
1447 Value *CondInfOrZero = B.CreateOr(LHS: IsInfX, RHS: XEqZero);
1448
1449 // (x == 0.0f) ^ (ny < 0) ? 0.0f : +inf
1450 Value *YLtZero = B.CreateICmpSLT(LHS: Y, RHS: ZeroI);
1451 Value *XorZeroInf = B.CreateXor(LHS: XEqZero, RHS: YLtZero);
1452 Value *SelVal = B.CreateSelect(C: XorZeroInf, True: Zero, False: PInf);
1453
1454 // copysign(selVal, is_odd_y ? x : 0.0f)
1455 Value *SelSign2 = B.CreateSelect(C: IsOddY, True: X, False: Zero);
1456 Value *Copysign = B.CreateCopySign(LHS: SelVal, RHS: SelSign2);
1457
1458 Ret = B.CreateSelect(C: CondInfOrZero, True: Copysign, False: Ret);
1459
1460 // if ((x < 0.0f && !is_odd_y) || ny == 0) ret = QNAN
1461 Value *XIsNeg = B.CreateFCmpOLT(LHS: X, RHS: Zero);
1462 Value *NotOddY = B.CreateNot(V: IsOddY);
1463 Value *CondNegAndNotOdd = B.CreateAnd(LHS: XIsNeg, RHS: NotOddY);
1464 Value *YEqZero = B.CreateICmpEQ(LHS: Y, RHS: ZeroI);
1465 Value *CondBad = B.CreateOr(LHS: CondNegAndNotOdd, RHS: YEqZero);
1466 return B.CreateSelect(C: CondBad, True: QNaN, False: Ret);
1467 }
1468 }
1469
1470 llvm_unreachable("covered switch");
1471}
1472
1473// TODO: Move the fold_pow folding to sqrt/fdiv here
1474bool AMDGPULibCalls::expandFastPow(FPMathOperator *FPOp, IRBuilder<> &B,
1475 PowKind Kind) {
1476 Type *Ty = FPOp->getType();
1477
1478 // There's currently no reason to do this for half. The correct path is
1479 // promote to float and use the fast float expansion.
1480 //
1481 // TODO: We could move this expansion to lowering to get half pow to work.
1482 if (!Ty->getScalarType()->isFloatTy())
1483 return false;
1484
1485 // TODO: Verify optimization for double and bfloat.
1486 Value *X = FPOp->getOperand(i: 0);
1487 Value *Y = FPOp->getOperand(i: 1);
1488
1489 switch (Kind) {
1490 case PowKind::Pow: {
1491 Constant *One = ConstantFP::get(Ty: X->getType(), V: 1.0);
1492
1493 // if (x == 1.0f) y = 1.0f;
1494 Value *XEqOne = B.CreateFCmpOEQ(LHS: X, RHS: One);
1495 Y = B.CreateSelect(C: XEqOne, True: One, False: Y);
1496
1497 // if (y == 0.0f) x = 1.0f;
1498 Value *YEqZero = B.CreateFCmpOEQ(LHS: Y, RHS: ConstantFP::getZero(Ty: X->getType()));
1499 X = B.CreateSelect(C: YEqZero, True: One, False: X);
1500
1501 Value *ExpYLnX = emitFastExpYLnx(B, X, Y);
1502 Value *Fixed = emitPowFixup(B, X, Y, ExpYLnX, Kind);
1503 replaceCall(I: FPOp, With: Fixed);
1504 return true;
1505 }
1506 case PowKind::PowR: {
1507 Value *NegX = B.CreateFCmpOLT(LHS: X, RHS: ConstantFP::getZero(Ty: X->getType()));
1508 X = B.CreateSelect(C: NegX, True: ConstantFP::getQNaN(Ty: X->getType()), False: X);
1509
1510 Value *ExpYLnX = emitFastExpYLnx(B, X, Y);
1511 Value *Fixed = emitPowFixup(B, X, Y, ExpYLnX, Kind);
1512 replaceCall(I: FPOp, With: Fixed);
1513 return true;
1514 }
1515 case PowKind::PowN: {
1516 // ny == 0
1517 Value *YEqZero = B.CreateICmpEQ(LHS: Y, RHS: ConstantInt::get(Ty: Y->getType(), V: 0));
1518
1519 // x = (ny == 0 ? 1.0f : x)
1520 X = B.CreateSelect(C: YEqZero, True: ConstantFP::get(Ty: X->getType(), V: 1.0), False: X);
1521
1522 Value *CastY = B.CreateSIToFP(V: Y, DestTy: X->getType());
1523 Value *ExpYLnX = emitFastExpYLnx(B, X, Y: CastY);
1524 Value *Fixed = emitPowFixup(B, X, Y, ExpYLnX, Kind);
1525 replaceCall(I: FPOp, With: Fixed);
1526 return true;
1527 }
1528 case PowKind::RootN: {
1529 Value *CastY = B.CreateSIToFP(V: Y, DestTy: X->getType());
1530
1531 // This is afn anyway, so we will turn into rcp.
1532 Value *RcpY = B.CreateFDiv(L: ConstantFP::get(Ty: X->getType(), V: 1.0), R: CastY);
1533
1534 Value *ExpYLnX = emitFastExpYLnx(B, X, Y: RcpY);
1535 Value *Fixed = emitPowFixup(B, X, Y, ExpYLnX, Kind);
1536 replaceCall(I: FPOp, With: Fixed);
1537 return true;
1538 }
1539 }
1540 llvm_unreachable("Unhandled PowKind enum");
1541}
1542
1543bool AMDGPULibCalls::tryOptimizePow(FPMathOperator *FPOp, IRBuilder<> &B,
1544 const FuncInfo &FInfo) {
1545 FastMathFlags FMF = FPOp->getFastMathFlags();
1546 CallInst *Call = cast<CallInst>(Val: FPOp);
1547 Module *M = Call->getModule();
1548
1549 FuncInfo PowrInfo;
1550 AMDGPULibFunc::EFuncId FastPowrFuncId =
1551 FMF.approxFunc() || FInfo.getId() == AMDGPULibFunc::EI_POW_FAST
1552 ? AMDGPULibFunc::EI_POWR_FAST
1553 : AMDGPULibFunc::EI_NONE;
1554 FunctionCallee PowrFunc = getFloatFastVariant(
1555 M, fInfo: FInfo, newInfo&: PowrInfo, NewFunc: AMDGPULibFunc::EI_POWR, FastVariant: FastPowrFuncId);
1556
1557 // TODO: Prefer fast pown to fast powr, but slow powr to slow pown.
1558
1559 // pow(x, y) -> powr(x, y) for x >= -0.0
1560 // TODO: Account for flags on current call
1561 if (PowrFunc && cannotBeOrderedLessThanZero(V: FPOp->getOperand(i: 0),
1562 SQ: SQ.getWithInstruction(I: Call))) {
1563 Call->setCalledFunction(PowrFunc);
1564 return fold_pow(FPOp, B, FInfo: PowrInfo) || true;
1565 }
1566
1567 // pow(x, y) -> pown(x, y) for known integral y
1568 if (isKnownIntegral(V: FPOp->getOperand(i: 1), SQ: SQ.getWithInstruction(I: Call),
1569 FMF: FPOp->getFastMathFlags())) {
1570 FunctionType *PownType = getPownType(FT: Call->getFunctionType());
1571
1572 FuncInfo PownInfo;
1573 AMDGPULibFunc::EFuncId FastPownFuncId =
1574 FMF.approxFunc() || FInfo.getId() == AMDGPULibFunc::EI_POW_FAST
1575 ? AMDGPULibFunc::EI_POWN_FAST
1576 : AMDGPULibFunc::EI_NONE;
1577 FunctionCallee PownFunc = getFloatFastVariant(
1578 M, fInfo: FInfo, newInfo&: PownInfo, NewFunc: AMDGPULibFunc::EI_POWN, FastVariant: FastPownFuncId);
1579
1580 if (PownFunc) {
1581 // TODO: If the incoming integral value is an sitofp/uitofp, it won't
1582 // fold out without a known range. We can probably take the source
1583 // value directly.
1584 Value *CastedArg =
1585 B.CreateFPToSI(V: FPOp->getOperand(i: 1), DestTy: PownType->getParamType(i: 1));
1586 // Have to drop any nofpclass attributes on the original call site.
1587 Call->removeParamAttrs(
1588 ArgNo: 1, AttrsToRemove: AttributeFuncs::typeIncompatible(Ty: CastedArg->getType(),
1589 AS: Call->getParamAttributes(ArgNo: 1)));
1590 Call->setCalledFunction(PownFunc);
1591 Call->setArgOperand(i: 1, v: CastedArg);
1592 return fold_pow(FPOp, B, FInfo: PownInfo) || true;
1593 }
1594 }
1595
1596 if (fold_pow(FPOp, B, FInfo))
1597 return true;
1598
1599 if (!FMF.approxFunc())
1600 return false;
1601
1602 if (FInfo.getId() == AMDGPULibFunc::EI_POW && FMF.approxFunc() &&
1603 getArgType(FInfo) == AMDGPULibFunc::F32) {
1604 AMDGPULibFunc PowFastInfo(AMDGPULibFunc::EI_POW_FAST, FInfo);
1605 if (FunctionCallee PowFastFunc = getFunction(M, fInfo: PowFastInfo)) {
1606 Call->setCalledFunction(PowFastFunc);
1607 return fold_pow(FPOp, B, FInfo: PowFastInfo) || true;
1608 }
1609 }
1610
1611 return expandFastPow(FPOp, B, Kind: PowKind::Pow);
1612}
1613
1614// Get a scalar native builtin single argument FP function
1615FunctionCallee AMDGPULibCalls::getNativeFunction(Module *M,
1616 const FuncInfo &FInfo) {
1617 if (getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(id: FInfo.getId()))
1618 return nullptr;
1619 FuncInfo nf = FInfo;
1620 nf.setPrefix(AMDGPULibFunc::NATIVE);
1621 return getFunction(M, fInfo: nf);
1622}
1623
1624// Some library calls are just wrappers around llvm intrinsics, but compiled
1625// conservatively. Preserve the flags from the original call site by
1626// substituting them with direct calls with all the flags.
1627bool AMDGPULibCalls::shouldReplaceLibcallWithIntrinsic(const CallInst *CI,
1628 bool AllowMinSizeF32,
1629 bool AllowF64,
1630 bool AllowStrictFP) {
1631 Type *FltTy = CI->getType()->getScalarType();
1632 const bool IsF32 = FltTy->isFloatTy();
1633
1634 // f64 intrinsics aren't implemented for most operations.
1635 if (!IsF32 && !FltTy->isHalfTy() && (!AllowF64 || !FltTy->isDoubleTy()))
1636 return false;
1637
1638 // We're implicitly inlining by replacing the libcall with the intrinsic, so
1639 // don't do it for noinline call sites.
1640 if (CI->isNoInline())
1641 return false;
1642
1643 const Function *ParentF = CI->getFunction();
1644 // TODO: Handle strictfp
1645 if (!AllowStrictFP && ParentF->hasFnAttribute(Kind: Attribute::StrictFP))
1646 return false;
1647
1648 if (IsF32 && !AllowMinSizeF32 && ParentF->hasMinSize())
1649 return false;
1650 return true;
1651}
1652
1653void AMDGPULibCalls::replaceLibCallWithSimpleIntrinsic(IRBuilder<> &B,
1654 CallInst *CI,
1655 Intrinsic::ID IntrID) {
1656 if (CI->arg_size() == 2) {
1657 Value *Arg0 = CI->getArgOperand(i: 0);
1658 Value *Arg1 = CI->getArgOperand(i: 1);
1659 VectorType *Arg0VecTy = dyn_cast<VectorType>(Val: Arg0->getType());
1660 VectorType *Arg1VecTy = dyn_cast<VectorType>(Val: Arg1->getType());
1661 if (Arg0VecTy && !Arg1VecTy) {
1662 Value *SplatRHS = B.CreateVectorSplat(EC: Arg0VecTy->getElementCount(), V: Arg1);
1663 CI->setArgOperand(i: 1, v: SplatRHS);
1664 } else if (!Arg0VecTy && Arg1VecTy) {
1665 Value *SplatLHS = B.CreateVectorSplat(EC: Arg1VecTy->getElementCount(), V: Arg0);
1666 CI->setArgOperand(i: 0, v: SplatLHS);
1667 }
1668 }
1669
1670 CI->setCalledFunction(Intrinsic::getOrInsertDeclaration(
1671 M: CI->getModule(), id: IntrID, OverloadTys: {CI->getType()}));
1672 CI->setCallingConv(CallingConv::C);
1673}
1674
1675bool AMDGPULibCalls::tryReplaceLibcallWithSimpleIntrinsic(
1676 IRBuilder<> &B, CallInst *CI, Intrinsic::ID IntrID, bool AllowMinSizeF32,
1677 bool AllowF64, bool AllowStrictFP) {
1678 if (!shouldReplaceLibcallWithIntrinsic(CI, AllowMinSizeF32, AllowF64,
1679 AllowStrictFP))
1680 return false;
1681 replaceLibCallWithSimpleIntrinsic(B, CI, IntrID);
1682 return true;
1683}
1684
1685std::tuple<Value *, Value *, Value *>
1686AMDGPULibCalls::insertSinCos(Value *Arg, FastMathFlags FMF, IRBuilder<> &B,
1687 FunctionCallee Fsincos) {
1688 DebugLoc DL = B.getCurrentDebugLocation();
1689 Function *F = B.GetInsertBlock()->getParent();
1690 B.SetInsertPointPastAllocas(F);
1691
1692 AllocaInst *Alloc = B.CreateAlloca(Ty: Arg->getType(), ArraySize: nullptr, Name: "__sincos_");
1693
1694 if (Instruction *ArgInst = dyn_cast<Instruction>(Val: Arg)) {
1695 // If the argument is an instruction, it must dominate all uses so put our
1696 // sincos call there. Otherwise, right after the allocas works well enough
1697 // if it's an argument or constant.
1698
1699 B.SetInsertPoint(*ArgInst->getInsertionPointAfterDef());
1700
1701 // SetInsertPoint unwelcomely always tries to set the debug loc.
1702 B.SetCurrentDebugLocation(DL);
1703 }
1704
1705 Type *CosPtrTy = Fsincos.getFunctionType()->getParamType(i: 1);
1706
1707 // The allocaInst allocates the memory in private address space. This need
1708 // to be addrspacecasted to point to the address space of cos pointer type.
1709 // In OpenCL 2.0 this is generic, while in 1.2 that is private.
1710 Value *CastAlloc = B.CreateAddrSpaceCast(V: Alloc, DestTy: CosPtrTy);
1711
1712 CallInst *SinCos = CreateCallEx2(B, Callee: Fsincos, Arg1: Arg, Arg2: CastAlloc);
1713
1714 // TODO: Is it worth trying to preserve the location for the cos calls for the
1715 // load?
1716
1717 LoadInst *LoadCos = B.CreateLoad(Ty: Arg->getType(), Ptr: Alloc);
1718 return {SinCos, LoadCos, SinCos};
1719}
1720
1721// fold sin, cos -> sincos.
1722bool AMDGPULibCalls::fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B,
1723 const FuncInfo &fInfo) {
1724 assert(fInfo.getId() == AMDGPULibFunc::EI_SIN ||
1725 fInfo.getId() == AMDGPULibFunc::EI_COS);
1726
1727 if ((getArgType(FInfo: fInfo) != AMDGPULibFunc::F32 &&
1728 getArgType(FInfo: fInfo) != AMDGPULibFunc::F64) ||
1729 fInfo.getPrefix() != AMDGPULibFunc::NOPFX)
1730 return false;
1731
1732 bool const isSin = fInfo.getId() == AMDGPULibFunc::EI_SIN;
1733
1734 Value *CArgVal = FPOp->getOperand(i: 0);
1735
1736 // TODO: Constant fold the call
1737 if (isa<ConstantData>(Val: CArgVal))
1738 return false;
1739
1740 CallInst *CI = cast<CallInst>(Val: FPOp);
1741
1742 Function *F = B.GetInsertBlock()->getParent();
1743 Module *M = F->getParent();
1744
1745 // Merge the sin and cos. For OpenCL 2.0, there may only be a generic pointer
1746 // implementation. Prefer the private form if available.
1747 AMDGPULibFunc SinCosLibFuncPrivate(AMDGPULibFunc::EI_SINCOS, fInfo);
1748 SinCosLibFuncPrivate.getLeads()[0].PtrKind =
1749 AMDGPULibFunc::getEPtrKindFromAddrSpace(AS: AMDGPUAS::PRIVATE_ADDRESS);
1750
1751 AMDGPULibFunc SinCosLibFuncGeneric(AMDGPULibFunc::EI_SINCOS, fInfo);
1752 SinCosLibFuncGeneric.getLeads()[0].PtrKind =
1753 AMDGPULibFunc::getEPtrKindFromAddrSpace(AS: AMDGPUAS::FLAT_ADDRESS);
1754
1755 FunctionCallee FSinCosPrivate = getFunction(M, fInfo: SinCosLibFuncPrivate);
1756 FunctionCallee FSinCosGeneric = getFunction(M, fInfo: SinCosLibFuncGeneric);
1757 FunctionCallee FSinCos = FSinCosPrivate ? FSinCosPrivate : FSinCosGeneric;
1758 if (!FSinCos)
1759 return false;
1760
1761 SmallVector<CallInst *> SinCalls;
1762 SmallVector<CallInst *> CosCalls;
1763 SmallVector<CallInst *> SinCosCalls;
1764 FuncInfo PartnerInfo(isSin ? AMDGPULibFunc::EI_COS : AMDGPULibFunc::EI_SIN,
1765 fInfo);
1766 const std::string PairName = PartnerInfo.mangle();
1767
1768 StringRef SinName = isSin ? CI->getCalledFunction()->getName() : PairName;
1769 StringRef CosName = isSin ? PairName : CI->getCalledFunction()->getName();
1770 const std::string SinCosPrivateName = SinCosLibFuncPrivate.mangle();
1771 const std::string SinCosGenericName = SinCosLibFuncGeneric.mangle();
1772
1773 // Intersect the two sets of flags.
1774 FastMathFlags FMF = FPOp->getFastMathFlags();
1775 MDNode *FPMath = CI->getMetadata(KindID: LLVMContext::MD_fpmath);
1776
1777 SmallVector<DILocation *> MergeDbgLocs = {CI->getDebugLoc()};
1778
1779 for (User* U : CArgVal->users()) {
1780 CallInst *XI = dyn_cast<CallInst>(Val: U);
1781 if (!XI || XI->getFunction() != F || XI->isNoBuiltin())
1782 continue;
1783
1784 Function *UCallee = XI->getCalledFunction();
1785 if (!UCallee)
1786 continue;
1787
1788 bool Handled = true;
1789
1790 if (UCallee->getName() == SinName)
1791 SinCalls.push_back(Elt: XI);
1792 else if (UCallee->getName() == CosName)
1793 CosCalls.push_back(Elt: XI);
1794 else if (UCallee->getName() == SinCosPrivateName ||
1795 UCallee->getName() == SinCosGenericName)
1796 SinCosCalls.push_back(Elt: XI);
1797 else
1798 Handled = false;
1799
1800 if (Handled) {
1801 MergeDbgLocs.push_back(Elt: XI->getDebugLoc());
1802 auto *OtherOp = cast<FPMathOperator>(Val: XI);
1803 FMF &= OtherOp->getFastMathFlags();
1804 FPMath = MDNode::getMostGenericFPMath(
1805 A: FPMath, B: XI->getMetadata(KindID: LLVMContext::MD_fpmath));
1806 }
1807 }
1808
1809 if (SinCalls.empty() || CosCalls.empty())
1810 return false;
1811
1812 // insertSinCos needs an insertion point after the argument's def.
1813 if (auto *ArgInst = dyn_cast<Instruction>(Val: CArgVal);
1814 ArgInst && !ArgInst->getInsertionPointAfterDef())
1815 return false;
1816
1817 B.setFastMathFlags(FMF);
1818 B.setDefaultFPMathTag(FPMath);
1819 DILocation *DbgLoc = DILocation::getMergedLocations(Locs: MergeDbgLocs);
1820 B.SetCurrentDebugLocation(DbgLoc);
1821
1822 auto [Sin, Cos, SinCos] = insertSinCos(Arg: CArgVal, FMF, B, Fsincos: FSinCos);
1823
1824 auto replaceTrigInsts = [](ArrayRef<CallInst *> Calls, Value *Res) {
1825 for (CallInst *C : Calls)
1826 C->replaceAllUsesWith(V: Res);
1827
1828 // Leave the other dead instructions to avoid clobbering iterators.
1829 };
1830
1831 replaceTrigInsts(SinCalls, Sin);
1832 replaceTrigInsts(CosCalls, Cos);
1833 replaceTrigInsts(SinCosCalls, SinCos);
1834
1835 // It's safe to delete the original now.
1836 CI->eraseFromParent();
1837 return true;
1838}
1839
1840bool AMDGPULibCalls::evaluateScalarMathFunc(const FuncInfo &FInfo,
1841 APFloat &Res0, APFloat &Res1,
1842 Constant *copr0, Constant *copr1) {
1843 // Every function handled below reads its first operand as a floating-point
1844 // value. Refuse anything else, e.g. a poison vector lane: silently treating
1845 // it as 0.0 misfolds the whole call.
1846 ConstantFP *fpopr0 = dyn_cast_or_null<ConstantFP>(Val: copr0);
1847 if (!fpopr0)
1848 return false;
1849
1850 double opr0 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1851 ? fpopr0->getValueAPF().convertToDouble()
1852 : (double)fpopr0->getValueAPF().convertToFloat();
1853
1854 switch (FInfo.getId()) {
1855 default:
1856 return false;
1857
1858 case AMDGPULibFunc::EI_ACOS:
1859 Res0 = APFloat{acos(x: opr0)};
1860 return true;
1861
1862 case AMDGPULibFunc::EI_ACOSH:
1863 // acosh(x) == log(x + sqrt(x*x - 1))
1864 Res0 = APFloat{log(x: opr0 + sqrt(x: opr0 * opr0 - 1.0))};
1865 return true;
1866
1867 case AMDGPULibFunc::EI_ACOSPI:
1868 Res0 = APFloat{acos(x: opr0) / MATH_PI};
1869 return true;
1870
1871 case AMDGPULibFunc::EI_ASIN:
1872 Res0 = APFloat{asin(x: opr0)};
1873 return true;
1874
1875 case AMDGPULibFunc::EI_ASINH:
1876 // asinh(x) == log(x + sqrt(x*x + 1))
1877 Res0 = APFloat{log(x: opr0 + sqrt(x: opr0 * opr0 + 1.0))};
1878 return true;
1879
1880 case AMDGPULibFunc::EI_ASINPI:
1881 Res0 = APFloat{asin(x: opr0) / MATH_PI};
1882 return true;
1883
1884 case AMDGPULibFunc::EI_ATAN:
1885 Res0 = APFloat{atan(x: opr0)};
1886 return true;
1887
1888 case AMDGPULibFunc::EI_ATANH:
1889 // atanh(x) == (log(x+1) - log(x-1))/2;
1890 Res0 = APFloat{(log(x: opr0 + 1.0) - log(x: opr0 - 1.0)) / 2.0};
1891 return true;
1892
1893 case AMDGPULibFunc::EI_ATANPI:
1894 Res0 = APFloat{atan(x: opr0) / MATH_PI};
1895 return true;
1896
1897 case AMDGPULibFunc::EI_CBRT:
1898 Res0 =
1899 APFloat{(opr0 < 0.0) ? -pow(x: -opr0, y: 1.0 / 3.0) : pow(x: opr0, y: 1.0 / 3.0)};
1900 return true;
1901
1902 case AMDGPULibFunc::EI_COS:
1903 Res0 = APFloat{cos(x: opr0)};
1904 return true;
1905
1906 case AMDGPULibFunc::EI_COSH:
1907 Res0 = APFloat{cosh(x: opr0)};
1908 return true;
1909
1910 case AMDGPULibFunc::EI_COSPI:
1911 Res0 = APFloat{cos(MATH_PI * opr0)};
1912 return true;
1913
1914 case AMDGPULibFunc::EI_EXP:
1915 Res0 = APFloat{std::exp(x: opr0)};
1916 return true;
1917
1918 case AMDGPULibFunc::EI_EXP2:
1919 Res0 = APFloat{pow(x: 2.0, y: opr0)};
1920 return true;
1921
1922 case AMDGPULibFunc::EI_EXP10:
1923 Res0 = APFloat{pow(x: 10.0, y: opr0)};
1924 return true;
1925
1926 case AMDGPULibFunc::EI_LOG:
1927 Res0 = APFloat{log(x: opr0)};
1928 return true;
1929
1930 case AMDGPULibFunc::EI_LOG2:
1931 Res0 = APFloat{log(x: opr0) / log(x: 2.0)};
1932 return true;
1933
1934 case AMDGPULibFunc::EI_LOG10:
1935 Res0 = APFloat{log(x: opr0) / log(x: 10.0)};
1936 return true;
1937
1938 case AMDGPULibFunc::EI_RSQRT:
1939 Res0 = APFloat{1.0 / sqrt(x: opr0)};
1940 return true;
1941
1942 case AMDGPULibFunc::EI_SIN:
1943 Res0 = APFloat{sin(x: opr0)};
1944 return true;
1945
1946 case AMDGPULibFunc::EI_SINH:
1947 Res0 = APFloat{sinh(x: opr0)};
1948 return true;
1949
1950 case AMDGPULibFunc::EI_SINPI:
1951 Res0 = APFloat{sin(MATH_PI * opr0)};
1952 return true;
1953
1954 case AMDGPULibFunc::EI_TAN:
1955 Res0 = APFloat{tan(x: opr0)};
1956 return true;
1957
1958 case AMDGPULibFunc::EI_TANH:
1959 Res0 = APFloat{tanh(x: opr0)};
1960 return true;
1961
1962 case AMDGPULibFunc::EI_TANPI:
1963 Res0 = APFloat{tan(MATH_PI * opr0)};
1964 return true;
1965
1966 // two-arg functions
1967 case AMDGPULibFunc::EI_POW:
1968 case AMDGPULibFunc::EI_POWR: {
1969 ConstantFP *fpopr1 = dyn_cast_or_null<ConstantFP>(Val: copr1);
1970 if (!fpopr1)
1971 return false;
1972 double opr1 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1973 ? fpopr1->getValueAPF().convertToDouble()
1974 : (double)fpopr1->getValueAPF().convertToFloat();
1975 Res0 = APFloat{pow(x: opr0, y: opr1)};
1976 return true;
1977 }
1978
1979 case AMDGPULibFunc::EI_POWN: {
1980 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(Val: copr1)) {
1981 double val = (double)iopr1->getSExtValue();
1982 Res0 = APFloat{pow(x: opr0, y: val)};
1983 return true;
1984 }
1985 return false;
1986 }
1987
1988 case AMDGPULibFunc::EI_ROOTN: {
1989 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(Val: copr1)) {
1990 double val = (double)iopr1->getSExtValue();
1991 Res0 = APFloat{pow(x: opr0, y: 1.0 / val)};
1992 return true;
1993 }
1994 return false;
1995 }
1996
1997 // with ptr arg
1998 case AMDGPULibFunc::EI_SINCOS:
1999 Res0 = APFloat{sin(x: opr0)};
2000 Res1 = APFloat{cos(x: opr0)};
2001 return true;
2002 }
2003
2004 return false;
2005}
2006
2007bool AMDGPULibCalls::evaluateCall(CallInst *aCI, const FuncInfo &FInfo) {
2008 int numArgs = (int)aCI->arg_size();
2009 if (numArgs > 3)
2010 return false;
2011
2012 Constant *copr0 = nullptr;
2013 Constant *copr1 = nullptr;
2014 if (numArgs > 0) {
2015 if ((copr0 = dyn_cast<Constant>(Val: aCI->getArgOperand(i: 0))) == nullptr)
2016 return false;
2017 }
2018
2019 if (numArgs > 1) {
2020 if ((copr1 = dyn_cast<Constant>(Val: aCI->getArgOperand(i: 1))) == nullptr) {
2021 if (FInfo.getId() != AMDGPULibFunc::EI_SINCOS)
2022 return false;
2023 }
2024 }
2025
2026 // At this point, all arguments to aCI are constants.
2027
2028 // max vector size is 16, and sincos will generate two results.
2029 SmallVector<APFloat, 16> Val0, Val1;
2030 int FuncVecSize = getVecSize(FInfo);
2031 if (FuncVecSize == 1) {
2032 if (!evaluateScalarMathFunc(FInfo, Res0&: Val0.emplace_back(Args: 0.0),
2033 Res1&: Val1.emplace_back(Args: 0.0), copr0, copr1)) {
2034 return false;
2035 }
2036 } else {
2037 // An operand of a vector variant is not necessarily a vector: sincos takes
2038 // a pointer as its second operand, and fmin/fmax/ldexp accept an
2039 // implicitly splatted scalar. Only index into actual vectors.
2040 Constant *CV0 = copr0 && copr0->getType()->isVectorTy() ? copr0 : nullptr;
2041 Constant *CV1 = copr1 && copr1->getType()->isVectorTy() ? copr1 : nullptr;
2042 for (int i = 0; i < FuncVecSize; ++i) {
2043 Constant *celt0 = CV0 ? CV0->getAggregateElement(Elt: (unsigned)i) : nullptr;
2044 Constant *celt1 = CV1 ? CV1->getAggregateElement(Elt: (unsigned)i) : nullptr;
2045 if (!evaluateScalarMathFunc(FInfo, Res0&: Val0.emplace_back(Args: 0.0),
2046 Res1&: Val1.emplace_back(Args: 0.0), copr0: celt0, copr1: celt1)) {
2047 return false;
2048 }
2049 }
2050 }
2051
2052 Constant *nval0 = getConstantFloat(Values: Val0, Ty: aCI->getType());
2053
2054 // sincos
2055 if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS) {
2056 Constant *nval1 = getConstantFloat(Values: Val1, Ty: aCI->getType());
2057 new StoreInst(nval1, aCI->getArgOperand(i: 1), aCI->getIterator());
2058 }
2059
2060 replaceCall(I: aCI, With: nval0);
2061 return true;
2062}
2063
2064PreservedAnalyses AMDGPUSimplifyLibCallsPass::run(Function &F,
2065 FunctionAnalysisManager &AM) {
2066 AMDGPULibCalls Simplifier(F, AM);
2067 Simplifier.initNativeFuncs();
2068
2069 bool Changed = false;
2070
2071 LLVM_DEBUG(dbgs() << "AMDIC: process function ";
2072 F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';);
2073
2074 for (auto &BB : F) {
2075 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) {
2076 // Ignore non-calls.
2077 CallInst *CI = dyn_cast<CallInst>(Val&: I);
2078 ++I;
2079
2080 if (CI) {
2081 if (Simplifier.fold(CI))
2082 Changed = true;
2083 }
2084 }
2085 }
2086 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
2087}
2088
2089PreservedAnalyses AMDGPUUseNativeCallsPass::run(Function &F,
2090 FunctionAnalysisManager &AM) {
2091 if (UseNative.empty())
2092 return PreservedAnalyses::all();
2093
2094 AMDGPULibCalls Simplifier(F, AM);
2095 Simplifier.initNativeFuncs();
2096
2097 bool Changed = false;
2098 for (auto &BB : F) {
2099 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) {
2100 // Ignore non-calls.
2101 CallInst *CI = dyn_cast<CallInst>(Val&: I);
2102 ++I;
2103 if (CI && Simplifier.useNative(aCI: CI))
2104 Changed = true;
2105 }
2106 }
2107 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
2108}
2109