1//===-- AMDGPULibFunc.h ----------------------------------------*- C++ -*--===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef _AMDGPU_LIBFUNC_H_
10#define _AMDGPU_LIBFUNC_H_
11
12#include "llvm/ADT/StringRef.h"
13#include <memory>
14
15namespace llvm {
16
17class FunctionCallee;
18class FunctionType;
19class Function;
20class Module;
21class Type;
22
23class AMDGPULibFuncBase {
24public:
25 enum EFuncId {
26 EI_NONE,
27
28 // IMPORTANT: enums below should go in ascending by 1 value order
29 // because they are used as indexes in the mangling rules table.
30 // don't use explicit value assignment.
31 //
32 // There are two types of library functions: those with mangled
33 // name and those with unmangled name. The enums for the library
34 // functions with mangled name are defined before enums for the
35 // library functions with unmangled name. The enum for the last
36 // library function with mangled name is EI_LAST_MANGLED.
37 //
38 // Library functions with mangled name.
39 EI_ABS,
40 EI_ABS_DIFF,
41 EI_ACOS,
42 EI_ACOSH,
43 EI_ACOSPI,
44 EI_ADD_SAT,
45 EI_ALL,
46 EI_ANY,
47 EI_ASIN,
48 EI_ASINH,
49 EI_ASINPI,
50 EI_ASYNC_WORK_GROUP_COPY,
51 EI_ASYNC_WORK_GROUP_STRIDED_COPY,
52 EI_ATAN,
53 EI_ATAN2,
54 EI_ATAN2PI,
55 EI_ATANH,
56 EI_ATANPI,
57 EI_ATOMIC_ADD,
58 EI_ATOMIC_AND,
59 EI_ATOMIC_CMPXCHG,
60 EI_ATOMIC_DEC,
61 EI_ATOMIC_INC,
62 EI_ATOMIC_MAX,
63 EI_ATOMIC_MIN,
64 EI_ATOMIC_OR,
65 EI_ATOMIC_SUB,
66 EI_ATOMIC_XCHG,
67 EI_ATOMIC_XOR,
68 EI_BITSELECT,
69 EI_CBRT,
70 EI_CEIL,
71 EI_CLAMP,
72 EI_CLZ,
73 EI_COMMIT_READ_PIPE,
74 EI_COMMIT_WRITE_PIPE,
75 EI_COPYSIGN,
76 EI_COS,
77 EI_COSH,
78 EI_COSPI,
79 EI_CROSS,
80 EI_CTZ,
81 EI_DEGREES,
82 EI_DISTANCE,
83 EI_DIVIDE,
84 EI_DOT,
85 EI_ERF,
86 EI_ERFC,
87 EI_EXP,
88 EI_EXP10,
89 EI_EXP2,
90 EI_EXPM1,
91 EI_FABS,
92 EI_FAST_DISTANCE,
93 EI_FAST_LENGTH,
94 EI_FAST_NORMALIZE,
95 EI_FDIM,
96 EI_FLOOR,
97 EI_FMA,
98 EI_FMAX,
99 EI_FMIN,
100 EI_FMOD,
101 EI_FRACT,
102 EI_FREXP,
103 EI_GET_IMAGE_ARRAY_SIZE,
104 EI_GET_IMAGE_CHANNEL_DATA_TYPE,
105 EI_GET_IMAGE_CHANNEL_ORDER,
106 EI_GET_IMAGE_DIM,
107 EI_GET_IMAGE_HEIGHT,
108 EI_GET_IMAGE_WIDTH,
109 EI_GET_PIPE_MAX_PACKETS,
110 EI_GET_PIPE_NUM_PACKETS,
111 EI_HADD,
112 EI_HYPOT,
113 EI_ILOGB,
114 EI_ISEQUAL,
115 EI_ISFINITE,
116 EI_ISGREATER,
117 EI_ISGREATEREQUAL,
118 EI_ISINF,
119 EI_ISLESS,
120 EI_ISLESSEQUAL,
121 EI_ISLESSGREATER,
122 EI_ISNAN,
123 EI_ISNORMAL,
124 EI_ISNOTEQUAL,
125 EI_ISORDERED,
126 EI_ISUNORDERED,
127 EI_LDEXP,
128 EI_LENGTH,
129 EI_LGAMMA,
130 EI_LGAMMA_R,
131 EI_LOG,
132 EI_LOG10,
133 EI_LOG1P,
134 EI_LOG2,
135 EI_LOGB,
136 EI_MAD,
137 EI_MAD24,
138 EI_MAD_HI,
139 EI_MAD_SAT,
140 EI_MAX,
141 EI_MAXMAG,
142 EI_MIN,
143 EI_MINMAG,
144 EI_MIX,
145 EI_MODF,
146 EI_MUL24,
147 EI_MUL_HI,
148 EI_NAN,
149 EI_NEXTAFTER,
150 EI_NORMALIZE,
151 EI_POPCOUNT,
152 EI_POW,
153 EI_POW_FAST,
154 EI_POWN,
155 EI_POWN_FAST,
156 EI_POWR,
157 EI_POWR_FAST,
158 EI_PREFETCH,
159 EI_RADIANS,
160 EI_RECIP,
161 EI_REMAINDER,
162 EI_REMQUO,
163 EI_RESERVE_READ_PIPE,
164 EI_RESERVE_WRITE_PIPE,
165 EI_RHADD,
166 EI_RINT,
167 EI_ROOTN,
168 EI_ROOTN_FAST,
169 EI_ROTATE,
170 EI_ROUND,
171 EI_RSQRT,
172 EI_SELECT,
173 EI_SHUFFLE,
174 EI_SHUFFLE2,
175 EI_SIGN,
176 EI_SIGNBIT,
177 EI_SIN,
178 EI_SINCOS,
179 EI_SINH,
180 EI_SINPI,
181 EI_SMOOTHSTEP,
182 EI_SQRT,
183 EI_STEP,
184 EI_SUB_GROUP_BROADCAST,
185 EI_SUB_GROUP_COMMIT_READ_PIPE,
186 EI_SUB_GROUP_COMMIT_WRITE_PIPE,
187 EI_SUB_GROUP_REDUCE_ADD,
188 EI_SUB_GROUP_REDUCE_MAX,
189 EI_SUB_GROUP_REDUCE_MIN,
190 EI_SUB_GROUP_RESERVE_READ_PIPE,
191 EI_SUB_GROUP_RESERVE_WRITE_PIPE,
192 EI_SUB_GROUP_SCAN_EXCLUSIVE_ADD,
193 EI_SUB_GROUP_SCAN_EXCLUSIVE_MAX,
194 EI_SUB_GROUP_SCAN_EXCLUSIVE_MIN,
195 EI_SUB_GROUP_SCAN_INCLUSIVE_ADD,
196 EI_SUB_GROUP_SCAN_INCLUSIVE_MAX,
197 EI_SUB_GROUP_SCAN_INCLUSIVE_MIN,
198 EI_SUB_SAT,
199 EI_TAN,
200 EI_TANH,
201 EI_TANPI,
202 EI_TGAMMA,
203 EI_TRUNC,
204 EI_UPSAMPLE,
205 EI_VEC_STEP,
206 EI_VSTORE,
207 EI_VSTORE16,
208 EI_VSTORE2,
209 EI_VSTORE3,
210 EI_VSTORE4,
211 EI_VSTORE8,
212 EI_WORK_GROUP_COMMIT_READ_PIPE,
213 EI_WORK_GROUP_COMMIT_WRITE_PIPE,
214 EI_WORK_GROUP_REDUCE_ADD,
215 EI_WORK_GROUP_REDUCE_MAX,
216 EI_WORK_GROUP_REDUCE_MIN,
217 EI_WORK_GROUP_RESERVE_READ_PIPE,
218 EI_WORK_GROUP_RESERVE_WRITE_PIPE,
219 EI_WORK_GROUP_SCAN_EXCLUSIVE_ADD,
220 EI_WORK_GROUP_SCAN_EXCLUSIVE_MAX,
221 EI_WORK_GROUP_SCAN_EXCLUSIVE_MIN,
222 EI_WORK_GROUP_SCAN_INCLUSIVE_ADD,
223 EI_WORK_GROUP_SCAN_INCLUSIVE_MAX,
224 EI_WORK_GROUP_SCAN_INCLUSIVE_MIN,
225 EI_WRITE_IMAGEF,
226 EI_WRITE_IMAGEI,
227 EI_WRITE_IMAGEUI,
228 EI_NCOS,
229 EI_NEXP2,
230 EI_NFMA,
231 EI_NLOG2,
232 EI_NRCP,
233 EI_NRSQRT,
234 EI_NSIN,
235 EI_NSQRT,
236 EI_FTZ,
237 EI_FLDEXP,
238 EI_CLASS,
239 EI_RCBRT,
240 EI_LAST_MANGLED =
241 EI_RCBRT, /* The last library function with mangled name */
242
243 // Library functions with unmangled name.
244 EI_READ_PIPE_2,
245 EI_READ_PIPE_4,
246 EI_WRITE_PIPE_2,
247 EI_WRITE_PIPE_4,
248
249 EX_INTRINSICS_COUNT
250 };
251
252 enum ENamePrefix {
253 NOPFX,
254 NATIVE,
255 HALF
256 };
257
258 enum EType {
259 B8 = 1,
260 B16 = 2,
261 B32 = 3,
262 B64 = 4,
263 SIZE_MASK = 7,
264 FLOAT = 0x10,
265 INT = 0x20,
266 UINT = 0x30,
267 BASE_TYPE_MASK = 0x30,
268 U8 = UINT | B8,
269 U16 = UINT | B16,
270 U32 = UINT | B32,
271 U64 = UINT | B64,
272 I8 = INT | B8,
273 I16 = INT | B16,
274 I32 = INT | B32,
275 I64 = INT | B64,
276 F16 = FLOAT | B16,
277 F32 = FLOAT | B32,
278 F64 = FLOAT | B64,
279 IMG1DA = 0x80,
280 IMG1DB,
281 IMG2DA,
282 IMG1D,
283 IMG2D,
284 IMG3D,
285 SAMPLER,
286 EVENT,
287 DUMMY
288 };
289
290 enum EPtrKind {
291 BYVALUE = 0,
292 ADDR_SPACE = 0xF, // Address space takes value 0x1 ~ 0xF.
293 CONST = 0x10,
294 VOLATILE = 0x20
295 };
296
297 struct Param {
298 unsigned char ArgType = 0;
299 unsigned char VectorSize = 1;
300 unsigned char PtrKind = 0;
301
302 unsigned char Reserved = 0;
303
304 void reset() {
305 ArgType = 0;
306 VectorSize = 1;
307 PtrKind = 0;
308 }
309
310 static Param getIntN(unsigned char NumElts) {
311 return Param{.ArgType: I32, .VectorSize: NumElts, .PtrKind: 0, .Reserved: 0};
312 }
313
314 static Param getFromTy(Type *Ty, bool Signed);
315 };
316 static bool isMangled(EFuncId Id) {
317 return static_cast<unsigned>(Id) <= static_cast<unsigned>(EI_LAST_MANGLED);
318 }
319
320 static unsigned getEPtrKindFromAddrSpace(unsigned AS) {
321 assert(((AS + 1) & ~ADDR_SPACE) == 0);
322 return AS + 1;
323 }
324
325 static unsigned getAddrSpaceFromEPtrKind(unsigned Kind) {
326 Kind = Kind & ADDR_SPACE;
327 assert(Kind >= 1);
328 return Kind - 1;
329 }
330};
331
332class AMDGPULibFuncImpl : public AMDGPULibFuncBase {
333public:
334 AMDGPULibFuncImpl() = default;
335 virtual ~AMDGPULibFuncImpl() = default;
336
337 /// Get unmangled name for mangled library function and name for unmangled
338 /// library function.
339 virtual std::string getName() const = 0;
340 virtual unsigned getNumArgs() const = 0;
341 EFuncId getId() const { return FuncId; }
342 ENamePrefix getPrefix() const { return FKind; }
343
344 bool isMangled() const { return AMDGPULibFuncBase::isMangled(Id: FuncId); }
345
346 void setId(EFuncId id) { FuncId = id; }
347 virtual bool parseFuncName(StringRef &mangledName) = 0;
348
349 /// \return The mangled function name for mangled library functions
350 /// and unmangled function name for unmangled library functions.
351 virtual std::string mangle() const = 0;
352
353 void setName(StringRef N) { Name = std::string(N); }
354 void setPrefix(ENamePrefix pfx) { FKind = pfx; }
355
356 virtual FunctionType *getFunctionType(const Module &M) const = 0;
357
358protected:
359 EFuncId FuncId;
360 std::string Name;
361 ENamePrefix FKind = NOPFX;
362};
363
364/// Wrapper class for AMDGPULIbFuncImpl
365class AMDGPULibFunc : public AMDGPULibFuncBase {
366public:
367 explicit AMDGPULibFunc() : Impl(std::unique_ptr<AMDGPULibFuncImpl>()) {}
368 AMDGPULibFunc(const AMDGPULibFunc &F);
369 /// Clone a mangled library func with the Id \p Id and argument info from \p
370 /// CopyFrom.
371 explicit AMDGPULibFunc(EFuncId Id, const AMDGPULibFunc &CopyFrom);
372 explicit AMDGPULibFunc(EFuncId Id, FunctionType *FT, bool SignedInts);
373
374 /// Construct an unmangled library function on the fly.
375 explicit AMDGPULibFunc(StringRef FName, FunctionType *FT);
376
377 AMDGPULibFunc &operator=(const AMDGPULibFunc &F);
378
379 /// Get unmangled name for mangled library function and name for unmangled
380 /// library function.
381 std::string getName() const { return Impl->getName(); }
382 unsigned getNumArgs() const { return Impl->getNumArgs(); }
383 EFuncId getId() const { return Impl->getId(); }
384 ENamePrefix getPrefix() const { return Impl->getPrefix(); }
385 /// Get leading parameters for mangled lib functions.
386 Param *getLeads();
387 const Param *getLeads() const;
388
389 bool isMangled() const { return Impl->isMangled(); }
390 void setId(EFuncId Id) { Impl->setId(Id); }
391 bool parseFuncName(StringRef &MangledName) {
392 return Impl->parseFuncName(mangledName&: MangledName);
393 }
394
395 /// Return true if it's legal to splat a scalar value passed in parameter \p
396 /// ArgIdx to a vector argument.
397 bool allowsImplicitVectorSplat(int ArgIdx) const {
398 switch (getId()) {
399 case EI_LDEXP:
400 return ArgIdx == 1;
401 case EI_FMIN:
402 case EI_FMAX:
403 return true;
404 default:
405 return false;
406 }
407 }
408
409 // Validate the call type matches the expected libfunc type.
410 bool isCompatibleSignature(const Module &M, const FunctionType *FuncTy) const;
411
412 /// \return The mangled function name for mangled library functions
413 /// and unmangled function name for unmangled library functions.
414 std::string mangle() const { return Impl->mangle(); }
415
416 void setName(StringRef N) { Impl->setName(N); }
417 void setPrefix(ENamePrefix PFX) { Impl->setPrefix(PFX); }
418
419 FunctionType *getFunctionType(const Module &M) const {
420 return Impl->getFunctionType(M);
421 }
422 static Function *getFunction(llvm::Module *M, const AMDGPULibFunc &fInfo);
423
424 static FunctionCallee getOrInsertFunction(llvm::Module *M,
425 const AMDGPULibFunc &fInfo);
426 static bool parse(StringRef MangledName, AMDGPULibFunc &Ptr);
427
428private:
429 /// Initialize as a mangled library function.
430 void initMangled();
431 std::unique_ptr<AMDGPULibFuncImpl> Impl;
432};
433
434class AMDGPUMangledLibFunc : public AMDGPULibFuncImpl {
435public:
436 Param Leads[2];
437
438 explicit AMDGPUMangledLibFunc();
439 explicit AMDGPUMangledLibFunc(EFuncId id,
440 const AMDGPUMangledLibFunc &copyFrom);
441 explicit AMDGPUMangledLibFunc(EFuncId id, FunctionType *FT,
442 bool SignedInts = true);
443
444 std::string getName() const override;
445 unsigned getNumArgs() const override;
446 FunctionType *getFunctionType(const Module &M) const override;
447
448 bool parseFuncName(StringRef &mangledName) override;
449
450 // Methods for support type inquiry through isa, cast, and dyn_cast:
451 static bool classof(const AMDGPULibFuncImpl *F) { return F->isMangled(); }
452
453 std::string mangle() const override;
454
455private:
456 std::string mangleNameItanium() const;
457
458 bool parseUnmangledName(StringRef MangledName);
459
460 template <typename Stream> void writeName(Stream &OS) const;
461};
462
463class AMDGPUUnmangledLibFunc : public AMDGPULibFuncImpl {
464 FunctionType *FuncTy;
465
466public:
467 explicit AMDGPUUnmangledLibFunc();
468 explicit AMDGPUUnmangledLibFunc(StringRef FName, FunctionType *FT) {
469 Name = std::string(FName);
470 FuncTy = FT;
471 }
472 std::string getName() const override { return Name; }
473 unsigned getNumArgs() const override;
474 FunctionType *getFunctionType(const Module &M) const override {
475 return FuncTy;
476 }
477
478 bool parseFuncName(StringRef &Name) override;
479
480 // Methods for support type inquiry through isa, cast, and dyn_cast:
481 static bool classof(const AMDGPULibFuncImpl *F) { return !F->isMangled(); }
482
483 std::string mangle() const override { return Name; }
484
485 void setFunctionType(FunctionType *FT) { FuncTy = FT; }
486};
487}
488#endif // _AMDGPU_LIBFUNC_H_
489