1//===-- AMDGPULibFunc.h ----------------------------------------*- C++ -*--===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef _AMDGPU_LIBFUNC_H_
10#define _AMDGPU_LIBFUNC_H_
11
12#include "llvm/ADT/StringRef.h"
13#include <memory>
14
15namespace llvm {
16
17class FunctionCallee;
18class FunctionType;
19class Function;
20class Module;
21class Type;
22
23class AMDGPULibFuncBase {
24public:
25 enum EFuncId {
26 EI_NONE,
27
28 // IMPORTANT: enums below should go in ascending by 1 value order
29 // because they are used as indexes in the mangling rules table.
30 // don't use explicit value assignment.
31 //
32 // There are two types of library functions: those with mangled
33 // name and those with unmangled name. The enums for the library
34 // functions with mangled name are defined before enums for the
35 // library functions with unmangled name. The enum for the last
36 // library function with mangled name is EI_LAST_MANGLED.
37 //
38 // Library functions with mangled name.
39 EI_ABS,
40 EI_ABS_DIFF,
41 EI_ACOS,
42 EI_ACOSH,
43 EI_ACOSPI,
44 EI_ADD_SAT,
45 EI_ALL,
46 EI_ANY,
47 EI_ASIN,
48 EI_ASINH,
49 EI_ASINPI,
50 EI_ASYNC_WORK_GROUP_COPY,
51 EI_ASYNC_WORK_GROUP_STRIDED_COPY,
52 EI_ATAN,
53 EI_ATAN2,
54 EI_ATAN2PI,
55 EI_ATANH,
56 EI_ATANPI,
57 EI_ATOMIC_ADD,
58 EI_ATOMIC_AND,
59 EI_ATOMIC_CMPXCHG,
60 EI_ATOMIC_DEC,
61 EI_ATOMIC_INC,
62 EI_ATOMIC_MAX,
63 EI_ATOMIC_MIN,
64 EI_ATOMIC_OR,
65 EI_ATOMIC_SUB,
66 EI_ATOMIC_XCHG,
67 EI_ATOMIC_XOR,
68 EI_BITSELECT,
69 EI_CBRT,
70 EI_CEIL,
71 EI_CLAMP,
72 EI_CLZ,
73 EI_COMMIT_READ_PIPE,
74 EI_COMMIT_WRITE_PIPE,
75 EI_COPYSIGN,
76 EI_COS,
77 EI_COSH,
78 EI_COSPI,
79 EI_CROSS,
80 EI_CTZ,
81 EI_DEGREES,
82 EI_DISTANCE,
83 EI_DIVIDE,
84 EI_DOT,
85 EI_ERF,
86 EI_ERFC,
87 EI_EXP,
88 EI_EXP10,
89 EI_EXP2,
90 EI_EXPM1,
91 EI_FABS,
92 EI_FAST_DISTANCE,
93 EI_FAST_LENGTH,
94 EI_FAST_NORMALIZE,
95 EI_FDIM,
96 EI_FLOOR,
97 EI_FMA,
98 EI_FMAX,
99 EI_FMIN,
100 EI_FMOD,
101 EI_FRACT,
102 EI_FREXP,
103 EI_GET_IMAGE_ARRAY_SIZE,
104 EI_GET_IMAGE_CHANNEL_DATA_TYPE,
105 EI_GET_IMAGE_CHANNEL_ORDER,
106 EI_GET_IMAGE_DIM,
107 EI_GET_IMAGE_HEIGHT,
108 EI_GET_IMAGE_WIDTH,
109 EI_GET_PIPE_MAX_PACKETS,
110 EI_GET_PIPE_NUM_PACKETS,
111 EI_HADD,
112 EI_HYPOT,
113 EI_ILOGB,
114 EI_ISEQUAL,
115 EI_ISFINITE,
116 EI_ISGREATER,
117 EI_ISGREATEREQUAL,
118 EI_ISINF,
119 EI_ISLESS,
120 EI_ISLESSEQUAL,
121 EI_ISLESSGREATER,
122 EI_ISNAN,
123 EI_ISNORMAL,
124 EI_ISNOTEQUAL,
125 EI_ISORDERED,
126 EI_ISUNORDERED,
127 EI_LDEXP,
128 EI_LENGTH,
129 EI_LGAMMA,
130 EI_LGAMMA_R,
131 EI_LOG,
132 EI_LOG10,
133 EI_LOG1P,
134 EI_LOG2,
135 EI_LOGB,
136 EI_MAD,
137 EI_MAD24,
138 EI_MAD_HI,
139 EI_MAD_SAT,
140 EI_MAX,
141 EI_MAXMAG,
142 EI_MIN,
143 EI_MINMAG,
144 EI_MIX,
145 EI_MODF,
146 EI_MUL24,
147 EI_MUL_HI,
148 EI_NAN,
149 EI_NEXTAFTER,
150 EI_NORMALIZE,
151 EI_POPCOUNT,
152 EI_POW,
153 EI_POW_FAST,
154 EI_POWN,
155 EI_POWN_FAST,
156 EI_POWR,
157 EI_POWR_FAST,
158 EI_PREFETCH,
159 EI_RADIANS,
160 EI_RECIP,
161 EI_REMAINDER,
162 EI_REMQUO,
163 EI_RESERVE_READ_PIPE,
164 EI_RESERVE_WRITE_PIPE,
165 EI_RHADD,
166 EI_RINT,
167 EI_ROOTN,
168 EI_ROOTN_FAST,
169 EI_ROTATE,
170 EI_ROUND,
171 EI_RSQRT,
172 EI_SELECT,
173 EI_SHUFFLE,
174 EI_SHUFFLE2,
175 EI_SIGN,
176 EI_SIGNBIT,
177 EI_SIN,
178 EI_SINCOS,
179 EI_SINH,
180 EI_SINPI,
181 EI_SMOOTHSTEP,
182 EI_SQRT,
183 EI_STEP,
184 EI_SUB_GROUP_BROADCAST,
185 EI_SUB_GROUP_COMMIT_READ_PIPE,
186 EI_SUB_GROUP_COMMIT_WRITE_PIPE,
187 EI_SUB_GROUP_REDUCE_ADD,
188 EI_SUB_GROUP_REDUCE_MAX,
189 EI_SUB_GROUP_REDUCE_MIN,
190 EI_SUB_GROUP_RESERVE_READ_PIPE,
191 EI_SUB_GROUP_RESERVE_WRITE_PIPE,
192 EI_SUB_GROUP_SCAN_EXCLUSIVE_ADD,
193 EI_SUB_GROUP_SCAN_EXCLUSIVE_MAX,
194 EI_SUB_GROUP_SCAN_EXCLUSIVE_MIN,
195 EI_SUB_GROUP_SCAN_INCLUSIVE_ADD,
196 EI_SUB_GROUP_SCAN_INCLUSIVE_MAX,
197 EI_SUB_GROUP_SCAN_INCLUSIVE_MIN,
198 EI_SUB_SAT,
199 EI_TAN,
200 EI_TANH,
201 EI_TANPI,
202 EI_TGAMMA,
203 EI_TRUNC,
204 EI_UPSAMPLE,
205 EI_VEC_STEP,
206 EI_VSTORE,
207 EI_VSTORE16,
208 EI_VSTORE2,
209 EI_VSTORE3,
210 EI_VSTORE4,
211 EI_VSTORE8,
212 EI_WORK_GROUP_COMMIT_READ_PIPE,
213 EI_WORK_GROUP_COMMIT_WRITE_PIPE,
214 EI_WORK_GROUP_REDUCE_ADD,
215 EI_WORK_GROUP_REDUCE_MAX,
216 EI_WORK_GROUP_REDUCE_MIN,
217 EI_WORK_GROUP_RESERVE_READ_PIPE,
218 EI_WORK_GROUP_RESERVE_WRITE_PIPE,
219 EI_WORK_GROUP_SCAN_EXCLUSIVE_ADD,
220 EI_WORK_GROUP_SCAN_EXCLUSIVE_MAX,
221 EI_WORK_GROUP_SCAN_EXCLUSIVE_MIN,
222 EI_WORK_GROUP_SCAN_INCLUSIVE_ADD,
223 EI_WORK_GROUP_SCAN_INCLUSIVE_MAX,
224 EI_WORK_GROUP_SCAN_INCLUSIVE_MIN,
225 EI_WRITE_IMAGEF,
226 EI_WRITE_IMAGEI,
227 EI_WRITE_IMAGEUI,
228 EI_NCOS,
229 EI_NEXP2,
230 EI_NFMA,
231 EI_NLOG2,
232 EI_NRCP,
233 EI_NRSQRT,
234 EI_NSIN,
235 EI_NSQRT,
236 EI_FTZ,
237 EI_FLDEXP,
238 EI_CLASS,
239 EI_RCBRT,
240 EI_LAST_MANGLED =
241 EI_RCBRT, /* The last library function with mangled name */
242
243 // Library functions with unmangled name.
244 EI_READ_PIPE_2,
245 EI_READ_PIPE_4,
246 EI_WRITE_PIPE_2,
247 EI_WRITE_PIPE_4,
248
249 EX_INTRINSICS_COUNT
250 };
251
252 enum ENamePrefix {
253 NOPFX,
254 NATIVE,
255 HALF
256 };
257
258 enum EType {
259 B8 = 1,
260 B16 = 2,
261 B32 = 3,
262 B64 = 4,
263 SIZE_MASK = 7,
264 FLOAT = 0x10,
265 INT = 0x20,
266 UINT = 0x30,
267 BASE_TYPE_MASK = 0x30,
268 U8 = UINT | B8,
269 U16 = UINT | B16,
270 U32 = UINT | B32,
271 U64 = UINT | B64,
272 I8 = INT | B8,
273 I16 = INT | B16,
274 I32 = INT | B32,
275 I64 = INT | B64,
276 F16 = FLOAT | B16,
277 F32 = FLOAT | B32,
278 F64 = FLOAT | B64,
279 IMG1DA = 0x80,
280 IMG1DB,
281 IMG2DA,
282 IMG1D,
283 IMG2D,
284 IMG3D,
285 SAMPLER,
286 EVENT,
287 DUMMY
288 };
289
290 enum EPtrKind {
291 BYVALUE = 0,
292 ADDR_SPACE = 0xF, // Address space takes value 0x1 ~ 0xF.
293 CONST = 0x10,
294 VOLATILE = 0x20
295 };
296
297 struct Param {
298 unsigned char ArgType = 0;
299 unsigned char VectorSize = 1;
300 unsigned char PtrKind = 0;
301
302 unsigned char Reserved = 0;
303
304 void reset() {
305 ArgType = 0;
306 VectorSize = 1;
307 PtrKind = 0;
308 }
309
310 static Param getIntN(unsigned char NumElts) {
311 return Param{.ArgType: I32, .VectorSize: NumElts, .PtrKind: 0, .Reserved: 0};
312 }
313
314 static Param getFromTy(Type *Ty, bool Signed);
315
316 template <typename Stream>
317 void mangleItanium(Stream& os);
318 };
319 static bool isMangled(EFuncId Id) {
320 return static_cast<unsigned>(Id) <= static_cast<unsigned>(EI_LAST_MANGLED);
321 }
322
323 static unsigned getEPtrKindFromAddrSpace(unsigned AS) {
324 assert(((AS + 1) & ~ADDR_SPACE) == 0);
325 return AS + 1;
326 }
327
328 static unsigned getAddrSpaceFromEPtrKind(unsigned Kind) {
329 Kind = Kind & ADDR_SPACE;
330 assert(Kind >= 1);
331 return Kind - 1;
332 }
333};
334
335class AMDGPULibFuncImpl : public AMDGPULibFuncBase {
336public:
337 AMDGPULibFuncImpl() = default;
338 virtual ~AMDGPULibFuncImpl() = default;
339
340 /// Get unmangled name for mangled library function and name for unmangled
341 /// library function.
342 virtual std::string getName() const = 0;
343 virtual unsigned getNumArgs() const = 0;
344 EFuncId getId() const { return FuncId; }
345 ENamePrefix getPrefix() const { return FKind; }
346
347 bool isMangled() const { return AMDGPULibFuncBase::isMangled(Id: FuncId); }
348
349 void setId(EFuncId id) { FuncId = id; }
350 virtual bool parseFuncName(StringRef &mangledName) = 0;
351
352 /// \return The mangled function name for mangled library functions
353 /// and unmangled function name for unmangled library functions.
354 virtual std::string mangle() const = 0;
355
356 void setName(StringRef N) { Name = std::string(N); }
357 void setPrefix(ENamePrefix pfx) { FKind = pfx; }
358
359 virtual FunctionType *getFunctionType(const Module &M) const = 0;
360
361protected:
362 EFuncId FuncId;
363 std::string Name;
364 ENamePrefix FKind = NOPFX;
365};
366
367/// Wrapper class for AMDGPULIbFuncImpl
368class AMDGPULibFunc : public AMDGPULibFuncBase {
369public:
370 explicit AMDGPULibFunc() : Impl(std::unique_ptr<AMDGPULibFuncImpl>()) {}
371 AMDGPULibFunc(const AMDGPULibFunc &F);
372 /// Clone a mangled library func with the Id \p Id and argument info from \p
373 /// CopyFrom.
374 explicit AMDGPULibFunc(EFuncId Id, const AMDGPULibFunc &CopyFrom);
375 explicit AMDGPULibFunc(EFuncId Id, FunctionType *FT, bool SignedInts);
376
377 /// Construct an unmangled library function on the fly.
378 explicit AMDGPULibFunc(StringRef FName, FunctionType *FT);
379
380 AMDGPULibFunc &operator=(const AMDGPULibFunc &F);
381
382 /// Get unmangled name for mangled library function and name for unmangled
383 /// library function.
384 std::string getName() const { return Impl->getName(); }
385 unsigned getNumArgs() const { return Impl->getNumArgs(); }
386 EFuncId getId() const { return Impl->getId(); }
387 ENamePrefix getPrefix() const { return Impl->getPrefix(); }
388 /// Get leading parameters for mangled lib functions.
389 Param *getLeads();
390 const Param *getLeads() const;
391
392 bool isMangled() const { return Impl->isMangled(); }
393 void setId(EFuncId Id) { Impl->setId(Id); }
394 bool parseFuncName(StringRef &MangledName) {
395 return Impl->parseFuncName(mangledName&: MangledName);
396 }
397
398 /// Return true if it's legal to splat a scalar value passed in parameter \p
399 /// ArgIdx to a vector argument.
400 bool allowsImplicitVectorSplat(int ArgIdx) const {
401 switch (getId()) {
402 case EI_LDEXP:
403 return ArgIdx == 1;
404 case EI_FMIN:
405 case EI_FMAX:
406 return true;
407 default:
408 return false;
409 }
410 }
411
412 // Validate the call type matches the expected libfunc type.
413 bool isCompatibleSignature(const Module &M, const FunctionType *FuncTy) const;
414
415 /// \return The mangled function name for mangled library functions
416 /// and unmangled function name for unmangled library functions.
417 std::string mangle() const { return Impl->mangle(); }
418
419 void setName(StringRef N) { Impl->setName(N); }
420 void setPrefix(ENamePrefix PFX) { Impl->setPrefix(PFX); }
421
422 FunctionType *getFunctionType(const Module &M) const {
423 return Impl->getFunctionType(M);
424 }
425 static Function *getFunction(llvm::Module *M, const AMDGPULibFunc &fInfo);
426
427 static FunctionCallee getOrInsertFunction(llvm::Module *M,
428 const AMDGPULibFunc &fInfo);
429 static bool parse(StringRef MangledName, AMDGPULibFunc &Ptr);
430
431private:
432 /// Initialize as a mangled library function.
433 void initMangled();
434 std::unique_ptr<AMDGPULibFuncImpl> Impl;
435};
436
437class AMDGPUMangledLibFunc : public AMDGPULibFuncImpl {
438public:
439 Param Leads[2];
440
441 explicit AMDGPUMangledLibFunc();
442 explicit AMDGPUMangledLibFunc(EFuncId id,
443 const AMDGPUMangledLibFunc &copyFrom);
444 explicit AMDGPUMangledLibFunc(EFuncId id, FunctionType *FT,
445 bool SignedInts = true);
446
447 std::string getName() const override;
448 unsigned getNumArgs() const override;
449 FunctionType *getFunctionType(const Module &M) const override;
450 static StringRef getUnmangledName(StringRef MangledName);
451
452 bool parseFuncName(StringRef &mangledName) override;
453
454 // Methods for support type inquiry through isa, cast, and dyn_cast:
455 static bool classof(const AMDGPULibFuncImpl *F) { return F->isMangled(); }
456
457 std::string mangle() const override;
458
459private:
460 std::string mangleNameItanium() const;
461
462 std::string mangleName(StringRef Name) const;
463 bool parseUnmangledName(StringRef MangledName);
464
465 template <typename Stream> void writeName(Stream &OS) const;
466};
467
468class AMDGPUUnmangledLibFunc : public AMDGPULibFuncImpl {
469 FunctionType *FuncTy;
470
471public:
472 explicit AMDGPUUnmangledLibFunc();
473 explicit AMDGPUUnmangledLibFunc(StringRef FName, FunctionType *FT) {
474 Name = std::string(FName);
475 FuncTy = FT;
476 }
477 std::string getName() const override { return Name; }
478 unsigned getNumArgs() const override;
479 FunctionType *getFunctionType(const Module &M) const override {
480 return FuncTy;
481 }
482
483 bool parseFuncName(StringRef &Name) override;
484
485 // Methods for support type inquiry through isa, cast, and dyn_cast:
486 static bool classof(const AMDGPULibFuncImpl *F) { return !F->isMangled(); }
487
488 std::string mangle() const override { return Name; }
489
490 void setFunctionType(FunctionType *FT) { FuncTy = FT; }
491};
492}
493#endif // _AMDGPU_LIBFUNC_H_
494