1//===-- AMDGPULibFunc.h ----------------------------------------*- C++ -*--===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef _AMDGPU_LIBFUNC_H_
10#define _AMDGPU_LIBFUNC_H_
11
12#include "llvm/ADT/StringRef.h"
13#include <memory>
14
15namespace llvm {
16
17struct AMDGPUOptions;
18class FunctionCallee;
19class FunctionType;
20class Function;
21class Module;
22class Type;
23
24class AMDGPULibFuncBase {
25public:
26 enum EFuncId {
27 EI_NONE,
28
29 // IMPORTANT: enums below should go in ascending by 1 value order
30 // because they are used as indexes in the mangling rules table.
31 // don't use explicit value assignment.
32 //
33 // There are two types of library functions: those with mangled
34 // name and those with unmangled name. The enums for the library
35 // functions with mangled name are defined before enums for the
36 // library functions with unmangled name. The enum for the last
37 // library function with mangled name is EI_LAST_MANGLED.
38 //
39 // Library functions with mangled name.
40 EI_ABS,
41 EI_ABS_DIFF,
42 EI_ACOS,
43 EI_ACOSH,
44 EI_ACOSPI,
45 EI_ADD_SAT,
46 EI_ALL,
47 EI_ANY,
48 EI_ASIN,
49 EI_ASINH,
50 EI_ASINPI,
51 EI_ASYNC_WORK_GROUP_COPY,
52 EI_ASYNC_WORK_GROUP_STRIDED_COPY,
53 EI_ATAN,
54 EI_ATAN2,
55 EI_ATAN2PI,
56 EI_ATANH,
57 EI_ATANPI,
58 EI_ATOMIC_ADD,
59 EI_ATOMIC_AND,
60 EI_ATOMIC_CMPXCHG,
61 EI_ATOMIC_DEC,
62 EI_ATOMIC_INC,
63 EI_ATOMIC_MAX,
64 EI_ATOMIC_MIN,
65 EI_ATOMIC_OR,
66 EI_ATOMIC_SUB,
67 EI_ATOMIC_XCHG,
68 EI_ATOMIC_XOR,
69 EI_BITSELECT,
70 EI_CBRT,
71 EI_CEIL,
72 EI_CLAMP,
73 EI_CLZ,
74 EI_COMMIT_READ_PIPE,
75 EI_COMMIT_WRITE_PIPE,
76 EI_COPYSIGN,
77 EI_COS,
78 EI_COSH,
79 EI_COSPI,
80 EI_CROSS,
81 EI_CTZ,
82 EI_DEGREES,
83 EI_DISTANCE,
84 EI_DIVIDE,
85 EI_DOT,
86 EI_ERF,
87 EI_ERFC,
88 EI_EXP,
89 EI_EXP10,
90 EI_EXP2,
91 EI_EXPM1,
92 EI_FABS,
93 EI_FAST_DISTANCE,
94 EI_FAST_LENGTH,
95 EI_FAST_NORMALIZE,
96 EI_FDIM,
97 EI_FLOOR,
98 EI_FMA,
99 EI_FMAX,
100 EI_FMIN,
101 EI_FMOD,
102 EI_FRACT,
103 EI_FREXP,
104 EI_GET_IMAGE_ARRAY_SIZE,
105 EI_GET_IMAGE_CHANNEL_DATA_TYPE,
106 EI_GET_IMAGE_CHANNEL_ORDER,
107 EI_GET_IMAGE_DIM,
108 EI_GET_IMAGE_HEIGHT,
109 EI_GET_IMAGE_WIDTH,
110 EI_GET_PIPE_MAX_PACKETS,
111 EI_GET_PIPE_NUM_PACKETS,
112 EI_HADD,
113 EI_HYPOT,
114 EI_ILOGB,
115 EI_ISEQUAL,
116 EI_ISFINITE,
117 EI_ISGREATER,
118 EI_ISGREATEREQUAL,
119 EI_ISINF,
120 EI_ISLESS,
121 EI_ISLESSEQUAL,
122 EI_ISLESSGREATER,
123 EI_ISNAN,
124 EI_ISNORMAL,
125 EI_ISNOTEQUAL,
126 EI_ISORDERED,
127 EI_ISUNORDERED,
128 EI_LDEXP,
129 EI_LENGTH,
130 EI_LGAMMA,
131 EI_LGAMMA_R,
132 EI_LOG,
133 EI_LOG10,
134 EI_LOG1P,
135 EI_LOG2,
136 EI_LOGB,
137 EI_MAD,
138 EI_MAD24,
139 EI_MAD_HI,
140 EI_MAD_SAT,
141 EI_MAX,
142 EI_MAXMAG,
143 EI_MIN,
144 EI_MINMAG,
145 EI_MIX,
146 EI_MODF,
147 EI_MUL24,
148 EI_MUL_HI,
149 EI_NAN,
150 EI_NEXTAFTER,
151 EI_NORMALIZE,
152 EI_POPCOUNT,
153 EI_POW,
154 EI_POW_FAST,
155 EI_POWN,
156 EI_POWN_FAST,
157 EI_POWR,
158 EI_POWR_FAST,
159 EI_PREFETCH,
160 EI_RADIANS,
161 EI_RECIP,
162 EI_REMAINDER,
163 EI_REMQUO,
164 EI_RESERVE_READ_PIPE,
165 EI_RESERVE_WRITE_PIPE,
166 EI_RHADD,
167 EI_RINT,
168 EI_ROOTN,
169 EI_ROOTN_FAST,
170 EI_ROTATE,
171 EI_ROUND,
172 EI_RSQRT,
173 EI_SELECT,
174 EI_SHUFFLE,
175 EI_SHUFFLE2,
176 EI_SIGN,
177 EI_SIGNBIT,
178 EI_SIN,
179 EI_SINCOS,
180 EI_SINH,
181 EI_SINPI,
182 EI_SMOOTHSTEP,
183 EI_SQRT,
184 EI_STEP,
185 EI_SUB_GROUP_BROADCAST,
186 EI_SUB_GROUP_COMMIT_READ_PIPE,
187 EI_SUB_GROUP_COMMIT_WRITE_PIPE,
188 EI_SUB_GROUP_REDUCE_ADD,
189 EI_SUB_GROUP_REDUCE_MAX,
190 EI_SUB_GROUP_REDUCE_MIN,
191 EI_SUB_GROUP_RESERVE_READ_PIPE,
192 EI_SUB_GROUP_RESERVE_WRITE_PIPE,
193 EI_SUB_GROUP_SCAN_EXCLUSIVE_ADD,
194 EI_SUB_GROUP_SCAN_EXCLUSIVE_MAX,
195 EI_SUB_GROUP_SCAN_EXCLUSIVE_MIN,
196 EI_SUB_GROUP_SCAN_INCLUSIVE_ADD,
197 EI_SUB_GROUP_SCAN_INCLUSIVE_MAX,
198 EI_SUB_GROUP_SCAN_INCLUSIVE_MIN,
199 EI_SUB_SAT,
200 EI_TAN,
201 EI_TANH,
202 EI_TANPI,
203 EI_TGAMMA,
204 EI_TRUNC,
205 EI_UPSAMPLE,
206 EI_VEC_STEP,
207 EI_VSTORE,
208 EI_VSTORE16,
209 EI_VSTORE2,
210 EI_VSTORE3,
211 EI_VSTORE4,
212 EI_VSTORE8,
213 EI_WORK_GROUP_COMMIT_READ_PIPE,
214 EI_WORK_GROUP_COMMIT_WRITE_PIPE,
215 EI_WORK_GROUP_REDUCE_ADD,
216 EI_WORK_GROUP_REDUCE_MAX,
217 EI_WORK_GROUP_REDUCE_MIN,
218 EI_WORK_GROUP_RESERVE_READ_PIPE,
219 EI_WORK_GROUP_RESERVE_WRITE_PIPE,
220 EI_WORK_GROUP_SCAN_EXCLUSIVE_ADD,
221 EI_WORK_GROUP_SCAN_EXCLUSIVE_MAX,
222 EI_WORK_GROUP_SCAN_EXCLUSIVE_MIN,
223 EI_WORK_GROUP_SCAN_INCLUSIVE_ADD,
224 EI_WORK_GROUP_SCAN_INCLUSIVE_MAX,
225 EI_WORK_GROUP_SCAN_INCLUSIVE_MIN,
226 EI_WRITE_IMAGEF,
227 EI_WRITE_IMAGEI,
228 EI_WRITE_IMAGEUI,
229 EI_NCOS,
230 EI_NEXP2,
231 EI_NFMA,
232 EI_NLOG2,
233 EI_NRCP,
234 EI_NRSQRT,
235 EI_NSIN,
236 EI_NSQRT,
237 EI_FTZ,
238 EI_FLDEXP,
239 EI_CLASS,
240 EI_RCBRT,
241 EI_LAST_MANGLED =
242 EI_RCBRT, /* The last library function with mangled name */
243
244 // Library functions with unmangled name.
245 EI_READ_PIPE_2,
246 EI_READ_PIPE_4,
247 EI_WRITE_PIPE_2,
248 EI_WRITE_PIPE_4,
249
250 EX_INTRINSICS_COUNT
251 };
252
253 enum ENamePrefix {
254 NOPFX,
255 NATIVE,
256 HALF
257 };
258
259 enum EType : unsigned char {
260 B8 = 1,
261 B16 = 2,
262 B32 = 3,
263 B64 = 4,
264 SIZE_MASK = 7,
265 FLOAT = 0x10,
266 INT = 0x20,
267 UINT = 0x30,
268 BASE_TYPE_MASK = 0x30,
269 U8 = UINT | B8,
270 U16 = UINT | B16,
271 U32 = UINT | B32,
272 U64 = UINT | B64,
273 I8 = INT | B8,
274 I16 = INT | B16,
275 I32 = INT | B32,
276 I64 = INT | B64,
277 F16 = FLOAT | B16,
278 F32 = FLOAT | B32,
279 F64 = FLOAT | B64,
280 IMG1DA = 0x80,
281 IMG1DB,
282 IMG2DA,
283 IMG1D,
284 IMG2D,
285 IMG3D,
286 SAMPLER,
287 EVENT,
288 DUMMY
289 };
290
291 enum EPtrKind {
292 BYVALUE = 0,
293 ADDR_SPACE = 0xF, // Address space takes value 0x1 ~ 0xF.
294 CONST = 0x10,
295 VOLATILE = 0x20
296 };
297
298 struct Param {
299 unsigned char ArgType = 0;
300 unsigned char VectorSize = 1;
301 unsigned char PtrKind = 0;
302
303 unsigned char Reserved = 0;
304
305 void reset() {
306 ArgType = 0;
307 VectorSize = 1;
308 PtrKind = 0;
309 }
310
311 static Param getIntN(unsigned char NumElts) {
312 return Param{.ArgType: I32, .VectorSize: NumElts, .PtrKind: 0, .Reserved: 0};
313 }
314
315 static Param getFromTy(Type *Ty, bool Signed);
316 };
317 static bool isMangled(EFuncId Id) {
318 return static_cast<unsigned>(Id) <= static_cast<unsigned>(EI_LAST_MANGLED);
319 }
320
321 static unsigned char getEPtrKindFromAddrSpace(unsigned AS) {
322 assert(((AS + 1) & ~ADDR_SPACE) == 0);
323 return static_cast<unsigned char>(AS + 1);
324 }
325
326 static unsigned getAddrSpaceFromEPtrKind(unsigned Kind) {
327 Kind = Kind & ADDR_SPACE;
328 assert(Kind >= 1);
329 return Kind - 1;
330 }
331};
332
333class AMDGPULibFuncImpl : public AMDGPULibFuncBase {
334public:
335 AMDGPULibFuncImpl() = default;
336 virtual ~AMDGPULibFuncImpl() = default;
337
338 /// Get unmangled name for mangled library function and name for unmangled
339 /// library function.
340 virtual std::string getName() const = 0;
341 virtual unsigned getNumArgs() const = 0;
342 EFuncId getId() const { return FuncId; }
343 ENamePrefix getPrefix() const { return FKind; }
344
345 bool isMangled() const { return AMDGPULibFuncBase::isMangled(Id: FuncId); }
346
347 void setId(EFuncId id) { FuncId = id; }
348 virtual bool parseFuncName(StringRef &mangledName) = 0;
349
350 /// \return The mangled function name for mangled library functions
351 /// and unmangled function name for unmangled library functions.
352 virtual std::string mangle(const AMDGPUOptions &CLOpts) const = 0;
353
354 void setName(StringRef N) { Name = std::string(N); }
355 void setPrefix(ENamePrefix pfx) { FKind = pfx; }
356
357 virtual FunctionType *getFunctionType(const Module &M) const = 0;
358
359protected:
360 EFuncId FuncId;
361 std::string Name;
362 ENamePrefix FKind = NOPFX;
363};
364
365/// Wrapper class for AMDGPULIbFuncImpl
366class AMDGPULibFunc : public AMDGPULibFuncBase {
367public:
368 explicit AMDGPULibFunc() : Impl(std::unique_ptr<AMDGPULibFuncImpl>()) {}
369 AMDGPULibFunc(const AMDGPULibFunc &F);
370 /// Clone a mangled library func with the Id \p Id and argument info from \p
371 /// CopyFrom.
372 explicit AMDGPULibFunc(EFuncId Id, const AMDGPULibFunc &CopyFrom);
373 explicit AMDGPULibFunc(EFuncId Id, FunctionType *FT, bool SignedInts);
374
375 /// Construct an unmangled library function on the fly.
376 explicit AMDGPULibFunc(StringRef FName, FunctionType *FT);
377
378 AMDGPULibFunc &operator=(const AMDGPULibFunc &F);
379
380 /// Get unmangled name for mangled library function and name for unmangled
381 /// library function.
382 std::string getName() const { return Impl->getName(); }
383 unsigned getNumArgs() const { return Impl->getNumArgs(); }
384 EFuncId getId() const { return Impl->getId(); }
385 ENamePrefix getPrefix() const { return Impl->getPrefix(); }
386 /// Get leading parameters for mangled lib functions.
387 Param *getLeads();
388 const Param *getLeads() const;
389
390 bool isMangled() const { return Impl->isMangled(); }
391 void setId(EFuncId Id) { Impl->setId(Id); }
392 bool parseFuncName(StringRef &MangledName) {
393 return Impl->parseFuncName(mangledName&: MangledName);
394 }
395
396 /// Return true if it's legal to splat a scalar value passed in parameter \p
397 /// ArgIdx to a vector argument.
398 bool allowsImplicitVectorSplat(int ArgIdx) const {
399 switch (getId()) {
400 case EI_LDEXP:
401 return ArgIdx == 1;
402 case EI_FMIN:
403 case EI_FMAX:
404 return true;
405 default:
406 return false;
407 }
408 }
409
410 // Validate the call type matches the expected libfunc type.
411 bool isCompatibleSignature(const Module &M, const FunctionType *FuncTy) const;
412
413 /// \return The mangled function name for mangled library functions
414 /// and unmangled function name for unmangled library functions.
415 std::string mangle(const AMDGPUOptions &CLOpts) const {
416 return Impl->mangle(CLOpts);
417 }
418
419 void setName(StringRef N) { Impl->setName(N); }
420 void setPrefix(ENamePrefix PFX) { Impl->setPrefix(PFX); }
421
422 FunctionType *getFunctionType(const Module &M) const {
423 return Impl->getFunctionType(M);
424 }
425 static Function *getFunction(llvm::Module *M, const AMDGPULibFunc &fInfo);
426
427 static FunctionCallee getOrInsertFunction(llvm::Module *M,
428 const AMDGPULibFunc &fInfo);
429 static bool parse(StringRef MangledName, AMDGPULibFunc &Ptr);
430
431private:
432 /// Initialize as a mangled library function.
433 void initMangled();
434 std::unique_ptr<AMDGPULibFuncImpl> Impl;
435};
436
437class AMDGPUMangledLibFunc : public AMDGPULibFuncImpl {
438public:
439 Param Leads[2];
440
441 explicit AMDGPUMangledLibFunc();
442 explicit AMDGPUMangledLibFunc(EFuncId id,
443 const AMDGPUMangledLibFunc &copyFrom);
444 explicit AMDGPUMangledLibFunc(EFuncId id, FunctionType *FT,
445 bool SignedInts = true);
446
447 std::string getName() const override;
448 unsigned getNumArgs() const override;
449 FunctionType *getFunctionType(const Module &M) const override;
450
451 bool parseFuncName(StringRef &mangledName) override;
452
453 // Methods for support type inquiry through isa, cast, and dyn_cast:
454 static bool classof(const AMDGPULibFuncImpl *F) { return F->isMangled(); }
455
456 std::string mangle(const AMDGPUOptions &CLOpts) const override;
457
458private:
459 std::string mangleNameItanium(const AMDGPUOptions &CLOpts) const;
460
461 bool parseUnmangledName(StringRef MangledName);
462
463 template <typename Stream> void writeName(Stream &OS) const;
464};
465
466class AMDGPUUnmangledLibFunc : public AMDGPULibFuncImpl {
467 FunctionType *FuncTy;
468
469public:
470 explicit AMDGPUUnmangledLibFunc();
471 explicit AMDGPUUnmangledLibFunc(StringRef FName, FunctionType *FT) {
472 Name = std::string(FName);
473 FuncTy = FT;
474 }
475 std::string getName() const override { return Name; }
476 unsigned getNumArgs() const override;
477 FunctionType *getFunctionType(const Module &M) const override {
478 return FuncTy;
479 }
480
481 bool parseFuncName(StringRef &Name) override;
482
483 // Methods for support type inquiry through isa, cast, and dyn_cast:
484 static bool classof(const AMDGPULibFuncImpl *F) { return !F->isMangled(); }
485
486 std::string mangle(const AMDGPUOptions &) const override { return Name; }
487
488 void setFunctionType(FunctionType *FT) { FuncTy = FT; }
489};
490}
491#endif // _AMDGPU_LIBFUNC_H_
492