1//===-- AMDGPULibFunc.cpp -------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains utility functions to work with Itanium mangled names
10//
11//===----------------------------------------------------------------------===//
12
13#include "AMDGPULibFunc.h"
14#include "AMDGPU.h"
15#include "llvm/ADT/StringExtras.h"
16#include "llvm/ADT/StringMap.h"
17#include "llvm/ADT/StringSwitch.h"
18#include "llvm/IR/DerivedTypes.h"
19#include "llvm/IR/Function.h"
20#include "llvm/IR/Module.h"
21#include "llvm/IR/ValueSymbolTable.h"
22#include "llvm/Support/CommandLine.h"
23#include "llvm/Support/ModRef.h"
24#include "llvm/Support/raw_ostream.h"
25
26using namespace llvm;
27
28static cl::opt<bool> EnableOCLManglingMismatchWA(
29 "amdgpu-enable-ocl-mangling-mismatch-workaround", cl::init(Val: true),
30 cl::ReallyHidden,
31 cl::desc("Enable the workaround for OCL name mangling mismatch."));
32
33namespace {
34
35enum EManglingParam {
36 E_NONE,
37 EX_EVENT,
38 EX_FLOAT4,
39 EX_INTV4,
40 EX_RESERVEDID,
41 EX_SAMPLER,
42 EX_SIZET,
43 EX_UINT,
44 EX_UINTV4,
45 E_ANY,
46 E_CONSTPTR_ANY,
47 E_CONSTPTR_SWAPGL,
48 E_COPY,
49 E_IMAGECOORDS,
50 E_POINTEE,
51 E_SETBASE_I32,
52 E_SETBASE_U32,
53 E_MAKEBASE_UNS,
54 E_V16_OF_POINTEE,
55 E_V2_OF_POINTEE,
56 E_V3_OF_POINTEE,
57 E_V4_OF_POINTEE,
58 E_V8_OF_POINTEE,
59 E_VLTLPTR_ANY,
60};
61
62struct ManglingRule {
63 const char *Name;
64 unsigned char Lead[2];
65 unsigned char Param[5];
66
67 int maxLeadIndex() const { return (std::max)(a: Lead[0], b: Lead[1]); }
68
69 unsigned getNumArgs() const;
70
71 static StringMap<int> buildManglingRulesMap();
72};
73
74// Information about library functions with unmangled names.
75class UnmangledFuncInfo {
76 const char *Name;
77 unsigned NumArgs;
78
79 // Table for all lib functions with unmangled names.
80 static const UnmangledFuncInfo Table[];
81
82 // Number of entries in Table.
83 static const unsigned TableSize;
84
85 static StringMap<unsigned> buildNameMap();
86
87public:
88 using ID = AMDGPULibFunc::EFuncId;
89 constexpr UnmangledFuncInfo(const char *_Name, unsigned _NumArgs)
90 : Name(_Name), NumArgs(_NumArgs) {}
91 // Get index to Table by function name.
92 static bool lookup(StringRef Name, ID &Id);
93 static unsigned toIndex(ID Id) {
94 assert(static_cast<unsigned>(Id) >
95 static_cast<unsigned>(AMDGPULibFunc::EI_LAST_MANGLED) &&
96 "Invalid unmangled library function");
97 return static_cast<unsigned>(Id) - 1 -
98 static_cast<unsigned>(AMDGPULibFunc::EI_LAST_MANGLED);
99 }
100 static ID toFuncId(unsigned Index) {
101 assert(Index < TableSize &&
102 "Invalid unmangled library function");
103 return static_cast<ID>(
104 Index + 1 + static_cast<unsigned>(AMDGPULibFunc::EI_LAST_MANGLED));
105 }
106 static unsigned getNumArgs(ID Id) { return Table[toIndex(Id)].NumArgs; }
107};
108
109unsigned ManglingRule::getNumArgs() const {
110 unsigned I=0;
111 while (I < (sizeof Param/sizeof Param[0]) && Param[I]) ++I;
112 return I;
113}
114
115// This table describes function formal argument type rules. The order of rules
116// corresponds to the EFuncId enum at AMDGPULibFunc.h
117//
118// "<func name>", { <leads> }, { <param rules> }
119// where:
120// <leads> - list of integers that are one-based indexes of formal argument
121// used to mangle a function name. Other argument types are derived from types
122// of these 'leads'. The order of integers in this list correspond to the
123// order in which these arguments are mangled in the EDG mangling scheme. The
124// same order should be preserved for arguments in the AMDGPULibFunc structure
125// when it is used for mangling. For example:
126// { "vstorea_half", {3,1}, {E_ANY,EX_SIZET,E_ANY}},
127// will be mangled in EDG scheme as vstorea_half_<3dparam>_<1stparam>
128// When mangling from code use:
129// AMDGPULibFunc insc;
130// insc.param[0] = ... // describe 3rd parameter
131// insc.param[1] = ... // describe 1rd parameter
132//
133// <param rules> - list of rules used to derive all of the function formal
134// argument types. EX_ prefixed are simple types, other derived from the
135// latest 'lead' argument type in the order of encoding from first to last.
136// E_ANY - use prev lead type, E_CONSTPTR_ANY - make const pointer out of
137// prev lead type, etc. see ParamIterator::getNextParam() for details.
138
139static constexpr ManglingRule manglingRules[] = {
140{ .Name: "", .Lead: {0}, .Param: {0} },
141{ .Name: "abs" , .Lead: {1}, .Param: {E_ANY}},
142{ .Name: "abs_diff" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
143{ .Name: "acos" , .Lead: {1}, .Param: {E_ANY}},
144{ .Name: "acosh" , .Lead: {1}, .Param: {E_ANY}},
145{ .Name: "acospi" , .Lead: {1}, .Param: {E_ANY}},
146{ .Name: "add_sat" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
147{ .Name: "all" , .Lead: {1}, .Param: {E_ANY}},
148{ .Name: "any" , .Lead: {1}, .Param: {E_ANY}},
149{ .Name: "asin" , .Lead: {1}, .Param: {E_ANY}},
150{ .Name: "asinh" , .Lead: {1}, .Param: {E_ANY}},
151{ .Name: "asinpi" , .Lead: {1}, .Param: {E_ANY}},
152{ .Name: "async_work_group_copy" , .Lead: {1}, .Param: {E_ANY,E_CONSTPTR_SWAPGL,EX_SIZET,EX_EVENT}},
153{ .Name: "async_work_group_strided_copy" , .Lead: {1}, .Param: {E_ANY,E_CONSTPTR_SWAPGL,EX_SIZET,EX_SIZET,EX_EVENT}},
154{ .Name: "atan" , .Lead: {1}, .Param: {E_ANY}},
155{ .Name: "atan2" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
156{ .Name: "atan2pi" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
157{ .Name: "atanh" , .Lead: {1}, .Param: {E_ANY}},
158{ .Name: "atanpi" , .Lead: {1}, .Param: {E_ANY}},
159{ .Name: "atomic_add" , .Lead: {1}, .Param: {E_VLTLPTR_ANY,E_POINTEE}},
160{ .Name: "atomic_and" , .Lead: {1}, .Param: {E_VLTLPTR_ANY,E_POINTEE}},
161{ .Name: "atomic_cmpxchg" , .Lead: {1}, .Param: {E_VLTLPTR_ANY,E_POINTEE,E_POINTEE}},
162{ .Name: "atomic_dec" , .Lead: {1}, .Param: {E_VLTLPTR_ANY}},
163{ .Name: "atomic_inc" , .Lead: {1}, .Param: {E_VLTLPTR_ANY}},
164{ .Name: "atomic_max" , .Lead: {1}, .Param: {E_VLTLPTR_ANY,E_POINTEE}},
165{ .Name: "atomic_min" , .Lead: {1}, .Param: {E_VLTLPTR_ANY,E_POINTEE}},
166{ .Name: "atomic_or" , .Lead: {1}, .Param: {E_VLTLPTR_ANY,E_POINTEE}},
167{ .Name: "atomic_sub" , .Lead: {1}, .Param: {E_VLTLPTR_ANY,E_POINTEE}},
168{ .Name: "atomic_xchg" , .Lead: {1}, .Param: {E_VLTLPTR_ANY,E_POINTEE}},
169{ .Name: "atomic_xor" , .Lead: {1}, .Param: {E_VLTLPTR_ANY,E_POINTEE}},
170{ .Name: "bitselect" , .Lead: {1}, .Param: {E_ANY,E_COPY,E_COPY}},
171{ .Name: "cbrt" , .Lead: {1}, .Param: {E_ANY}},
172{ .Name: "ceil" , .Lead: {1}, .Param: {E_ANY}},
173{ .Name: "clamp" , .Lead: {1}, .Param: {E_ANY,E_COPY,E_COPY}},
174{ .Name: "clz" , .Lead: {1}, .Param: {E_ANY}},
175{ .Name: "commit_read_pipe" , .Lead: {1}, .Param: {E_ANY,EX_RESERVEDID}},
176{ .Name: "commit_write_pipe" , .Lead: {1}, .Param: {E_ANY,EX_RESERVEDID}},
177{ .Name: "copysign" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
178{ .Name: "cos" , .Lead: {1}, .Param: {E_ANY}},
179{ .Name: "cosh" , .Lead: {1}, .Param: {E_ANY}},
180{ .Name: "cospi" , .Lead: {1}, .Param: {E_ANY}},
181{ .Name: "cross" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
182{ .Name: "ctz" , .Lead: {1}, .Param: {E_ANY}},
183{ .Name: "degrees" , .Lead: {1}, .Param: {E_ANY}},
184{ .Name: "distance" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
185{ .Name: "divide" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
186{ .Name: "dot" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
187{ .Name: "erf" , .Lead: {1}, .Param: {E_ANY}},
188{ .Name: "erfc" , .Lead: {1}, .Param: {E_ANY}},
189{ .Name: "exp" , .Lead: {1}, .Param: {E_ANY}},
190{ .Name: "exp10" , .Lead: {1}, .Param: {E_ANY}},
191{ .Name: "exp2" , .Lead: {1}, .Param: {E_ANY}},
192{ .Name: "expm1" , .Lead: {1}, .Param: {E_ANY}},
193{ .Name: "fabs" , .Lead: {1}, .Param: {E_ANY}},
194{ .Name: "fast_distance" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
195{ .Name: "fast_length" , .Lead: {1}, .Param: {E_ANY}},
196{ .Name: "fast_normalize" , .Lead: {1}, .Param: {E_ANY}},
197{ .Name: "fdim" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
198{ .Name: "floor" , .Lead: {1}, .Param: {E_ANY}},
199{ .Name: "fma" , .Lead: {1}, .Param: {E_ANY,E_COPY,E_COPY}},
200{ .Name: "fmax" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
201{ .Name: "fmin" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
202{ .Name: "fmod" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
203{ .Name: "fract" , .Lead: {2}, .Param: {E_POINTEE,E_ANY}},
204{ .Name: "frexp" , .Lead: {1,2}, .Param: {E_ANY,E_ANY}},
205{ .Name: "get_image_array_size" , .Lead: {1}, .Param: {E_ANY}},
206{ .Name: "get_image_channel_data_type" , .Lead: {1}, .Param: {E_ANY}},
207{ .Name: "get_image_channel_order" , .Lead: {1}, .Param: {E_ANY}},
208{ .Name: "get_image_dim" , .Lead: {1}, .Param: {E_ANY}},
209{ .Name: "get_image_height" , .Lead: {1}, .Param: {E_ANY}},
210{ .Name: "get_image_width" , .Lead: {1}, .Param: {E_ANY}},
211{ .Name: "get_pipe_max_packets" , .Lead: {1}, .Param: {E_ANY}},
212{ .Name: "get_pipe_num_packets" , .Lead: {1}, .Param: {E_ANY}},
213{ .Name: "hadd" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
214{ .Name: "hypot" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
215{ .Name: "ilogb" , .Lead: {1}, .Param: {E_ANY}},
216{ .Name: "isequal" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
217{ .Name: "isfinite" , .Lead: {1}, .Param: {E_ANY}},
218{ .Name: "isgreater" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
219{ .Name: "isgreaterequal" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
220{ .Name: "isinf" , .Lead: {1}, .Param: {E_ANY}},
221{ .Name: "isless" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
222{ .Name: "islessequal" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
223{ .Name: "islessgreater" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
224{ .Name: "isnan" , .Lead: {1}, .Param: {E_ANY}},
225{ .Name: "isnormal" , .Lead: {1}, .Param: {E_ANY}},
226{ .Name: "isnotequal" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
227{ .Name: "isordered" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
228{ .Name: "isunordered" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
229{ .Name: "ldexp" , .Lead: {1}, .Param: {E_ANY,E_SETBASE_I32}},
230{ .Name: "length" , .Lead: {1}, .Param: {E_ANY}},
231{ .Name: "lgamma" , .Lead: {1}, .Param: {E_ANY}},
232{ .Name: "lgamma_r" , .Lead: {1,2}, .Param: {E_ANY,E_ANY}},
233{ .Name: "log" , .Lead: {1}, .Param: {E_ANY}},
234{ .Name: "log10" , .Lead: {1}, .Param: {E_ANY}},
235{ .Name: "log1p" , .Lead: {1}, .Param: {E_ANY}},
236{ .Name: "log2" , .Lead: {1}, .Param: {E_ANY}},
237{ .Name: "logb" , .Lead: {1}, .Param: {E_ANY}},
238{ .Name: "mad" , .Lead: {1}, .Param: {E_ANY,E_COPY,E_COPY}},
239{ .Name: "mad24" , .Lead: {1}, .Param: {E_ANY,E_COPY,E_COPY}},
240{ .Name: "mad_hi" , .Lead: {1}, .Param: {E_ANY,E_COPY,E_COPY}},
241{ .Name: "mad_sat" , .Lead: {1}, .Param: {E_ANY,E_COPY,E_COPY}},
242{ .Name: "max" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
243{ .Name: "maxmag" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
244{ .Name: "min" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
245{ .Name: "minmag" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
246{ .Name: "mix" , .Lead: {1}, .Param: {E_ANY,E_COPY,E_COPY}},
247{ .Name: "modf" , .Lead: {2}, .Param: {E_POINTEE,E_ANY}},
248{ .Name: "mul24" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
249{ .Name: "mul_hi" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
250{ .Name: "nan" , .Lead: {1}, .Param: {E_ANY}},
251{ .Name: "nextafter" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
252{ .Name: "normalize" , .Lead: {1}, .Param: {E_ANY}},
253{ .Name: "popcount" , .Lead: {1}, .Param: {E_ANY}},
254{ .Name: "pow" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
255{ .Name: "__pow_fast" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
256{ .Name: "pown" , .Lead: {1}, .Param: {E_ANY,E_SETBASE_I32}},
257{ .Name: "__pown_fast" , .Lead: {1}, .Param: {E_ANY,E_SETBASE_I32}},
258{ .Name: "powr" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
259{ .Name: "__powr_fast" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
260{ .Name: "prefetch" , .Lead: {1}, .Param: {E_CONSTPTR_ANY,EX_SIZET}},
261{ .Name: "radians" , .Lead: {1}, .Param: {E_ANY}},
262{ .Name: "recip" , .Lead: {1}, .Param: {E_ANY}},
263{ .Name: "remainder" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
264{ .Name: "remquo" , .Lead: {1,3}, .Param: {E_ANY,E_COPY,E_ANY}},
265{ .Name: "reserve_read_pipe" , .Lead: {1}, .Param: {E_ANY,EX_UINT}},
266{ .Name: "reserve_write_pipe" , .Lead: {1}, .Param: {E_ANY,EX_UINT}},
267{ .Name: "rhadd" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
268{ .Name: "rint" , .Lead: {1}, .Param: {E_ANY}},
269{ .Name: "rootn" , .Lead: {1}, .Param: {E_ANY,E_SETBASE_I32}},
270{ .Name: "__rootn_fast" , .Lead: {1}, .Param: {E_ANY,E_SETBASE_I32}},
271{ .Name: "rotate" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
272{ .Name: "round" , .Lead: {1}, .Param: {E_ANY}},
273{ .Name: "rsqrt" , .Lead: {1}, .Param: {E_ANY}},
274{ .Name: "select" , .Lead: {1,3}, .Param: {E_ANY,E_COPY,E_ANY}},
275{ .Name: "shuffle" , .Lead: {1,2}, .Param: {E_ANY,E_ANY}},
276{ .Name: "shuffle2" , .Lead: {1,3}, .Param: {E_ANY,E_COPY,E_ANY}},
277{ .Name: "sign" , .Lead: {1}, .Param: {E_ANY}},
278{ .Name: "signbit" , .Lead: {1}, .Param: {E_ANY}},
279{ .Name: "sin" , .Lead: {1}, .Param: {E_ANY}},
280{ .Name: "sincos" , .Lead: {2}, .Param: {E_POINTEE,E_ANY}},
281{ .Name: "sinh" , .Lead: {1}, .Param: {E_ANY}},
282{ .Name: "sinpi" , .Lead: {1}, .Param: {E_ANY}},
283{ .Name: "smoothstep" , .Lead: {1}, .Param: {E_ANY,E_COPY,E_COPY}},
284{ .Name: "sqrt" , .Lead: {1}, .Param: {E_ANY}},
285{ .Name: "step" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
286{ .Name: "sub_group_broadcast" , .Lead: {1}, .Param: {E_ANY,EX_UINT}},
287{ .Name: "sub_group_commit_read_pipe" , .Lead: {1}, .Param: {E_ANY,EX_RESERVEDID}},
288{ .Name: "sub_group_commit_write_pipe" , .Lead: {1}, .Param: {E_ANY,EX_RESERVEDID}},
289{ .Name: "sub_group_reduce_add" , .Lead: {1}, .Param: {E_ANY}},
290{ .Name: "sub_group_reduce_max" , .Lead: {1}, .Param: {E_ANY}},
291{ .Name: "sub_group_reduce_min" , .Lead: {1}, .Param: {E_ANY}},
292{ .Name: "sub_group_reserve_read_pipe" , .Lead: {1}, .Param: {E_ANY,EX_UINT}},
293{ .Name: "sub_group_reserve_write_pipe" , .Lead: {1}, .Param: {E_ANY,EX_UINT}},
294{ .Name: "sub_group_scan_exclusive_add" , .Lead: {1}, .Param: {E_ANY}},
295{ .Name: "sub_group_scan_exclusive_max" , .Lead: {1}, .Param: {E_ANY}},
296{ .Name: "sub_group_scan_exclusive_min" , .Lead: {1}, .Param: {E_ANY}},
297{ .Name: "sub_group_scan_inclusive_add" , .Lead: {1}, .Param: {E_ANY}},
298{ .Name: "sub_group_scan_inclusive_max" , .Lead: {1}, .Param: {E_ANY}},
299{ .Name: "sub_group_scan_inclusive_min" , .Lead: {1}, .Param: {E_ANY}},
300{ .Name: "sub_sat" , .Lead: {1}, .Param: {E_ANY,E_COPY}},
301{ .Name: "tan" , .Lead: {1}, .Param: {E_ANY}},
302{ .Name: "tanh" , .Lead: {1}, .Param: {E_ANY}},
303{ .Name: "tanpi" , .Lead: {1}, .Param: {E_ANY}},
304{ .Name: "tgamma" , .Lead: {1}, .Param: {E_ANY}},
305{ .Name: "trunc" , .Lead: {1}, .Param: {E_ANY}},
306{ .Name: "upsample" , .Lead: {1}, .Param: {E_ANY,E_MAKEBASE_UNS}},
307{ .Name: "vec_step" , .Lead: {1}, .Param: {E_ANY}},
308{ .Name: "vstore" , .Lead: {3}, .Param: {E_POINTEE,EX_SIZET,E_ANY}},
309{ .Name: "vstore16" , .Lead: {3}, .Param: {E_V16_OF_POINTEE,EX_SIZET,E_ANY}},
310{ .Name: "vstore2" , .Lead: {3}, .Param: {E_V2_OF_POINTEE,EX_SIZET,E_ANY}},
311{ .Name: "vstore3" , .Lead: {3}, .Param: {E_V3_OF_POINTEE,EX_SIZET,E_ANY}},
312{ .Name: "vstore4" , .Lead: {3}, .Param: {E_V4_OF_POINTEE,EX_SIZET,E_ANY}},
313{ .Name: "vstore8" , .Lead: {3}, .Param: {E_V8_OF_POINTEE,EX_SIZET,E_ANY}},
314{ .Name: "work_group_commit_read_pipe" , .Lead: {1}, .Param: {E_ANY,EX_RESERVEDID}},
315{ .Name: "work_group_commit_write_pipe" , .Lead: {1}, .Param: {E_ANY,EX_RESERVEDID}},
316{ .Name: "work_group_reduce_add" , .Lead: {1}, .Param: {E_ANY}},
317{ .Name: "work_group_reduce_max" , .Lead: {1}, .Param: {E_ANY}},
318{ .Name: "work_group_reduce_min" , .Lead: {1}, .Param: {E_ANY}},
319{ .Name: "work_group_reserve_read_pipe" , .Lead: {1}, .Param: {E_ANY,EX_UINT}},
320{ .Name: "work_group_reserve_write_pipe" , .Lead: {1}, .Param: {E_ANY,EX_UINT}},
321{ .Name: "work_group_scan_exclusive_add" , .Lead: {1}, .Param: {E_ANY}},
322{ .Name: "work_group_scan_exclusive_max" , .Lead: {1}, .Param: {E_ANY}},
323{ .Name: "work_group_scan_exclusive_min" , .Lead: {1}, .Param: {E_ANY}},
324{ .Name: "work_group_scan_inclusive_add" , .Lead: {1}, .Param: {E_ANY}},
325{ .Name: "work_group_scan_inclusive_max" , .Lead: {1}, .Param: {E_ANY}},
326{ .Name: "work_group_scan_inclusive_min" , .Lead: {1}, .Param: {E_ANY}},
327{ .Name: "write_imagef" , .Lead: {1}, .Param: {E_ANY,E_IMAGECOORDS,EX_FLOAT4}},
328{ .Name: "write_imagei" , .Lead: {1}, .Param: {E_ANY,E_IMAGECOORDS,EX_INTV4}},
329{ .Name: "write_imageui" , .Lead: {1}, .Param: {E_ANY,E_IMAGECOORDS,EX_UINTV4}},
330{ .Name: "ncos" , .Lead: {1}, .Param: {E_ANY} },
331{ .Name: "nexp2" , .Lead: {1}, .Param: {E_ANY} },
332{ .Name: "nfma" , .Lead: {1}, .Param: {E_ANY, E_COPY, E_COPY} },
333{ .Name: "nlog2" , .Lead: {1}, .Param: {E_ANY} },
334{ .Name: "nrcp" , .Lead: {1}, .Param: {E_ANY} },
335{ .Name: "nrsqrt" , .Lead: {1}, .Param: {E_ANY} },
336{ .Name: "nsin" , .Lead: {1}, .Param: {E_ANY} },
337{ .Name: "nsqrt" , .Lead: {1}, .Param: {E_ANY} },
338{ .Name: "ftz" , .Lead: {1}, .Param: {E_ANY} },
339{ .Name: "fldexp" , .Lead: {1}, .Param: {E_ANY, EX_UINT} },
340{ .Name: "class" , .Lead: {1}, .Param: {E_ANY, EX_UINT} },
341{ .Name: "rcbrt" , .Lead: {1}, .Param: {E_ANY} },
342};
343
344// Library functions with unmangled name.
345const UnmangledFuncInfo UnmangledFuncInfo::Table[] = {
346 {"__read_pipe_2", 4},
347 {"__read_pipe_4", 6},
348 {"__write_pipe_2", 4},
349 {"__write_pipe_4", 6},
350};
351
352const unsigned UnmangledFuncInfo::TableSize =
353 std::size(UnmangledFuncInfo::Table);
354
355static AMDGPULibFunc::Param getRetType(AMDGPULibFunc::EFuncId id,
356 const AMDGPULibFunc::Param (&Leads)[2]) {
357 AMDGPULibFunc::Param Res = Leads[0];
358 // TBD - This switch may require to be extended for other intrinsics
359 switch (id) {
360 case AMDGPULibFunc::EI_SINCOS:
361 Res.PtrKind = AMDGPULibFunc::BYVALUE;
362 break;
363 default:
364 break;
365 }
366 return Res;
367}
368
369class ParamIterator {
370 const AMDGPULibFunc::Param (&Leads)[2];
371 const ManglingRule& Rule;
372 int Index = 0;
373public:
374 ParamIterator(const AMDGPULibFunc::Param (&leads)[2],
375 const ManglingRule& rule)
376 : Leads(leads), Rule(rule) {}
377
378 AMDGPULibFunc::Param getNextParam();
379};
380
381AMDGPULibFunc::Param ParamIterator::getNextParam() {
382 AMDGPULibFunc::Param P;
383 if (Index >= int(sizeof Rule.Param/sizeof Rule.Param[0])) return P;
384
385 const char R = Rule.Param[Index];
386 switch (R) {
387 case E_NONE: break;
388 case EX_UINT:
389 P.ArgType = AMDGPULibFunc::U32; break;
390 case EX_INTV4:
391 P.ArgType = AMDGPULibFunc::I32; P.VectorSize = 4; break;
392 case EX_UINTV4:
393 P.ArgType = AMDGPULibFunc::U32; P.VectorSize = 4; break;
394 case EX_FLOAT4:
395 P.ArgType = AMDGPULibFunc::F32; P.VectorSize = 4; break;
396 case EX_SIZET:
397 P.ArgType = AMDGPULibFunc::U64; break;
398 case EX_EVENT:
399 P.ArgType = AMDGPULibFunc::EVENT; break;
400 case EX_SAMPLER:
401 P.ArgType = AMDGPULibFunc::SAMPLER; break;
402 case EX_RESERVEDID: break; // TBD
403 default:
404 if (Index == (Rule.Lead[1] - 1)) P = Leads[1];
405 else P = Leads[0];
406
407 switch (R) {
408 case E_ANY:
409 case E_COPY: break;
410
411 case E_POINTEE:
412 P.PtrKind = AMDGPULibFunc::BYVALUE; break;
413 case E_V2_OF_POINTEE:
414 P.VectorSize = 2; P.PtrKind = AMDGPULibFunc::BYVALUE; break;
415 case E_V3_OF_POINTEE:
416 P.VectorSize = 3; P.PtrKind = AMDGPULibFunc::BYVALUE; break;
417 case E_V4_OF_POINTEE:
418 P.VectorSize = 4; P.PtrKind = AMDGPULibFunc::BYVALUE; break;
419 case E_V8_OF_POINTEE:
420 P.VectorSize = 8; P.PtrKind = AMDGPULibFunc::BYVALUE; break;
421 case E_V16_OF_POINTEE:
422 P.VectorSize = 16; P.PtrKind = AMDGPULibFunc::BYVALUE; break;
423 case E_CONSTPTR_ANY:
424 P.PtrKind |= AMDGPULibFunc::CONST; break;
425 case E_VLTLPTR_ANY:
426 P.PtrKind |= AMDGPULibFunc::VOLATILE; break;
427 case E_SETBASE_I32:
428 P.ArgType = AMDGPULibFunc::I32; break;
429 case E_SETBASE_U32:
430 P.ArgType = AMDGPULibFunc::U32; break;
431
432 case E_MAKEBASE_UNS:
433 P.ArgType &= ~AMDGPULibFunc::BASE_TYPE_MASK;
434 P.ArgType |= AMDGPULibFunc::UINT;
435 break;
436
437 case E_IMAGECOORDS:
438 switch (P.ArgType) {
439 case AMDGPULibFunc::IMG1DA: P.VectorSize = 2; break;
440 case AMDGPULibFunc::IMG1DB: P.VectorSize = 1; break;
441 case AMDGPULibFunc::IMG2DA: P.VectorSize = 4; break;
442 case AMDGPULibFunc::IMG1D: P.VectorSize = 1; break;
443 case AMDGPULibFunc::IMG2D: P.VectorSize = 2; break;
444 case AMDGPULibFunc::IMG3D: P.VectorSize = 4; break;
445 }
446 P.PtrKind = AMDGPULibFunc::BYVALUE;
447 P.ArgType = AMDGPULibFunc::I32;
448 break;
449
450 case E_CONSTPTR_SWAPGL: {
451 unsigned AS = AMDGPULibFunc::getAddrSpaceFromEPtrKind(Kind: P.PtrKind);
452 switch (AS) {
453 case AMDGPUAS::GLOBAL_ADDRESS: AS = AMDGPUAS::LOCAL_ADDRESS; break;
454 case AMDGPUAS::LOCAL_ADDRESS: AS = AMDGPUAS::GLOBAL_ADDRESS; break;
455 }
456 P.PtrKind = AMDGPULibFunc::getEPtrKindFromAddrSpace(AS);
457 P.PtrKind |= AMDGPULibFunc::CONST;
458 break;
459 }
460
461 default:
462 llvm_unreachable("Unhandled param rule");
463 }
464 }
465 ++Index;
466 return P;
467}
468
469inline static void drop_front(StringRef& str, size_t n = 1) {
470 str = str.drop_front(N: n);
471}
472
473static bool eatTerm(StringRef& mangledName, const char c) {
474 if (mangledName.front() == c) {
475 drop_front(str&: mangledName);
476 return true;
477 }
478 return false;
479}
480
481template <size_t N>
482static bool eatTerm(StringRef& mangledName, const char (&str)[N]) {
483 if (mangledName.starts_with(Prefix: StringRef(str, N - 1))) {
484 drop_front(str&: mangledName, n: N-1);
485 return true;
486 }
487 return false;
488}
489
490static int eatNumber(StringRef& s) {
491 size_t const savedSize = s.size();
492 int n = 0;
493 while (!s.empty() && isDigit(C: s.front())) {
494 n = n*10 + s.front() - '0';
495 drop_front(str&: s);
496 }
497 return s.size() < savedSize ? n : -1;
498}
499
500static StringRef eatLengthPrefixedName(StringRef& mangledName) {
501 int const Len = eatNumber(s&: mangledName);
502 if (Len <= 0 || static_cast<size_t>(Len) > mangledName.size())
503 return StringRef();
504 StringRef Res = mangledName.substr(Start: 0, N: Len);
505 drop_front(str&: mangledName, n: Len);
506 return Res;
507}
508
509} // end anonymous namespace
510
511AMDGPUMangledLibFunc::AMDGPUMangledLibFunc() {
512 FuncId = EI_NONE;
513 FKind = NOPFX;
514 Leads[0].reset();
515 Leads[1].reset();
516 Name.clear();
517}
518
519AMDGPUUnmangledLibFunc::AMDGPUUnmangledLibFunc() {
520 FuncId = EI_NONE;
521 FuncTy = nullptr;
522}
523
524AMDGPUMangledLibFunc::AMDGPUMangledLibFunc(
525 EFuncId id, const AMDGPUMangledLibFunc &copyFrom) {
526 FuncId = id;
527 FKind = copyFrom.FKind;
528 Leads[0] = copyFrom.Leads[0];
529 Leads[1] = copyFrom.Leads[1];
530}
531
532AMDGPUMangledLibFunc::AMDGPUMangledLibFunc(EFuncId id, FunctionType *FT,
533 bool SignedInts) {
534 FuncId = id;
535 unsigned NumArgs = FT->getNumParams();
536 if (NumArgs >= 1)
537 Leads[0] = Param::getFromTy(Ty: FT->getParamType(i: 0), Signed: SignedInts);
538 if (NumArgs >= 2)
539 Leads[1] = Param::getFromTy(Ty: FT->getParamType(i: 1), Signed: SignedInts);
540}
541
542///////////////////////////////////////////////////////////////////////////////
543// Demangling
544
545static unsigned char parseVecSize(StringRef &mangledName) {
546 int const Len = eatNumber(s&: mangledName);
547 switch (Len) {
548 case 2: case 3: case 4: case 8: case 16:
549 return static_cast<unsigned char>(Len);
550 default:
551 return 1;
552 }
553}
554
555static AMDGPULibFunc::ENamePrefix parseNamePrefix(StringRef& mangledName) {
556 std::pair<StringRef, StringRef> const P = mangledName.split(Separator: '_');
557 AMDGPULibFunc::ENamePrefix Pfx =
558 StringSwitch<AMDGPULibFunc::ENamePrefix>(P.first)
559 .Case(S: "native", Value: AMDGPULibFunc::NATIVE)
560 .Case(S: "half" , Value: AMDGPULibFunc::HALF)
561 .Default(Value: AMDGPULibFunc::NOPFX);
562
563 if (Pfx != AMDGPULibFunc::NOPFX)
564 mangledName = P.second;
565
566 return Pfx;
567}
568
569StringMap<int> ManglingRule::buildManglingRulesMap() {
570 StringMap<int> Map(std::size(manglingRules));
571 int Id = 0;
572 for (auto Rule : manglingRules)
573 Map.insert(KV: {Rule.Name, Id++});
574 return Map;
575}
576
577bool AMDGPUMangledLibFunc::parseUnmangledName(StringRef FullName) {
578 static const StringMap<int> manglingRulesMap =
579 ManglingRule::buildManglingRulesMap();
580 FuncId = static_cast<EFuncId>(manglingRulesMap.lookup(Key: FullName));
581 return FuncId != EI_NONE;
582}
583
584///////////////////////////////////////////////////////////////////////////////
585// Itanium Demangling
586
587namespace {
588struct ItaniumParamParser {
589 AMDGPULibFunc::Param Prev;
590 bool parseItaniumParam(StringRef& param, AMDGPULibFunc::Param &res);
591};
592} // namespace
593
594bool ItaniumParamParser::parseItaniumParam(StringRef& param,
595 AMDGPULibFunc::Param &res) {
596 res.reset();
597 if (param.empty()) return false;
598
599 // parse pointer prefix
600 if (eatTerm(mangledName&: param, c: 'P')) {
601 if (eatTerm(mangledName&: param, c: 'K')) res.PtrKind |= AMDGPULibFunc::CONST;
602 if (eatTerm(mangledName&: param, c: 'V')) res.PtrKind |= AMDGPULibFunc::VOLATILE;
603 unsigned AS;
604 if (!eatTerm(mangledName&: param, str: "U3AS")) {
605 AS = 0;
606 } else {
607 AS = param.front() - '0';
608 drop_front(str&: param, n: 1);
609 }
610 res.PtrKind |= AMDGPULibFuncBase::getEPtrKindFromAddrSpace(AS);
611 } else {
612 res.PtrKind = AMDGPULibFunc::BYVALUE;
613 }
614
615 // parse vector size
616 if (eatTerm(mangledName&: param,str: "Dv")) {
617 res.VectorSize = parseVecSize(mangledName&: param);
618 if (res.VectorSize==1 || !eatTerm(mangledName&: param, c: '_')) return false;
619 }
620
621 // parse type
622 char const TC = param.front();
623 if (isDigit(C: TC)) {
624 res.ArgType =
625 StringSwitch<AMDGPULibFunc::EType>(eatLengthPrefixedName(mangledName&: param))
626 .StartsWith(S: "ocl_image1d_array", Value: AMDGPULibFunc::IMG1DA)
627 .StartsWith(S: "ocl_image1d_buffer", Value: AMDGPULibFunc::IMG1DB)
628 .StartsWith(S: "ocl_image2d_array", Value: AMDGPULibFunc::IMG2DA)
629 .StartsWith(S: "ocl_image1d", Value: AMDGPULibFunc::IMG1D)
630 .StartsWith(S: "ocl_image2d", Value: AMDGPULibFunc::IMG2D)
631 .StartsWith(S: "ocl_image3d", Value: AMDGPULibFunc::IMG3D)
632 .Case(S: "ocl_event", Value: AMDGPULibFunc::DUMMY)
633 .Case(S: "ocl_sampler", Value: AMDGPULibFunc::DUMMY)
634 .Default(Value: AMDGPULibFunc::DUMMY);
635 } else {
636 drop_front(str&: param);
637 switch (TC) {
638 case 'h': res.ArgType = AMDGPULibFunc::U8; break;
639 case 't': res.ArgType = AMDGPULibFunc::U16; break;
640 case 'j': res.ArgType = AMDGPULibFunc::U32; break;
641 case 'm': res.ArgType = AMDGPULibFunc::U64; break;
642 case 'c': res.ArgType = AMDGPULibFunc::I8; break;
643 case 's': res.ArgType = AMDGPULibFunc::I16; break;
644 case 'i': res.ArgType = AMDGPULibFunc::I32; break;
645 case 'l': res.ArgType = AMDGPULibFunc::I64; break;
646 case 'f': res.ArgType = AMDGPULibFunc::F32; break;
647 case 'd': res.ArgType = AMDGPULibFunc::F64; break;
648 case 'D': if (!eatTerm(mangledName&: param, c: 'h')) return false;
649 res.ArgType = AMDGPULibFunc::F16; break;
650 case 'S':
651 if (!eatTerm(mangledName&: param, c: '_')) {
652 eatNumber(s&: param);
653 if (!eatTerm(mangledName&: param, c: '_')) return false;
654 }
655 res.VectorSize = Prev.VectorSize;
656 res.ArgType = Prev.ArgType;
657 break;
658 default:;
659 }
660 }
661 if (res.ArgType == 0) return false;
662 Prev.VectorSize = res.VectorSize;
663 Prev.ArgType = res.ArgType;
664 return true;
665}
666
667bool AMDGPUMangledLibFunc::parseFuncName(StringRef &mangledName) {
668 StringRef Name = eatLengthPrefixedName(mangledName);
669 FKind = parseNamePrefix(mangledName&: Name);
670 if (!parseUnmangledName(FullName: Name))
671 return false;
672
673 const ManglingRule& Rule = manglingRules[FuncId];
674 ItaniumParamParser Parser;
675 for (int I=0; I < Rule.maxLeadIndex(); ++I) {
676 Param P;
677 if (!Parser.parseItaniumParam(param&: mangledName, res&: P))
678 return false;
679
680 if ((I + 1) == Rule.Lead[0]) Leads[0] = P;
681 if ((I + 1) == Rule.Lead[1]) Leads[1] = P;
682 }
683 return true;
684}
685
686bool AMDGPUUnmangledLibFunc::parseFuncName(StringRef &Name) {
687 if (!UnmangledFuncInfo::lookup(Name, Id&: FuncId))
688 return false;
689 setName(Name);
690 return true;
691}
692
693bool AMDGPULibFunc::parse(StringRef FuncName, AMDGPULibFunc &F) {
694 if (FuncName.empty()) {
695 F.Impl = std::unique_ptr<AMDGPULibFuncImpl>();
696 return false;
697 }
698
699 if (eatTerm(mangledName&: FuncName, str: "_Z"))
700 F.Impl = std::make_unique<AMDGPUMangledLibFunc>();
701 else
702 F.Impl = std::make_unique<AMDGPUUnmangledLibFunc>();
703 if (F.Impl->parseFuncName(mangledName&: FuncName))
704 return true;
705
706 F.Impl = std::unique_ptr<AMDGPULibFuncImpl>();
707 return false;
708}
709
710///////////////////////////////////////////////////////////////////////////////
711// Mangling
712
713template <typename Stream>
714void AMDGPUMangledLibFunc::writeName(Stream &OS) const {
715 const char *Pfx = "";
716 switch (FKind) {
717 case NATIVE: Pfx = "native_"; break;
718 case HALF: Pfx = "half_"; break;
719 default: break;
720 }
721 if (!Name.empty()) {
722 OS << Pfx << Name;
723 } else if (FuncId != EI_NONE) {
724 OS << Pfx;
725 const StringRef& S = manglingRules[FuncId].Name;
726 OS.write(S.data(), S.size());
727 }
728}
729
730std::string AMDGPUMangledLibFunc::mangle() const { return mangleNameItanium(); }
731
732///////////////////////////////////////////////////////////////////////////////
733// Itanium Mangling
734
735static const char *getItaniumTypeName(AMDGPULibFunc::EType T) {
736 switch (T) {
737 case AMDGPULibFunc::U8: return "h";
738 case AMDGPULibFunc::U16: return "t";
739 case AMDGPULibFunc::U32: return "j";
740 case AMDGPULibFunc::U64: return "m";
741 case AMDGPULibFunc::I8: return "c";
742 case AMDGPULibFunc::I16: return "s";
743 case AMDGPULibFunc::I32: return "i";
744 case AMDGPULibFunc::I64: return "l";
745 case AMDGPULibFunc::F16: return "Dh";
746 case AMDGPULibFunc::F32: return "f";
747 case AMDGPULibFunc::F64: return "d";
748 case AMDGPULibFunc::IMG1DA: return "16ocl_image1darray";
749 case AMDGPULibFunc::IMG1DB: return "17ocl_image1dbuffer";
750 case AMDGPULibFunc::IMG2DA: return "16ocl_image2darray";
751 case AMDGPULibFunc::IMG1D: return "11ocl_image1d";
752 case AMDGPULibFunc::IMG2D: return "11ocl_image2d";
753 case AMDGPULibFunc::IMG3D: return "11ocl_image3d";
754 case AMDGPULibFunc::SAMPLER: return "11ocl_sampler";
755 case AMDGPULibFunc::EVENT: return "9ocl_event";
756 default:
757 llvm_unreachable("Unhandled param type");
758 }
759 return nullptr;
760}
761
762namespace {
763// Itanium mangling ABI says:
764// "5.1.8. Compression
765// ... Each non-terminal in the grammar for which <substitution> appears on the
766// right-hand side is both a source of future substitutions and a candidate
767// for being substituted. There are two exceptions that appear to be
768// substitution candidates from the grammar, but are explicitly excluded:
769// 1. <builtin-type> other than vendor extended types ..."
770
771// For the purpose of functions the following productions make sense for the
772// substitution:
773// <type> ::= <builtin-type>
774// ::= <class-enum-type>
775// ::= <array-type>
776// ::=<CV-qualifiers> <type>
777// ::= P <type> # pointer-to
778// ::= <substitution>
779//
780// Note that while types like images, samplers and events are by the ABI encoded
781// using <class-enum-type> production rule they're not used for substitution
782// because clang consider them as builtin types.
783//
784// DvNN_ type is GCC extension for vectors and is a subject for the
785// substitution.
786
787class ItaniumMangler {
788 SmallVector<AMDGPULibFunc::Param, 10> Str; // list of accumulated substitutions
789 bool UseAddrSpace;
790
791 int findSubst(const AMDGPULibFunc::Param& P) const {
792 for(unsigned I = 0; I < Str.size(); ++I) {
793 const AMDGPULibFunc::Param& T = Str[I];
794 if (P.PtrKind == T.PtrKind &&
795 P.VectorSize == T.VectorSize &&
796 P.ArgType == T.ArgType) {
797 return I;
798 }
799 }
800 return -1;
801 }
802
803 template <typename Stream>
804 bool trySubst(Stream& os, const AMDGPULibFunc::Param& p) {
805 int const subst = findSubst(P: p);
806 if (subst < 0) return false;
807 // Substitutions are mangled as S(XX)?_ where XX is a hexadecimal number
808 // 0 1 2
809 // S_ S0_ S1_
810 if (subst == 0) os << "S_";
811 else os << 'S' << (subst-1) << '_';
812 return true;
813 }
814
815public:
816 ItaniumMangler(bool useAddrSpace)
817 : UseAddrSpace(useAddrSpace) {}
818
819 template <typename Stream>
820 void operator()(Stream& os, AMDGPULibFunc::Param p) {
821
822 // Itanium mangling ABI 5.1.8. Compression:
823 // Logically, the substitutable components of a mangled name are considered
824 // left-to-right, components before the composite structure of which they
825 // are a part. If a component has been encountered before, it is substituted
826 // as described below. This decision is independent of whether its components
827 // have been substituted, so an implementation may optimize by considering
828 // large structures for substitution before their components. If a component
829 // has not been encountered before, its mangling is identified, and it is
830 // added to a dictionary of substitution candidates. No entity is added to
831 // the dictionary twice.
832 AMDGPULibFunc::Param Ptr;
833
834 if (p.PtrKind) {
835 if (trySubst(os, p)) return;
836 os << 'P';
837 if (p.PtrKind & AMDGPULibFunc::CONST) os << 'K';
838 if (p.PtrKind & AMDGPULibFunc::VOLATILE) os << 'V';
839 unsigned AS = UseAddrSpace
840 ? AMDGPULibFuncBase::getAddrSpaceFromEPtrKind(Kind: p.PtrKind)
841 : 0;
842 if (EnableOCLManglingMismatchWA || AS != 0)
843 os << "U3AS" << AS;
844 Ptr = p;
845 p.PtrKind = 0;
846 }
847
848 if (p.VectorSize > 1) {
849 if (trySubst(os, p)) goto exit;
850 Str.push_back(Elt: p);
851 os << "Dv" << static_cast<unsigned>(p.VectorSize) << '_';
852 }
853
854 os << getItaniumTypeName(T: (AMDGPULibFunc::EType)p.ArgType);
855
856 exit:
857 if (Ptr.ArgType) Str.push_back(Elt: Ptr);
858 }
859};
860} // namespace
861
862std::string AMDGPUMangledLibFunc::mangleNameItanium() const {
863 SmallString<128> Buf;
864 raw_svector_ostream S(Buf);
865 SmallString<128> NameBuf;
866 raw_svector_ostream Name(NameBuf);
867 writeName(OS&: Name);
868 const StringRef& NameStr = Name.str();
869 S << "_Z" << static_cast<int>(NameStr.size()) << NameStr;
870
871 ItaniumMangler Mangler(true);
872 ParamIterator I(Leads, manglingRules[FuncId]);
873 Param P;
874 while ((P = I.getNextParam()).ArgType != 0)
875 Mangler(S, P);
876 return std::string(S.str());
877}
878
879///////////////////////////////////////////////////////////////////////////////
880// Misc
881
882AMDGPULibFuncBase::Param AMDGPULibFuncBase::Param::getFromTy(Type *Ty,
883 bool Signed) {
884 Param P;
885 if (FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
886 P.VectorSize = static_cast<unsigned char>(VT->getNumElements());
887 Ty = VT->getElementType();
888 }
889
890 switch (Ty->getTypeID()) {
891 case Type::FloatTyID:
892 P.ArgType = AMDGPULibFunc::F32;
893 break;
894 case Type::DoubleTyID:
895 P.ArgType = AMDGPULibFunc::F64;
896 break;
897 case Type::HalfTyID:
898 P.ArgType = AMDGPULibFunc::F16;
899 break;
900 case Type::IntegerTyID:
901 switch (cast<IntegerType>(Val: Ty)->getBitWidth()) {
902 case 8:
903 P.ArgType = Signed ? AMDGPULibFunc::I8 : AMDGPULibFunc::U8;
904 break;
905 case 16:
906 P.ArgType = Signed ? AMDGPULibFunc::I16 : AMDGPULibFunc::U16;
907 break;
908 case 32:
909 P.ArgType = Signed ? AMDGPULibFunc::I32 : AMDGPULibFunc::U32;
910 break;
911 case 64:
912 P.ArgType = Signed ? AMDGPULibFunc::I64 : AMDGPULibFunc::U64;
913 break;
914 default:
915 llvm_unreachable("unhandled libcall argument type");
916 }
917
918 break;
919 default:
920 llvm_unreachable("unhandled libcall argument type");
921 }
922
923 return P;
924}
925
926static Type *getIntrinsicParamType(LLVMContext &C,
927 const AMDGPULibFunc::Param &P,
928 bool UseAddrSpace) {
929 Type *T = nullptr;
930 switch (P.ArgType) {
931 default:
932 return nullptr;
933 case AMDGPULibFunc::U8:
934 case AMDGPULibFunc::I8:
935 T = Type::getInt8Ty(C);
936 break;
937 case AMDGPULibFunc::U16:
938 case AMDGPULibFunc::I16:
939 T = Type::getInt16Ty(C);
940 break;
941 case AMDGPULibFunc::U32:
942 case AMDGPULibFunc::I32:
943 T = Type::getInt32Ty(C);
944 break;
945 case AMDGPULibFunc::U64:
946 case AMDGPULibFunc::I64:
947 T = Type::getInt64Ty(C);
948 break;
949 case AMDGPULibFunc::F16:
950 T = Type::getHalfTy(C);
951 break;
952 case AMDGPULibFunc::F32:
953 T = Type::getFloatTy(C);
954 break;
955 case AMDGPULibFunc::F64:
956 T = Type::getDoubleTy(C);
957 break;
958
959 case AMDGPULibFunc::IMG1DA:
960 case AMDGPULibFunc::IMG1DB:
961 case AMDGPULibFunc::IMG2DA:
962 case AMDGPULibFunc::IMG1D:
963 case AMDGPULibFunc::IMG2D:
964 case AMDGPULibFunc::IMG3D:
965 case AMDGPULibFunc::SAMPLER:
966 case AMDGPULibFunc::EVENT:
967 T = PointerType::getUnqual(C);
968 break;
969 case AMDGPULibFunc::B8:
970 case AMDGPULibFunc::B16:
971 case AMDGPULibFunc::B32:
972 case AMDGPULibFunc::B64:
973 case AMDGPULibFunc::SIZE_MASK:
974 case AMDGPULibFunc::FLOAT:
975 case AMDGPULibFunc::INT:
976 case AMDGPULibFunc::UINT:
977 case AMDGPULibFunc::DUMMY:
978 return nullptr;
979 }
980 if (P.VectorSize > 1)
981 T = FixedVectorType::get(ElementType: T, NumElts: P.VectorSize);
982 if (P.PtrKind != AMDGPULibFunc::BYVALUE)
983 T = PointerType::get(
984 C, AddressSpace: UseAddrSpace ? ((P.PtrKind & AMDGPULibFunc::ADDR_SPACE) - 1) : 0);
985 return T;
986}
987
988FunctionType *AMDGPUMangledLibFunc::getFunctionType(const Module &M) const {
989 LLVMContext& C = M.getContext();
990 std::vector<Type*> Args;
991 ParamIterator I(Leads, manglingRules[FuncId]);
992 Param P;
993 while ((P = I.getNextParam()).ArgType != 0) {
994 Type *ParamTy = getIntrinsicParamType(C, P, UseAddrSpace: true);
995 if (!ParamTy)
996 return nullptr;
997
998 Args.push_back(x: ParamTy);
999 }
1000
1001 Type *RetTy = getIntrinsicParamType(C, P: getRetType(id: FuncId, Leads), UseAddrSpace: true);
1002 if (!RetTy)
1003 return nullptr;
1004
1005 return FunctionType::get(Result: RetTy, Params: Args, isVarArg: false);
1006}
1007
1008unsigned AMDGPUMangledLibFunc::getNumArgs() const {
1009 return manglingRules[FuncId].getNumArgs();
1010}
1011
1012unsigned AMDGPUUnmangledLibFunc::getNumArgs() const {
1013 return UnmangledFuncInfo::getNumArgs(Id: FuncId);
1014}
1015
1016std::string AMDGPUMangledLibFunc::getName() const {
1017 SmallString<128> Buf;
1018 raw_svector_ostream OS(Buf);
1019 writeName(OS);
1020 return std::string(OS.str());
1021}
1022
1023bool AMDGPULibFunc::isCompatibleSignature(const Module &M,
1024 const FunctionType *CallTy) const {
1025 const FunctionType *FuncTy = getFunctionType(M);
1026
1027 if (!FuncTy) {
1028 // Give up on mangled functions with unexpected types.
1029 if (AMDGPULibFuncBase::isMangled(Id: getId()))
1030 return false;
1031
1032 // FIXME: UnmangledFuncInfo does not have any type information other than
1033 // the number of arguments.
1034 return getNumArgs() == CallTy->getNumParams();
1035 }
1036
1037 // Normally the types should exactly match.
1038 if (FuncTy == CallTy)
1039 return true;
1040
1041 const unsigned NumParams = FuncTy->getNumParams();
1042 if (NumParams != CallTy->getNumParams())
1043 return false;
1044
1045 for (unsigned I = 0; I != NumParams; ++I) {
1046 Type *FuncArgTy = FuncTy->getParamType(i: I);
1047 Type *CallArgTy = CallTy->getParamType(i: I);
1048 if (FuncArgTy == CallArgTy)
1049 continue;
1050
1051 // Some cases permit implicit splatting a scalar value to a vector argument.
1052 auto *FuncVecTy = dyn_cast<VectorType>(Val: FuncArgTy);
1053 if (FuncVecTy && FuncVecTy->getElementType() == CallArgTy &&
1054 allowsImplicitVectorSplat(ArgIdx: I))
1055 continue;
1056
1057 return false;
1058 }
1059
1060 return true;
1061}
1062
1063Function *AMDGPULibFunc::getFunction(Module *M, const AMDGPULibFunc &fInfo) {
1064 std::string FuncName = fInfo.mangle();
1065 Function *F = dyn_cast_or_null<Function>(
1066 Val: M->getValueSymbolTable().lookup(Name: FuncName));
1067 if (!F || F->isDeclaration())
1068 return nullptr;
1069
1070 if (F->hasFnAttribute(Kind: Attribute::NoBuiltin))
1071 return nullptr;
1072
1073 if (!fInfo.isCompatibleSignature(M: *M, CallTy: F->getFunctionType()))
1074 return nullptr;
1075
1076 switch (fInfo.getId()) {
1077 case AMDGPULibFunc::EI_POW_FAST:
1078 case AMDGPULibFunc::EI_POWR_FAST:
1079 case AMDGPULibFunc::EI_POWN_FAST:
1080 case AMDGPULibFunc::EI_ROOTN_FAST:
1081 // TODO: Remove this. This is not a real module flag used anywhere. This is
1082 // a bringup hack so this transform is testable prior to the library
1083 // functions existing.
1084 if (!M->getModuleFlag(Key: "amdgpu-libcall-have-fast-pow"))
1085 return nullptr;
1086 break;
1087 default:
1088 break;
1089 }
1090
1091 return F;
1092}
1093
1094FunctionCallee AMDGPULibFunc::getOrInsertFunction(Module *M,
1095 const AMDGPULibFunc &fInfo) {
1096 std::string const FuncName = fInfo.mangle();
1097 Function *F = dyn_cast_or_null<Function>(
1098 Val: M->getValueSymbolTable().lookup(Name: FuncName));
1099
1100 if (F) {
1101 if (F->hasFnAttribute(Kind: Attribute::NoBuiltin))
1102 return nullptr;
1103 if (!F->isDeclaration() &&
1104 fInfo.isCompatibleSignature(M: *M, CallTy: F->getFunctionType()))
1105 return F;
1106 }
1107
1108 FunctionType *FuncTy = fInfo.getFunctionType(M: *M);
1109 assert(FuncTy);
1110
1111 bool hasPtr = false;
1112 for (FunctionType::param_iterator
1113 PI = FuncTy->param_begin(),
1114 PE = FuncTy->param_end();
1115 PI != PE; ++PI) {
1116 const Type* argTy = static_cast<const Type*>(*PI);
1117 if (argTy->isPointerTy()) {
1118 hasPtr = true;
1119 break;
1120 }
1121 }
1122
1123 FunctionCallee C;
1124 if (hasPtr) {
1125 // Do not set extra attributes for functions with pointer arguments.
1126 C = M->getOrInsertFunction(Name: FuncName, T: FuncTy);
1127 } else {
1128 AttributeList Attr;
1129 LLVMContext &Ctx = M->getContext();
1130 Attr = Attr.addFnAttribute(
1131 C&: Ctx, Attr: Attribute::getWithMemoryEffects(Context&: Ctx, ME: MemoryEffects::readOnly()));
1132 Attr = Attr.addFnAttribute(C&: Ctx, Kind: Attribute::NoUnwind);
1133 C = M->getOrInsertFunction(Name: FuncName, T: FuncTy, AttributeList: Attr);
1134 }
1135
1136 return C;
1137}
1138
1139StringMap<unsigned> UnmangledFuncInfo::buildNameMap() {
1140 StringMap<unsigned> Map;
1141 for (unsigned I = 0; I != TableSize; ++I)
1142 Map[Table[I].Name] = I;
1143 return Map;
1144}
1145
1146bool UnmangledFuncInfo::lookup(StringRef Name, ID &Id) {
1147 static const StringMap<unsigned> Map = buildNameMap();
1148 auto Loc = Map.find(Key: Name);
1149 if (Loc != Map.end()) {
1150 Id = toFuncId(Index: Loc->second);
1151 return true;
1152 }
1153 Id = AMDGPULibFunc::EI_NONE;
1154 return false;
1155}
1156
1157AMDGPULibFunc::AMDGPULibFunc(const AMDGPULibFunc &F) {
1158 if (auto *MF = dyn_cast<AMDGPUMangledLibFunc>(Val: F.Impl.get()))
1159 Impl = std::make_unique<AMDGPUMangledLibFunc>(args&: *MF);
1160 else if (auto *UMF = dyn_cast<AMDGPUUnmangledLibFunc>(Val: F.Impl.get()))
1161 Impl = std::make_unique<AMDGPUUnmangledLibFunc>(args&: *UMF);
1162 else
1163 Impl = std::unique_ptr<AMDGPULibFuncImpl>();
1164}
1165
1166AMDGPULibFunc &AMDGPULibFunc::operator=(const AMDGPULibFunc &F) {
1167 if (this == &F)
1168 return *this;
1169
1170 this->~AMDGPULibFunc();
1171 new (this) AMDGPULibFunc(F);
1172 return *this;
1173}
1174
1175AMDGPULibFunc::AMDGPULibFunc(EFuncId Id, const AMDGPULibFunc &CopyFrom) {
1176 assert(AMDGPULibFuncBase::isMangled(Id) && CopyFrom.isMangled() &&
1177 "not supported");
1178 Impl = std::make_unique<AMDGPUMangledLibFunc>(
1179 args&: Id, args&: *cast<AMDGPUMangledLibFunc>(Val: CopyFrom.Impl.get()));
1180}
1181
1182AMDGPULibFunc::AMDGPULibFunc(EFuncId Id, FunctionType *FT, bool SignedInts) {
1183 Impl = std::make_unique<AMDGPUMangledLibFunc>(args&: Id, args&: FT, args&: SignedInts);
1184}
1185
1186AMDGPULibFunc::AMDGPULibFunc(StringRef Name, FunctionType *FT) {
1187 Impl = std::make_unique<AMDGPUUnmangledLibFunc>(args&: Name, args&: FT);
1188}
1189
1190void AMDGPULibFunc::initMangled() {
1191 Impl = std::make_unique<AMDGPUMangledLibFunc>();
1192}
1193
1194AMDGPULibFunc::Param *AMDGPULibFunc::getLeads() {
1195 if (!Impl)
1196 initMangled();
1197 return cast<AMDGPUMangledLibFunc>(Val: Impl.get())->Leads;
1198}
1199
1200const AMDGPULibFunc::Param *AMDGPULibFunc::getLeads() const {
1201 return cast<const AMDGPUMangledLibFunc>(Val: Impl.get())->Leads;
1202}
1203