1//===-------- NVPTX.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "clang/Basic/TargetBuiltins.h"
15#include "llvm/IR/IntrinsicsNVPTX.h"
16#include "llvm/TargetParser/AtomicScope.h"
17
18using namespace clang;
19using namespace CodeGen;
20using namespace llvm;
21
22namespace {
23// Helper classes for mapping MMA builtins to particular LLVM intrinsic variant.
24struct NVPTXMmaLdstInfo {
25 unsigned NumResults; // Number of elements to load/store
26 // Intrinsic IDs for row/col variants. 0 if particular layout is unsupported.
27 unsigned IID_col;
28 unsigned IID_row;
29};
30
31#define MMA_INTR(geom_op_type, layout) \
32 Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
33#define MMA_LDST(n, geom_op_type) \
34 { n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row) }
35
36static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(unsigned BuiltinID) {
37 switch (BuiltinID) {
38 // FP MMA loads
39 case NVPTX::BI__hmma_m16n16k16_ld_a:
40 return MMA_LDST(8, m16n16k16_load_a_f16);
41 case NVPTX::BI__hmma_m16n16k16_ld_b:
42 return MMA_LDST(8, m16n16k16_load_b_f16);
43 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
44 return MMA_LDST(4, m16n16k16_load_c_f16);
45 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
46 return MMA_LDST(8, m16n16k16_load_c_f32);
47 case NVPTX::BI__hmma_m32n8k16_ld_a:
48 return MMA_LDST(8, m32n8k16_load_a_f16);
49 case NVPTX::BI__hmma_m32n8k16_ld_b:
50 return MMA_LDST(8, m32n8k16_load_b_f16);
51 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
52 return MMA_LDST(4, m32n8k16_load_c_f16);
53 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
54 return MMA_LDST(8, m32n8k16_load_c_f32);
55 case NVPTX::BI__hmma_m8n32k16_ld_a:
56 return MMA_LDST(8, m8n32k16_load_a_f16);
57 case NVPTX::BI__hmma_m8n32k16_ld_b:
58 return MMA_LDST(8, m8n32k16_load_b_f16);
59 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
60 return MMA_LDST(4, m8n32k16_load_c_f16);
61 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
62 return MMA_LDST(8, m8n32k16_load_c_f32);
63
64 // Integer MMA loads
65 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
66 return MMA_LDST(2, m16n16k16_load_a_s8);
67 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
68 return MMA_LDST(2, m16n16k16_load_a_u8);
69 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
70 return MMA_LDST(2, m16n16k16_load_b_s8);
71 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
72 return MMA_LDST(2, m16n16k16_load_b_u8);
73 case NVPTX::BI__imma_m16n16k16_ld_c:
74 return MMA_LDST(8, m16n16k16_load_c_s32);
75 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
76 return MMA_LDST(4, m32n8k16_load_a_s8);
77 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
78 return MMA_LDST(4, m32n8k16_load_a_u8);
79 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
80 return MMA_LDST(1, m32n8k16_load_b_s8);
81 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
82 return MMA_LDST(1, m32n8k16_load_b_u8);
83 case NVPTX::BI__imma_m32n8k16_ld_c:
84 return MMA_LDST(8, m32n8k16_load_c_s32);
85 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
86 return MMA_LDST(1, m8n32k16_load_a_s8);
87 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
88 return MMA_LDST(1, m8n32k16_load_a_u8);
89 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
90 return MMA_LDST(4, m8n32k16_load_b_s8);
91 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
92 return MMA_LDST(4, m8n32k16_load_b_u8);
93 case NVPTX::BI__imma_m8n32k16_ld_c:
94 return MMA_LDST(8, m8n32k16_load_c_s32);
95
96 // Sub-integer MMA loads.
97 // Only row/col layout is supported by A/B fragments.
98 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
99 return {.NumResults: 1, .IID_col: 0, MMA_INTR(m8n8k32_load_a_s4, row)};
100 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
101 return {.NumResults: 1, .IID_col: 0, MMA_INTR(m8n8k32_load_a_u4, row)};
102 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
103 return {.NumResults: 1, MMA_INTR(m8n8k32_load_b_s4, col), .IID_row: 0};
104 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
105 return {.NumResults: 1, MMA_INTR(m8n8k32_load_b_u4, col), .IID_row: 0};
106 case NVPTX::BI__imma_m8n8k32_ld_c:
107 return MMA_LDST(2, m8n8k32_load_c_s32);
108 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
109 return {.NumResults: 1, .IID_col: 0, MMA_INTR(m8n8k128_load_a_b1, row)};
110 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
111 return {.NumResults: 1, MMA_INTR(m8n8k128_load_b_b1, col), .IID_row: 0};
112 case NVPTX::BI__bmma_m8n8k128_ld_c:
113 return MMA_LDST(2, m8n8k128_load_c_s32);
114
115 // Double MMA loads
116 case NVPTX::BI__dmma_m8n8k4_ld_a:
117 return MMA_LDST(1, m8n8k4_load_a_f64);
118 case NVPTX::BI__dmma_m8n8k4_ld_b:
119 return MMA_LDST(1, m8n8k4_load_b_f64);
120 case NVPTX::BI__dmma_m8n8k4_ld_c:
121 return MMA_LDST(2, m8n8k4_load_c_f64);
122
123 // Alternate float MMA loads
124 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
125 return MMA_LDST(4, m16n16k16_load_a_bf16);
126 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
127 return MMA_LDST(4, m16n16k16_load_b_bf16);
128 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
129 return MMA_LDST(2, m8n32k16_load_a_bf16);
130 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
131 return MMA_LDST(8, m8n32k16_load_b_bf16);
132 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
133 return MMA_LDST(8, m32n8k16_load_a_bf16);
134 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
135 return MMA_LDST(2, m32n8k16_load_b_bf16);
136 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
137 return MMA_LDST(4, m16n16k8_load_a_tf32);
138 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
139 return MMA_LDST(4, m16n16k8_load_b_tf32);
140 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
141 return MMA_LDST(8, m16n16k8_load_c_f32);
142
143 // NOTE: We need to follow inconsitent naming scheme used by NVCC. Unlike
144 // PTX and LLVM IR where stores always use fragment D, NVCC builtins always
145 // use fragment C for both loads and stores.
146 // FP MMA stores.
147 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
148 return MMA_LDST(4, m16n16k16_store_d_f16);
149 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
150 return MMA_LDST(8, m16n16k16_store_d_f32);
151 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
152 return MMA_LDST(4, m32n8k16_store_d_f16);
153 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
154 return MMA_LDST(8, m32n8k16_store_d_f32);
155 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
156 return MMA_LDST(4, m8n32k16_store_d_f16);
157 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
158 return MMA_LDST(8, m8n32k16_store_d_f32);
159
160 // Integer and sub-integer MMA stores.
161 // Another naming quirk. Unlike other MMA builtins that use PTX types in the
162 // name, integer loads/stores use LLVM's i32.
163 case NVPTX::BI__imma_m16n16k16_st_c_i32:
164 return MMA_LDST(8, m16n16k16_store_d_s32);
165 case NVPTX::BI__imma_m32n8k16_st_c_i32:
166 return MMA_LDST(8, m32n8k16_store_d_s32);
167 case NVPTX::BI__imma_m8n32k16_st_c_i32:
168 return MMA_LDST(8, m8n32k16_store_d_s32);
169 case NVPTX::BI__imma_m8n8k32_st_c_i32:
170 return MMA_LDST(2, m8n8k32_store_d_s32);
171 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
172 return MMA_LDST(2, m8n8k128_store_d_s32);
173
174 // Double MMA store
175 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
176 return MMA_LDST(2, m8n8k4_store_d_f64);
177
178 // Alternate float MMA store
179 case NVPTX::BI__mma_m16n16k8_st_c_f32:
180 return MMA_LDST(8, m16n16k8_store_d_f32);
181
182 default:
183 llvm_unreachable("Unknown MMA builtin");
184 }
185}
186#undef MMA_LDST
187#undef MMA_INTR
188
189
190struct NVPTXMmaInfo {
191 unsigned NumEltsA;
192 unsigned NumEltsB;
193 unsigned NumEltsC;
194 unsigned NumEltsD;
195
196 // Variants are ordered by layout-A/layout-B/satf, where 'row' has priority
197 // over 'col' for layout. The index of non-satf variants is expected to match
198 // the undocumented layout constants used by CUDA's mma.hpp.
199 std::array<unsigned, 8> Variants;
200
201 unsigned getMMAIntrinsic(int Layout, bool Satf) {
202 unsigned Index = Layout + 4 * Satf;
203 if (Index >= Variants.size())
204 return 0;
205 return Variants[Index];
206 }
207};
208
209 // Returns an intrinsic that matches Layout and Satf for valid combinations of
210 // Layout and Satf, 0 otherwise.
211static NVPTXMmaInfo getNVPTXMmaInfo(unsigned BuiltinID) {
212 // clang-format off
213#define MMA_VARIANTS(geom, type) \
214 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type, \
215 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
216 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type, \
217 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type
218#define MMA_SATF_VARIANTS(geom, type) \
219 MMA_VARIANTS(geom, type), \
220 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \
221 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
222 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \
223 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite
224// Sub-integer MMA only supports row.col layout.
225#define MMA_VARIANTS_I4(geom, type) \
226 0, \
227 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
228 0, \
229 0, \
230 0, \
231 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
232 0, \
233 0
234// b1 MMA does not support .satfinite.
235#define MMA_VARIANTS_B1_XOR(geom, type) \
236 0, \
237 Intrinsic::nvvm_wmma_##geom##_mma_xor_popc_row_col_##type, \
238 0, \
239 0, \
240 0, \
241 0, \
242 0, \
243 0
244#define MMA_VARIANTS_B1_AND(geom, type) \
245 0, \
246 Intrinsic::nvvm_wmma_##geom##_mma_and_popc_row_col_##type, \
247 0, \
248 0, \
249 0, \
250 0, \
251 0, \
252 0
253 // clang-format on
254 switch (BuiltinID) {
255 // FP MMA
256 // Note that 'type' argument of MMA_SATF_VARIANTS uses D_C notation, while
257 // NumEltsN of return value are ordered as A,B,C,D.
258 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
259 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 4, .NumEltsD: 4, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m16n16k16, f16_f16)}}};
260 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
261 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 4, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m16n16k16, f32_f16)}}};
262 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
263 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 8, .NumEltsD: 4, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m16n16k16, f16_f32)}}};
264 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
265 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m16n16k16, f32_f32)}}};
266 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
267 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 4, .NumEltsD: 4, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m32n8k16, f16_f16)}}};
268 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
269 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 4, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m32n8k16, f32_f16)}}};
270 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
271 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 8, .NumEltsD: 4, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m32n8k16, f16_f32)}}};
272 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
273 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m32n8k16, f32_f32)}}};
274 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
275 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 4, .NumEltsD: 4, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m8n32k16, f16_f16)}}};
276 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
277 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 4, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m8n32k16, f32_f16)}}};
278 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
279 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 8, .NumEltsD: 4, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m8n32k16, f16_f32)}}};
280 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
281 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m8n32k16, f32_f32)}}};
282
283 // Integer MMA
284 case NVPTX::BI__imma_m16n16k16_mma_s8:
285 return {.NumEltsA: 2, .NumEltsB: 2, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m16n16k16, s8)}}};
286 case NVPTX::BI__imma_m16n16k16_mma_u8:
287 return {.NumEltsA: 2, .NumEltsB: 2, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m16n16k16, u8)}}};
288 case NVPTX::BI__imma_m32n8k16_mma_s8:
289 return {.NumEltsA: 4, .NumEltsB: 1, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m32n8k16, s8)}}};
290 case NVPTX::BI__imma_m32n8k16_mma_u8:
291 return {.NumEltsA: 4, .NumEltsB: 1, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m32n8k16, u8)}}};
292 case NVPTX::BI__imma_m8n32k16_mma_s8:
293 return {.NumEltsA: 1, .NumEltsB: 4, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m8n32k16, s8)}}};
294 case NVPTX::BI__imma_m8n32k16_mma_u8:
295 return {.NumEltsA: 1, .NumEltsB: 4, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m8n32k16, u8)}}};
296
297 // Sub-integer MMA
298 case NVPTX::BI__imma_m8n8k32_mma_s4:
299 return {.NumEltsA: 1, .NumEltsB: 1, .NumEltsC: 2, .NumEltsD: 2, .Variants: {._M_elems: {MMA_VARIANTS_I4(m8n8k32, s4)}}};
300 case NVPTX::BI__imma_m8n8k32_mma_u4:
301 return {.NumEltsA: 1, .NumEltsB: 1, .NumEltsC: 2, .NumEltsD: 2, .Variants: {._M_elems: {MMA_VARIANTS_I4(m8n8k32, u4)}}};
302 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
303 return {.NumEltsA: 1, .NumEltsB: 1, .NumEltsC: 2, .NumEltsD: 2, .Variants: {._M_elems: {MMA_VARIANTS_B1_XOR(m8n8k128, b1)}}};
304 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
305 return {.NumEltsA: 1, .NumEltsB: 1, .NumEltsC: 2, .NumEltsD: 2, .Variants: {._M_elems: {MMA_VARIANTS_B1_AND(m8n8k128, b1)}}};
306
307 // Double MMA
308 case NVPTX::BI__dmma_m8n8k4_mma_f64:
309 return {.NumEltsA: 1, .NumEltsB: 1, .NumEltsC: 2, .NumEltsD: 2, .Variants: {._M_elems: {MMA_VARIANTS(m8n8k4, f64)}}};
310
311 // Alternate FP MMA
312 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
313 return {.NumEltsA: 4, .NumEltsB: 4, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_VARIANTS(m16n16k16, bf16)}}};
314 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
315 return {.NumEltsA: 2, .NumEltsB: 8, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_VARIANTS(m8n32k16, bf16)}}};
316 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
317 return {.NumEltsA: 8, .NumEltsB: 2, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_VARIANTS(m32n8k16, bf16)}}};
318 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
319 return {.NumEltsA: 4, .NumEltsB: 4, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_VARIANTS(m16n16k8, tf32)}}};
320 default:
321 llvm_unreachable("Unexpected builtin ID.");
322 }
323#undef MMA_VARIANTS
324#undef MMA_SATF_VARIANTS
325#undef MMA_VARIANTS_I4
326#undef MMA_VARIANTS_B1_AND
327#undef MMA_VARIANTS_B1_XOR
328}
329
330static Value *MakeLdu(unsigned IntrinsicID, CodeGenFunction &CGF,
331 const CallExpr *E) {
332 Value *Ptr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
333 QualType ArgType = E->getArg(Arg: 0)->getType();
334 clang::CharUnits Align = CGF.CGM.getNaturalPointeeTypeAlignment(T: ArgType);
335 llvm::Type *ElemTy = CGF.ConvertTypeForMem(T: ArgType->getPointeeType());
336 return CGF.Builder.CreateCall(
337 Callee: CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {ElemTy, Ptr->getType()}),
338 Args: {Ptr, ConstantInt::get(Ty: CGF.Builder.getInt32Ty(), V: Align.getQuantity())});
339}
340
341static Value *MakeLdg(CodeGenFunction &CGF, const CallExpr *E) {
342 Value *Ptr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
343 QualType ArgType = E->getArg(Arg: 0)->getType();
344 clang::CharUnits AlignV = CGF.CGM.getNaturalPointeeTypeAlignment(T: ArgType);
345 llvm::Type *ElemTy = CGF.ConvertTypeForMem(T: ArgType->getPointeeType());
346
347 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
348 auto *ASC = CGF.Builder.CreateAddrSpaceCast(V: Ptr, DestTy: CGF.Builder.getPtrTy(AddrSpace: 1));
349 auto *LD = CGF.Builder.CreateAlignedLoad(Ty: ElemTy, Ptr: ASC, Align: AlignV.getAsAlign());
350 MDNode *MD = MDNode::get(Context&: CGF.Builder.getContext(), MDs: {});
351 LD->setMetadata(KindID: LLVMContext::MD_invariant_load, Node: MD);
352
353 return LD;
354}
355
356// `Scope` is AtomicScope::Workgroup for _cta builtins and AtomicScope::System
357// for _sys builtins.
358static Value *MakeScopedAtomicRMW(CodeGenFunction &CGF, const CallExpr *E,
359 llvm::AtomicRMWInst::BinOp Kind,
360 llvm::AtomicScope Scope) {
361 Address Ptr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
362 Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
363 llvm::SyncScope::ID SSID = CGF.getLLVMContext().getOrInsertSyncScopeID(
364 SSN: *llvm::getAtomicScopeIRString(T: CGF.getTarget().getTriple(), S: Scope));
365 return CGF.Builder.CreateAtomicRMW(Op: Kind, Addr: Ptr, Val,
366 Ordering: llvm::AtomicOrdering::Monotonic, SSID);
367}
368
369// `Scope` is AtomicScope::Workgroup for _cta builtins and AtomicScope::System
370// for _sys builtins.
371static Value *MakeScopedAtomicCAS(CodeGenFunction &CGF, const CallExpr *E,
372 llvm::AtomicScope Scope) {
373 Address Ptr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
374 Value *Cmp = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
375 Value *New = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
376 llvm::SyncScope::ID SSID = CGF.getLLVMContext().getOrInsertSyncScopeID(
377 SSN: *llvm::getAtomicScopeIRString(T: CGF.getTarget().getTriple(), S: Scope));
378 Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
379 Addr: Ptr, Cmp, New, SuccessOrdering: llvm::AtomicOrdering::Monotonic,
380 FailureOrdering: llvm::AtomicOrdering::Monotonic, SSID);
381 return CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 0);
382}
383
384static Value *MakeCpAsync(unsigned IntrinsicID, unsigned IntrinsicIDS,
385 CodeGenFunction &CGF, const CallExpr *E,
386 int SrcSize) {
387 return E->getNumArgs() == 3
388 ? CGF.Builder.CreateCall(Callee: CGF.CGM.getIntrinsic(IID: IntrinsicIDS),
389 Args: {CGF.EmitScalarExpr(E: E->getArg(Arg: 0)),
390 CGF.EmitScalarExpr(E: E->getArg(Arg: 1)),
391 CGF.EmitScalarExpr(E: E->getArg(Arg: 2))})
392 : CGF.Builder.CreateCall(Callee: CGF.CGM.getIntrinsic(IID: IntrinsicID),
393 Args: {CGF.EmitScalarExpr(E: E->getArg(Arg: 0)),
394 CGF.EmitScalarExpr(E: E->getArg(Arg: 1))});
395}
396
397static Value *MakeHalfType(Function *Intrinsic, unsigned BuiltinID,
398 const CallExpr *E, CodeGenFunction &CGF,
399 ArrayRef<Value *> TrailingArgs = {}) {
400 SmallVector<Value *, 16> Args;
401 auto *FTy = Intrinsic->getFunctionType();
402 unsigned ICEArguments = 0;
403 ASTContext::GetBuiltinTypeError Error;
404 CGF.CGM.getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
405 assert(Error == ASTContext::GE_None && "Should not codegen an error");
406 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
407 assert((ICEArguments & (1 << i)) == 0);
408 auto *ArgValue = CGF.EmitScalarExpr(E: E->getArg(Arg: i));
409 auto *PTy = FTy->getParamType(i);
410 if (PTy != ArgValue->getType())
411 ArgValue = CGF.Builder.CreateBitCast(V: ArgValue, DestTy: PTy);
412 Args.push_back(Elt: ArgValue);
413 }
414
415 llvm::append_range(C&: Args, R&: TrailingArgs);
416 appendDefaultIntrinsicArgs(Args, F: Intrinsic);
417
418 return CGF.Builder.CreateCall(Callee: Intrinsic, Args);
419}
420
421static Value *MakeHalfType(unsigned IntrinsicID, unsigned BuiltinID,
422 const CallExpr *E, CodeGenFunction &CGF,
423 ArrayRef<Value *> TrailingArgs = {}) {
424 return MakeHalfType(Intrinsic: CGF.CGM.getIntrinsic(IID: IntrinsicID), BuiltinID, E, CGF,
425 TrailingArgs);
426}
427
428static Value *MakeFMAOOB(unsigned IntrinsicID, llvm::Type *Ty,
429 const CallExpr *E, CodeGenFunction &CGF) {
430 return CGF.Builder.CreateCall(Callee: CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Ty}),
431 Args: {CGF.EmitScalarExpr(E: E->getArg(Arg: 0)),
432 CGF.EmitScalarExpr(E: E->getArg(Arg: 1)),
433 CGF.EmitScalarExpr(E: E->getArg(Arg: 2))});
434}
435
436static Value *MakeFPArith(unsigned IntrinsicID, APFloat::roundingMode RM,
437 unsigned BuiltinID, const CallExpr *E,
438 CodeGenFunction &CGF) {
439 llvm::Type *Ty = CGF.ConvertType(T: E->getType());
440 return MakeHalfType(Intrinsic: CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Ty), BuiltinID, E, CGF,
441 TrailingArgs: {CGF.Builder.getInt32(C: static_cast<int>(RM))});
442}
443
444} // namespace
445
446Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
447 const CallExpr *E) {
448 switch (BuiltinID) {
449 case NVPTX::BI__nvvm_mulhi_i:
450 case NVPTX::BI__nvvm_mulhi_ui:
451 case NVPTX::BI__nvvm_mulhi_ll:
452 case NVPTX::BI__nvvm_mulhi_ull:
453 return Builder.CreateBinaryIntrinsic(
454 ID: E->getType()->hasSignedIntegerRepresentation() ? Intrinsic::smulh
455 : Intrinsic::umulh,
456 LHS: EmitScalarExpr(E: E->getArg(Arg: 0)), RHS: EmitScalarExpr(E: E->getArg(Arg: 1)));
457 case NVPTX::BI__nvvm_atom_add_gen_i:
458 case NVPTX::BI__nvvm_atom_add_gen_l:
459 case NVPTX::BI__nvvm_atom_add_gen_ll:
460 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::Add, E,
461 Ordering: AtomicOrdering::Monotonic);
462
463 case NVPTX::BI__nvvm_atom_sub_gen_i:
464 case NVPTX::BI__nvvm_atom_sub_gen_l:
465 case NVPTX::BI__nvvm_atom_sub_gen_ll:
466 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::Sub, E,
467 Ordering: AtomicOrdering::Monotonic);
468
469 case NVPTX::BI__nvvm_atom_and_gen_i:
470 case NVPTX::BI__nvvm_atom_and_gen_l:
471 case NVPTX::BI__nvvm_atom_and_gen_ll:
472 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::And, E,
473 Ordering: AtomicOrdering::Monotonic);
474
475 case NVPTX::BI__nvvm_atom_or_gen_i:
476 case NVPTX::BI__nvvm_atom_or_gen_l:
477 case NVPTX::BI__nvvm_atom_or_gen_ll:
478 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::Or, E,
479 Ordering: AtomicOrdering::Monotonic);
480
481 case NVPTX::BI__nvvm_atom_xor_gen_i:
482 case NVPTX::BI__nvvm_atom_xor_gen_l:
483 case NVPTX::BI__nvvm_atom_xor_gen_ll:
484 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::Xor, E,
485 Ordering: AtomicOrdering::Monotonic);
486
487 case NVPTX::BI__nvvm_atom_xchg_gen_i:
488 case NVPTX::BI__nvvm_atom_xchg_gen_l:
489 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
490 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::Xchg, E,
491 Ordering: AtomicOrdering::Monotonic);
492
493 case NVPTX::BI__nvvm_atom_max_gen_i:
494 case NVPTX::BI__nvvm_atom_max_gen_l:
495 case NVPTX::BI__nvvm_atom_max_gen_ll:
496 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::Max, E,
497 Ordering: AtomicOrdering::Monotonic);
498
499 case NVPTX::BI__nvvm_atom_max_gen_ui:
500 case NVPTX::BI__nvvm_atom_max_gen_ul:
501 case NVPTX::BI__nvvm_atom_max_gen_ull:
502 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::UMax, E,
503 Ordering: AtomicOrdering::Monotonic);
504
505 case NVPTX::BI__nvvm_atom_min_gen_i:
506 case NVPTX::BI__nvvm_atom_min_gen_l:
507 case NVPTX::BI__nvvm_atom_min_gen_ll:
508 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::Min, E,
509 Ordering: AtomicOrdering::Monotonic);
510
511 case NVPTX::BI__nvvm_atom_min_gen_ui:
512 case NVPTX::BI__nvvm_atom_min_gen_ul:
513 case NVPTX::BI__nvvm_atom_min_gen_ull:
514 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::UMin, E,
515 Ordering: AtomicOrdering::Monotonic);
516
517 case NVPTX::BI__nvvm_atom_cas_gen_us:
518 case NVPTX::BI__nvvm_atom_cas_gen_i:
519 case NVPTX::BI__nvvm_atom_cas_gen_l:
520 case NVPTX::BI__nvvm_atom_cas_gen_ll:
521 // __nvvm_atom_cas_gen_* should return the old value rather than the
522 // success flag.
523 return MakeAtomicCmpXchgValue(CGF&: *this, E, /*ReturnBool=*/false,
524 SuccessOrdering: AtomicOrdering::Monotonic,
525 FailureOrdering: AtomicOrdering::Monotonic);
526
527 case NVPTX::BI__nvvm_atom_add_gen_f:
528 case NVPTX::BI__nvvm_atom_add_gen_d: {
529 Address DestAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
530 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 1));
531
532 return Builder.CreateAtomicRMW(Op: llvm::AtomicRMWInst::FAdd, Addr: DestAddr, Val,
533 Ordering: AtomicOrdering::Monotonic);
534 }
535
536 case NVPTX::BI__nvvm_atom_inc_gen_ui:
537 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::UIncWrap, E,
538 Ordering: AtomicOrdering::Monotonic);
539
540 case NVPTX::BI__nvvm_atom_dec_gen_ui:
541 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::UDecWrap, E,
542 Ordering: AtomicOrdering::Monotonic);
543
544 case NVPTX::BI__nvvm_ldg_c:
545 case NVPTX::BI__nvvm_ldg_sc:
546 case NVPTX::BI__nvvm_ldg_c2:
547 case NVPTX::BI__nvvm_ldg_sc2:
548 case NVPTX::BI__nvvm_ldg_c4:
549 case NVPTX::BI__nvvm_ldg_sc4:
550 case NVPTX::BI__nvvm_ldg_s:
551 case NVPTX::BI__nvvm_ldg_s2:
552 case NVPTX::BI__nvvm_ldg_s4:
553 case NVPTX::BI__nvvm_ldg_i:
554 case NVPTX::BI__nvvm_ldg_i2:
555 case NVPTX::BI__nvvm_ldg_i4:
556 case NVPTX::BI__nvvm_ldg_l:
557 case NVPTX::BI__nvvm_ldg_l2:
558 case NVPTX::BI__nvvm_ldg_ll:
559 case NVPTX::BI__nvvm_ldg_ll2:
560 case NVPTX::BI__nvvm_ldg_uc:
561 case NVPTX::BI__nvvm_ldg_uc2:
562 case NVPTX::BI__nvvm_ldg_uc4:
563 case NVPTX::BI__nvvm_ldg_us:
564 case NVPTX::BI__nvvm_ldg_us2:
565 case NVPTX::BI__nvvm_ldg_us4:
566 case NVPTX::BI__nvvm_ldg_ui:
567 case NVPTX::BI__nvvm_ldg_ui2:
568 case NVPTX::BI__nvvm_ldg_ui4:
569 case NVPTX::BI__nvvm_ldg_ul:
570 case NVPTX::BI__nvvm_ldg_ul2:
571 case NVPTX::BI__nvvm_ldg_ull:
572 case NVPTX::BI__nvvm_ldg_ull2:
573 case NVPTX::BI__nvvm_ldg_f:
574 case NVPTX::BI__nvvm_ldg_f2:
575 case NVPTX::BI__nvvm_ldg_f4:
576 case NVPTX::BI__nvvm_ldg_d:
577 case NVPTX::BI__nvvm_ldg_d2:
578 // PTX Interoperability section 2.2: "For a vector with an even number of
579 // elements, its alignment is set to number of elements times the alignment
580 // of its member: n*alignof(t)."
581 return MakeLdg(CGF&: *this, E);
582
583 case NVPTX::BI__nvvm_ldu_c:
584 case NVPTX::BI__nvvm_ldu_sc:
585 case NVPTX::BI__nvvm_ldu_c2:
586 case NVPTX::BI__nvvm_ldu_sc2:
587 case NVPTX::BI__nvvm_ldu_c4:
588 case NVPTX::BI__nvvm_ldu_sc4:
589 case NVPTX::BI__nvvm_ldu_s:
590 case NVPTX::BI__nvvm_ldu_s2:
591 case NVPTX::BI__nvvm_ldu_s4:
592 case NVPTX::BI__nvvm_ldu_i:
593 case NVPTX::BI__nvvm_ldu_i2:
594 case NVPTX::BI__nvvm_ldu_i4:
595 case NVPTX::BI__nvvm_ldu_l:
596 case NVPTX::BI__nvvm_ldu_l2:
597 case NVPTX::BI__nvvm_ldu_ll:
598 case NVPTX::BI__nvvm_ldu_ll2:
599 case NVPTX::BI__nvvm_ldu_uc:
600 case NVPTX::BI__nvvm_ldu_uc2:
601 case NVPTX::BI__nvvm_ldu_uc4:
602 case NVPTX::BI__nvvm_ldu_us:
603 case NVPTX::BI__nvvm_ldu_us2:
604 case NVPTX::BI__nvvm_ldu_us4:
605 case NVPTX::BI__nvvm_ldu_ui:
606 case NVPTX::BI__nvvm_ldu_ui2:
607 case NVPTX::BI__nvvm_ldu_ui4:
608 case NVPTX::BI__nvvm_ldu_ul:
609 case NVPTX::BI__nvvm_ldu_ul2:
610 case NVPTX::BI__nvvm_ldu_ull:
611 case NVPTX::BI__nvvm_ldu_ull2:
612 return MakeLdu(IntrinsicID: Intrinsic::nvvm_ldu_global_i, CGF&: *this, E);
613 case NVPTX::BI__nvvm_ldu_f:
614 case NVPTX::BI__nvvm_ldu_f2:
615 case NVPTX::BI__nvvm_ldu_f4:
616 case NVPTX::BI__nvvm_ldu_d:
617 case NVPTX::BI__nvvm_ldu_d2:
618 return MakeLdu(IntrinsicID: Intrinsic::nvvm_ldu_global_f, CGF&: *this, E);
619
620 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
621 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
622 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
623 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Add,
624 Scope: llvm::AtomicScope::Workgroup);
625 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
626 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
627 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
628 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Add,
629 Scope: llvm::AtomicScope::System);
630 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
631 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
632 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::FAdd,
633 Scope: llvm::AtomicScope::Workgroup);
634 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
635 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
636 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::FAdd,
637 Scope: llvm::AtomicScope::System);
638 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
639 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
640 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
641 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Xchg,
642 Scope: llvm::AtomicScope::Workgroup);
643 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
644 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
645 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
646 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Xchg,
647 Scope: llvm::AtomicScope::System);
648 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
649 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
650 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
651 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Max,
652 Scope: llvm::AtomicScope::Workgroup);
653 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
654 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
655 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
656 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::UMax,
657 Scope: llvm::AtomicScope::Workgroup);
658 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
659 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
660 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
661 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Max,
662 Scope: llvm::AtomicScope::System);
663 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
664 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
665 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
666 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::UMax,
667 Scope: llvm::AtomicScope::System);
668 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
669 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
670 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
671 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Min,
672 Scope: llvm::AtomicScope::Workgroup);
673 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
674 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
675 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
676 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::UMin,
677 Scope: llvm::AtomicScope::Workgroup);
678 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
679 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
680 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
681 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Min,
682 Scope: llvm::AtomicScope::System);
683 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
684 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
685 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
686 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::UMin,
687 Scope: llvm::AtomicScope::System);
688 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
689 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::UIncWrap,
690 Scope: llvm::AtomicScope::Workgroup);
691 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
692 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::UDecWrap,
693 Scope: llvm::AtomicScope::Workgroup);
694 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
695 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::UIncWrap,
696 Scope: llvm::AtomicScope::System);
697 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
698 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::UDecWrap,
699 Scope: llvm::AtomicScope::System);
700 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
701 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
702 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
703 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::And,
704 Scope: llvm::AtomicScope::Workgroup);
705 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
706 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
707 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
708 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::And,
709 Scope: llvm::AtomicScope::System);
710 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
711 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
712 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
713 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Or,
714 Scope: llvm::AtomicScope::Workgroup);
715 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
716 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
717 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
718 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Or,
719 Scope: llvm::AtomicScope::System);
720 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
721 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
722 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
723 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Xor,
724 Scope: llvm::AtomicScope::Workgroup);
725 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
726 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
727 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
728 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Xor,
729 Scope: llvm::AtomicScope::System);
730 case NVPTX::BI__nvvm_atom_cta_cas_gen_us:
731 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
732 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
733 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll:
734 return MakeScopedAtomicCAS(CGF&: *this, E, Scope: llvm::AtomicScope::Workgroup);
735 case NVPTX::BI__nvvm_atom_sys_cas_gen_us:
736 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
737 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
738 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll:
739 return MakeScopedAtomicCAS(CGF&: *this, E, Scope: llvm::AtomicScope::System);
740 case NVPTX::BI__nvvm_match_all_sync_i32p:
741 case NVPTX::BI__nvvm_match_all_sync_i64p: {
742 Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
743 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 1));
744 Address PredOutPtr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 2));
745 Value *ResultPair = Builder.CreateCall(
746 Callee: CGM.getIntrinsic(IID: BuiltinID == NVPTX::BI__nvvm_match_all_sync_i32p
747 ? Intrinsic::nvvm_match_all_sync_i32p
748 : Intrinsic::nvvm_match_all_sync_i64p),
749 Args: {Mask, Val});
750 Value *Pred = Builder.CreateZExt(V: Builder.CreateExtractValue(Agg: ResultPair, Idxs: 1),
751 DestTy: PredOutPtr.getElementType());
752 Builder.CreateStore(Val: Pred, Addr: PredOutPtr);
753 return Builder.CreateExtractValue(Agg: ResultPair, Idxs: 0);
754 }
755
756 // FP MMA loads
757 case NVPTX::BI__hmma_m16n16k16_ld_a:
758 case NVPTX::BI__hmma_m16n16k16_ld_b:
759 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
760 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
761 case NVPTX::BI__hmma_m32n8k16_ld_a:
762 case NVPTX::BI__hmma_m32n8k16_ld_b:
763 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
764 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
765 case NVPTX::BI__hmma_m8n32k16_ld_a:
766 case NVPTX::BI__hmma_m8n32k16_ld_b:
767 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
768 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
769 // Integer MMA loads.
770 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
771 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
772 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
773 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
774 case NVPTX::BI__imma_m16n16k16_ld_c:
775 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
776 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
777 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
778 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
779 case NVPTX::BI__imma_m32n8k16_ld_c:
780 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
781 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
782 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
783 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
784 case NVPTX::BI__imma_m8n32k16_ld_c:
785 // Sub-integer MMA loads.
786 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
787 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
788 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
789 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
790 case NVPTX::BI__imma_m8n8k32_ld_c:
791 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
792 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
793 case NVPTX::BI__bmma_m8n8k128_ld_c:
794 // Double MMA loads.
795 case NVPTX::BI__dmma_m8n8k4_ld_a:
796 case NVPTX::BI__dmma_m8n8k4_ld_b:
797 case NVPTX::BI__dmma_m8n8k4_ld_c:
798 // Alternate float MMA loads.
799 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
800 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
801 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
802 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
803 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
804 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
805 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
806 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
807 case NVPTX::BI__mma_tf32_m16n16k8_ld_c: {
808 Address Dst = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
809 Value *Src = EmitScalarExpr(E: E->getArg(Arg: 1));
810 Value *Ldm = EmitScalarExpr(E: E->getArg(Arg: 2));
811 std::optional<llvm::APSInt> isColMajorArg =
812 E->getArg(Arg: 3)->getIntegerConstantExpr(Ctx: getContext());
813 if (!isColMajorArg)
814 return nullptr;
815 bool isColMajor = isColMajorArg->getSExtValue();
816 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
817 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
818 if (IID == 0)
819 return nullptr;
820
821 Value *Result =
822 Builder.CreateCall(Callee: CGM.getIntrinsic(IID, Tys: Src->getType()), Args: {Src, Ldm});
823
824 // Save returned values.
825 assert(II.NumResults);
826 if (II.NumResults == 1) {
827 Builder.CreateAlignedStore(Val: Result, Addr: Dst.emitRawPointer(CGF&: *this),
828 Align: CharUnits::fromQuantity(Quantity: 4));
829 } else {
830 for (unsigned i = 0; i < II.NumResults; ++i) {
831 Builder.CreateAlignedStore(
832 Val: Builder.CreateBitCast(V: Builder.CreateExtractValue(Agg: Result, Idxs: i),
833 DestTy: Dst.getElementType()),
834 Addr: Builder.CreateGEP(Ty: Dst.getElementType(), Ptr: Dst.emitRawPointer(CGF&: *this),
835 IdxList: llvm::ConstantInt::get(Ty: IntTy, V: i)),
836 Align: CharUnits::fromQuantity(Quantity: 4));
837 }
838 }
839 return Result;
840 }
841
842 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
843 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
844 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
845 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
846 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
847 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
848 case NVPTX::BI__imma_m16n16k16_st_c_i32:
849 case NVPTX::BI__imma_m32n8k16_st_c_i32:
850 case NVPTX::BI__imma_m8n32k16_st_c_i32:
851 case NVPTX::BI__imma_m8n8k32_st_c_i32:
852 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
853 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
854 case NVPTX::BI__mma_m16n16k8_st_c_f32: {
855 Value *Dst = EmitScalarExpr(E: E->getArg(Arg: 0));
856 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
857 Value *Ldm = EmitScalarExpr(E: E->getArg(Arg: 2));
858 std::optional<llvm::APSInt> isColMajorArg =
859 E->getArg(Arg: 3)->getIntegerConstantExpr(Ctx: getContext());
860 if (!isColMajorArg)
861 return nullptr;
862 bool isColMajor = isColMajorArg->getSExtValue();
863 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
864 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
865 if (IID == 0)
866 return nullptr;
867 Function *Intrinsic =
868 CGM.getIntrinsic(IID, Tys: Dst->getType());
869 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(i: 1);
870 SmallVector<Value *, 10> Values = {Dst};
871 for (unsigned i = 0; i < II.NumResults; ++i) {
872 Value *V = Builder.CreateAlignedLoad(
873 Ty: Src.getElementType(),
874 Addr: Builder.CreateGEP(Ty: Src.getElementType(), Ptr: Src.emitRawPointer(CGF&: *this),
875 IdxList: llvm::ConstantInt::get(Ty: IntTy, V: i)),
876 Align: CharUnits::fromQuantity(Quantity: 4));
877 Values.push_back(Elt: Builder.CreateBitCast(V, DestTy: ParamType));
878 }
879 Values.push_back(Elt: Ldm);
880 Value *Result = Builder.CreateCall(Callee: Intrinsic, Args: Values);
881 return Result;
882 }
883
884 // BI__hmma_m16n16k16_mma_<Dtype><CType>(d, a, b, c, layout, satf) -->
885 // Intrinsic::nvvm_wmma_m16n16k16_mma_sync<layout A,B><DType><CType><Satf>
886 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
887 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
888 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
889 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
890 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
891 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
892 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
893 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
894 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
895 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
896 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
897 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
898 case NVPTX::BI__imma_m16n16k16_mma_s8:
899 case NVPTX::BI__imma_m16n16k16_mma_u8:
900 case NVPTX::BI__imma_m32n8k16_mma_s8:
901 case NVPTX::BI__imma_m32n8k16_mma_u8:
902 case NVPTX::BI__imma_m8n32k16_mma_s8:
903 case NVPTX::BI__imma_m8n32k16_mma_u8:
904 case NVPTX::BI__imma_m8n8k32_mma_s4:
905 case NVPTX::BI__imma_m8n8k32_mma_u4:
906 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
907 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
908 case NVPTX::BI__dmma_m8n8k4_mma_f64:
909 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
910 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
911 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
912 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32: {
913 Address Dst = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
914 Address SrcA = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
915 Address SrcB = EmitPointerWithAlignment(Addr: E->getArg(Arg: 2));
916 Address SrcC = EmitPointerWithAlignment(Addr: E->getArg(Arg: 3));
917 std::optional<llvm::APSInt> LayoutArg =
918 E->getArg(Arg: 4)->getIntegerConstantExpr(Ctx: getContext());
919 if (!LayoutArg)
920 return nullptr;
921 int Layout = LayoutArg->getSExtValue();
922 if (Layout < 0 || Layout > 3)
923 return nullptr;
924 llvm::APSInt SatfArg;
925 if (BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1 ||
926 BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1)
927 SatfArg = 0; // .b1 does not have satf argument.
928 else if (std::optional<llvm::APSInt> OptSatfArg =
929 E->getArg(Arg: 5)->getIntegerConstantExpr(Ctx: getContext()))
930 SatfArg = *OptSatfArg;
931 else
932 return nullptr;
933 bool Satf = SatfArg.getSExtValue();
934 NVPTXMmaInfo MI = getNVPTXMmaInfo(BuiltinID);
935 unsigned IID = MI.getMMAIntrinsic(Layout, Satf);
936 if (IID == 0) // Unsupported combination of Layout/Satf.
937 return nullptr;
938
939 SmallVector<Value *, 24> Values;
940 Function *Intrinsic = CGM.getIntrinsic(IID);
941 llvm::Type *AType = Intrinsic->getFunctionType()->getParamType(i: 0);
942 // Load A
943 for (unsigned i = 0; i < MI.NumEltsA; ++i) {
944 Value *V = Builder.CreateAlignedLoad(
945 Ty: SrcA.getElementType(),
946 Addr: Builder.CreateGEP(Ty: SrcA.getElementType(), Ptr: SrcA.emitRawPointer(CGF&: *this),
947 IdxList: llvm::ConstantInt::get(Ty: IntTy, V: i)),
948 Align: CharUnits::fromQuantity(Quantity: 4));
949 Values.push_back(Elt: Builder.CreateBitCast(V, DestTy: AType));
950 }
951 // Load B
952 llvm::Type *BType = Intrinsic->getFunctionType()->getParamType(i: MI.NumEltsA);
953 for (unsigned i = 0; i < MI.NumEltsB; ++i) {
954 Value *V = Builder.CreateAlignedLoad(
955 Ty: SrcB.getElementType(),
956 Addr: Builder.CreateGEP(Ty: SrcB.getElementType(), Ptr: SrcB.emitRawPointer(CGF&: *this),
957 IdxList: llvm::ConstantInt::get(Ty: IntTy, V: i)),
958 Align: CharUnits::fromQuantity(Quantity: 4));
959 Values.push_back(Elt: Builder.CreateBitCast(V, DestTy: BType));
960 }
961 // Load C
962 llvm::Type *CType =
963 Intrinsic->getFunctionType()->getParamType(i: MI.NumEltsA + MI.NumEltsB);
964 for (unsigned i = 0; i < MI.NumEltsC; ++i) {
965 Value *V = Builder.CreateAlignedLoad(
966 Ty: SrcC.getElementType(),
967 Addr: Builder.CreateGEP(Ty: SrcC.getElementType(), Ptr: SrcC.emitRawPointer(CGF&: *this),
968 IdxList: llvm::ConstantInt::get(Ty: IntTy, V: i)),
969 Align: CharUnits::fromQuantity(Quantity: 4));
970 Values.push_back(Elt: Builder.CreateBitCast(V, DestTy: CType));
971 }
972 Value *Result = Builder.CreateCall(Callee: Intrinsic, Args: Values);
973 llvm::Type *DType = Dst.getElementType();
974 for (unsigned i = 0; i < MI.NumEltsD; ++i)
975 Builder.CreateAlignedStore(
976 Val: Builder.CreateBitCast(V: Builder.CreateExtractValue(Agg: Result, Idxs: i), DestTy: DType),
977 Addr: Builder.CreateGEP(Ty: Dst.getElementType(), Ptr: Dst.emitRawPointer(CGF&: *this),
978 IdxList: llvm::ConstantInt::get(Ty: IntTy, V: i)),
979 Align: CharUnits::fromQuantity(Quantity: 4));
980 return Result;
981 }
982 // The following builtins require half type support
983 case NVPTX::BI__nvvm_ex2_approx_f16:
984 return MakeHalfType(
985 Intrinsic: CGM.getIntrinsic(IID: Intrinsic::nvvm_ex2_approx, Tys: Builder.getHalfTy()),
986 BuiltinID, E, CGF&: *this);
987 case NVPTX::BI__nvvm_ex2_approx_f16x2:
988 return MakeHalfType(
989 Intrinsic: CGM.getIntrinsic(IID: Intrinsic::nvvm_ex2_approx,
990 Tys: FixedVectorType::get(ElementType: Builder.getHalfTy(), NumElts: 2)),
991 BuiltinID, E, CGF&: *this);
992 case NVPTX::BI__nvvm_ff2f16x2_rn:
993 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_ff2f16x2_rn, BuiltinID, E, CGF&: *this);
994 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
995 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_ff2f16x2_rn_relu, BuiltinID, E, CGF&: *this);
996 case NVPTX::BI__nvvm_ff2f16x2_rz:
997 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_ff2f16x2_rz, BuiltinID, E, CGF&: *this);
998 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
999 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_ff2f16x2_rz_relu, BuiltinID, E, CGF&: *this);
1000
1001#define PZO_CVT(cvt) \
1002 case NVPTX::BI__nvvm_##cvt##_pzo: \
1003 return MakeHalfType(Intrinsic::nvvm_##cvt, BuiltinID, E, *this, \
1004 {Builder.getTrue()})
1005
1006 PZO_CVT(ff2f16x2_rn);
1007 PZO_CVT(ff2f16x2_rn_relu);
1008 PZO_CVT(ff2f16x2_rz);
1009 PZO_CVT(ff2f16x2_rz_relu);
1010 PZO_CVT(ff2f16x2_rn_satfinite);
1011 PZO_CVT(ff2f16x2_rn_relu_satfinite);
1012 PZO_CVT(ff2f16x2_rz_satfinite);
1013 PZO_CVT(ff2f16x2_rz_relu_satfinite);
1014 PZO_CVT(ff2bf16x2_rn);
1015 PZO_CVT(ff2bf16x2_rn_relu);
1016 PZO_CVT(ff2bf16x2_rz);
1017 PZO_CVT(ff2bf16x2_rz_relu);
1018 PZO_CVT(ff2bf16x2_rn_satfinite);
1019 PZO_CVT(ff2bf16x2_rn_relu_satfinite);
1020 PZO_CVT(ff2bf16x2_rz_satfinite);
1021 PZO_CVT(ff2bf16x2_rz_relu_satfinite);
1022 PZO_CVT(f2f16_rn);
1023 PZO_CVT(f2f16_rn_relu);
1024 PZO_CVT(f2f16_rz);
1025 PZO_CVT(f2f16_rz_relu);
1026 PZO_CVT(f2f16_rn_satfinite);
1027 PZO_CVT(f2f16_rn_relu_satfinite);
1028 PZO_CVT(f2f16_rz_satfinite);
1029 PZO_CVT(f2f16_rz_relu_satfinite);
1030 PZO_CVT(f2bf16_rn);
1031 PZO_CVT(f2bf16_rn_relu);
1032 PZO_CVT(f2bf16_rz);
1033 PZO_CVT(f2bf16_rz_relu);
1034 PZO_CVT(f2bf16_rn_satfinite);
1035 PZO_CVT(f2bf16_rn_relu_satfinite);
1036 PZO_CVT(f2bf16_rz_satfinite);
1037 PZO_CVT(f2bf16_rz_relu_satfinite);
1038
1039 PZO_CVT(ff_to_e4m3x2_rn);
1040 PZO_CVT(ff_to_e4m3x2_rn_relu);
1041 PZO_CVT(ff_to_e4m3x2_rz);
1042 PZO_CVT(ff_to_e4m3x2_rz_relu);
1043 PZO_CVT(ff_to_e5m2x2_rn);
1044 PZO_CVT(ff_to_e5m2x2_rn_relu);
1045 PZO_CVT(ff_to_e5m2x2_rz);
1046 PZO_CVT(ff_to_e5m2x2_rz_relu);
1047 PZO_CVT(f16x2_to_e4m3x2_rn);
1048 PZO_CVT(f16x2_to_e4m3x2_rn_relu);
1049 PZO_CVT(f16x2_to_e4m3x2_rz);
1050 PZO_CVT(f16x2_to_e4m3x2_rz_relu);
1051 PZO_CVT(f16x2_to_e5m2x2_rn);
1052 PZO_CVT(f16x2_to_e5m2x2_rn_relu);
1053 PZO_CVT(f16x2_to_e5m2x2_rz);
1054 PZO_CVT(f16x2_to_e5m2x2_rz_relu);
1055 PZO_CVT(bf16x2_to_e4m3x2_rn_satfinite);
1056 PZO_CVT(bf16x2_to_e4m3x2_rn_relu_satfinite);
1057 PZO_CVT(bf16x2_to_e4m3x2_rz_satfinite);
1058 PZO_CVT(bf16x2_to_e4m3x2_rz_relu_satfinite);
1059 PZO_CVT(bf16x2_to_e5m2x2_rn_satfinite);
1060 PZO_CVT(bf16x2_to_e5m2x2_rn_relu_satfinite);
1061 PZO_CVT(bf16x2_to_e5m2x2_rz_satfinite);
1062 PZO_CVT(bf16x2_to_e5m2x2_rz_relu_satfinite);
1063
1064 PZO_CVT(ff_to_e4m3x2_rn_scale_n1_ue8m0);
1065 PZO_CVT(ff_to_e4m3x2_rn_relu_scale_n1_ue8m0);
1066 PZO_CVT(ff_to_e4m3x2_rz_scale_n1_ue8m0);
1067 PZO_CVT(ff_to_e4m3x2_rz_relu_scale_n1_ue8m0);
1068 PZO_CVT(ff_to_e5m2x2_rn_scale_n1_ue8m0);
1069 PZO_CVT(ff_to_e5m2x2_rn_relu_scale_n1_ue8m0);
1070 PZO_CVT(ff_to_e5m2x2_rz_scale_n1_ue8m0);
1071 PZO_CVT(ff_to_e5m2x2_rz_relu_scale_n1_ue8m0);
1072 PZO_CVT(f16x2_to_e4m3x2_rn_scale_n1_ue8m0);
1073 PZO_CVT(f16x2_to_e4m3x2_rn_relu_scale_n1_ue8m0);
1074 PZO_CVT(f16x2_to_e4m3x2_rz_scale_n1_ue8m0);
1075 PZO_CVT(f16x2_to_e4m3x2_rz_relu_scale_n1_ue8m0);
1076 PZO_CVT(f16x2_to_e5m2x2_rn_scale_n1_ue8m0);
1077 PZO_CVT(f16x2_to_e5m2x2_rn_relu_scale_n1_ue8m0);
1078 PZO_CVT(f16x2_to_e5m2x2_rz_scale_n1_ue8m0);
1079 PZO_CVT(f16x2_to_e5m2x2_rz_relu_scale_n1_ue8m0);
1080 PZO_CVT(bf16x2_to_e4m3x2_rn_satfinite_scale_n1_ue8m0);
1081 PZO_CVT(bf16x2_to_e4m3x2_rn_relu_satfinite_scale_n1_ue8m0);
1082 PZO_CVT(bf16x2_to_e4m3x2_rz_satfinite_scale_n1_ue8m0);
1083 PZO_CVT(bf16x2_to_e4m3x2_rz_relu_satfinite_scale_n1_ue8m0);
1084 PZO_CVT(bf16x2_to_e5m2x2_rn_satfinite_scale_n1_ue8m0);
1085 PZO_CVT(bf16x2_to_e5m2x2_rn_relu_satfinite_scale_n1_ue8m0);
1086 PZO_CVT(bf16x2_to_e5m2x2_rz_satfinite_scale_n1_ue8m0);
1087 PZO_CVT(bf16x2_to_e5m2x2_rz_relu_satfinite_scale_n1_ue8m0);
1088
1089 PZO_CVT(ff_to_e2m3x2_rn_satfinite);
1090 PZO_CVT(ff_to_e2m3x2_rn_relu_satfinite);
1091 PZO_CVT(ff_to_e2m3x2_rz_satfinite);
1092 PZO_CVT(ff_to_e2m3x2_rz_relu_satfinite);
1093 PZO_CVT(ff_to_e3m2x2_rn_satfinite);
1094 PZO_CVT(ff_to_e3m2x2_rn_relu_satfinite);
1095 PZO_CVT(ff_to_e3m2x2_rz_satfinite);
1096 PZO_CVT(ff_to_e3m2x2_rz_relu_satfinite);
1097 PZO_CVT(f16x2_to_e2m3x2_rn_satfinite);
1098 PZO_CVT(f16x2_to_e2m3x2_rn_relu_satfinite);
1099 PZO_CVT(f16x2_to_e2m3x2_rz_satfinite);
1100 PZO_CVT(f16x2_to_e2m3x2_rz_relu_satfinite);
1101 PZO_CVT(f16x2_to_e3m2x2_rn_satfinite);
1102 PZO_CVT(f16x2_to_e3m2x2_rn_relu_satfinite);
1103 PZO_CVT(f16x2_to_e3m2x2_rz_satfinite);
1104 PZO_CVT(f16x2_to_e3m2x2_rz_relu_satfinite);
1105 PZO_CVT(bf16x2_to_e2m3x2_rn_satfinite);
1106 PZO_CVT(bf16x2_to_e2m3x2_rn_relu_satfinite);
1107 PZO_CVT(bf16x2_to_e2m3x2_rz_satfinite);
1108 PZO_CVT(bf16x2_to_e2m3x2_rz_relu_satfinite);
1109 PZO_CVT(bf16x2_to_e3m2x2_rn_satfinite);
1110 PZO_CVT(bf16x2_to_e3m2x2_rn_relu_satfinite);
1111 PZO_CVT(bf16x2_to_e3m2x2_rz_satfinite);
1112 PZO_CVT(bf16x2_to_e3m2x2_rz_relu_satfinite);
1113
1114 PZO_CVT(ff_to_e2m3x2_rn_satfinite_scale_n1_ue8m0);
1115 PZO_CVT(ff_to_e2m3x2_rn_relu_satfinite_scale_n1_ue8m0);
1116 PZO_CVT(ff_to_e2m3x2_rz_satfinite_scale_n1_ue8m0);
1117 PZO_CVT(ff_to_e2m3x2_rz_relu_satfinite_scale_n1_ue8m0);
1118 PZO_CVT(ff_to_e3m2x2_rn_satfinite_scale_n1_ue8m0);
1119 PZO_CVT(ff_to_e3m2x2_rn_relu_satfinite_scale_n1_ue8m0);
1120 PZO_CVT(ff_to_e3m2x2_rz_satfinite_scale_n1_ue8m0);
1121 PZO_CVT(ff_to_e3m2x2_rz_relu_satfinite_scale_n1_ue8m0);
1122 PZO_CVT(f16x2_to_e2m3x2_rn_satfinite_scale_n1_ue8m0);
1123 PZO_CVT(f16x2_to_e2m3x2_rn_relu_satfinite_scale_n1_ue8m0);
1124 PZO_CVT(f16x2_to_e2m3x2_rz_satfinite_scale_n1_ue8m0);
1125 PZO_CVT(f16x2_to_e2m3x2_rz_relu_satfinite_scale_n1_ue8m0);
1126 PZO_CVT(f16x2_to_e3m2x2_rn_satfinite_scale_n1_ue8m0);
1127 PZO_CVT(f16x2_to_e3m2x2_rn_relu_satfinite_scale_n1_ue8m0);
1128 PZO_CVT(f16x2_to_e3m2x2_rz_satfinite_scale_n1_ue8m0);
1129 PZO_CVT(f16x2_to_e3m2x2_rz_relu_satfinite_scale_n1_ue8m0);
1130 PZO_CVT(bf16x2_to_e2m3x2_rn_satfinite_scale_n1_ue8m0);
1131 PZO_CVT(bf16x2_to_e2m3x2_rn_relu_satfinite_scale_n1_ue8m0);
1132 PZO_CVT(bf16x2_to_e2m3x2_rz_satfinite_scale_n1_ue8m0);
1133 PZO_CVT(bf16x2_to_e2m3x2_rz_relu_satfinite_scale_n1_ue8m0);
1134 PZO_CVT(bf16x2_to_e3m2x2_rn_satfinite_scale_n1_ue8m0);
1135 PZO_CVT(bf16x2_to_e3m2x2_rn_relu_satfinite_scale_n1_ue8m0);
1136 PZO_CVT(bf16x2_to_e3m2x2_rz_satfinite_scale_n1_ue8m0);
1137 PZO_CVT(bf16x2_to_e3m2x2_rz_relu_satfinite_scale_n1_ue8m0);
1138
1139 PZO_CVT(ff_to_e2m1x2_rn_satfinite);
1140 PZO_CVT(ff_to_e2m1x2_rn_relu_satfinite);
1141 PZO_CVT(ff_to_e2m1x2_rz_satfinite);
1142 PZO_CVT(ff_to_e2m1x2_rz_relu_satfinite);
1143 PZO_CVT(f16x2_to_e2m1x2_rn_satfinite);
1144 PZO_CVT(f16x2_to_e2m1x2_rn_relu_satfinite);
1145 PZO_CVT(f16x2_to_e2m1x2_rz_satfinite);
1146 PZO_CVT(f16x2_to_e2m1x2_rz_relu_satfinite);
1147 PZO_CVT(bf16x2_to_e2m1x2_rn_satfinite);
1148 PZO_CVT(bf16x2_to_e2m1x2_rn_relu_satfinite);
1149 PZO_CVT(bf16x2_to_e2m1x2_rz_satfinite);
1150 PZO_CVT(bf16x2_to_e2m1x2_rz_relu_satfinite);
1151
1152 PZO_CVT(ff_to_e2m1x2_rn_satfinite_scale_n1_ue8m0);
1153 PZO_CVT(ff_to_e2m1x2_rn_relu_satfinite_scale_n1_ue8m0);
1154 PZO_CVT(ff_to_e2m1x2_rz_satfinite_scale_n1_ue8m0);
1155 PZO_CVT(ff_to_e2m1x2_rz_relu_satfinite_scale_n1_ue8m0);
1156 PZO_CVT(f16x2_to_e2m1x2_rn_satfinite_scale_n1_ue8m0);
1157 PZO_CVT(f16x2_to_e2m1x2_rn_relu_satfinite_scale_n1_ue8m0);
1158 PZO_CVT(f16x2_to_e2m1x2_rz_satfinite_scale_n1_ue8m0);
1159 PZO_CVT(f16x2_to_e2m1x2_rz_relu_satfinite_scale_n1_ue8m0);
1160 PZO_CVT(bf16x2_to_e2m1x2_rn_satfinite_scale_n1_ue8m0);
1161 PZO_CVT(bf16x2_to_e2m1x2_rn_relu_satfinite_scale_n1_ue8m0);
1162 PZO_CVT(bf16x2_to_e2m1x2_rz_satfinite_scale_n1_ue8m0);
1163 PZO_CVT(bf16x2_to_e2m1x2_rz_relu_satfinite_scale_n1_ue8m0);
1164
1165#undef PZO_CVT
1166
1167 case NVPTX::BI__nvvm_fma_rn_f16:
1168 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fma_rn_f16, BuiltinID, E, CGF&: *this);
1169 case NVPTX::BI__nvvm_fma_rn_f16x2:
1170 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fma_rn_f16x2, BuiltinID, E, CGF&: *this);
1171 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
1172 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fma_rn_ftz_f16, BuiltinID, E, CGF&: *this);
1173 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
1174 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fma_rn_ftz_f16x2, BuiltinID, E, CGF&: *this);
1175 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
1176 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fma_rn_ftz_relu_f16, BuiltinID, E,
1177 CGF&: *this);
1178 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
1179 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fma_rn_ftz_relu_f16x2, BuiltinID, E,
1180 CGF&: *this);
1181 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
1182 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fma_rn_ftz_sat_f16, BuiltinID, E,
1183 CGF&: *this);
1184 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
1185 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fma_rn_ftz_sat_f16x2, BuiltinID, E,
1186 CGF&: *this);
1187 case NVPTX::BI__nvvm_fma_rn_relu_f16:
1188 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fma_rn_relu_f16, BuiltinID, E, CGF&: *this);
1189 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
1190 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fma_rn_relu_f16x2, BuiltinID, E, CGF&: *this);
1191 case NVPTX::BI__nvvm_fma_rn_sat_f16:
1192 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fma_rn_sat_f16, BuiltinID, E, CGF&: *this);
1193 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
1194 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fma_rn_sat_f16x2, BuiltinID, E, CGF&: *this);
1195 case NVPTX::BI__nvvm_fma_rn_oob_f16:
1196 return MakeFMAOOB(IntrinsicID: Intrinsic::nvvm_fma_rn_oob, Ty: Builder.getHalfTy(), E,
1197 CGF&: *this);
1198 case NVPTX::BI__nvvm_fma_rn_oob_f16x2:
1199 return MakeFMAOOB(IntrinsicID: Intrinsic::nvvm_fma_rn_oob,
1200 Ty: llvm::FixedVectorType::get(ElementType: Builder.getHalfTy(), NumElts: 2), E,
1201 CGF&: *this);
1202 case NVPTX::BI__nvvm_fma_rn_oob_bf16:
1203 return MakeFMAOOB(IntrinsicID: Intrinsic::nvvm_fma_rn_oob, Ty: Builder.getBFloatTy(), E,
1204 CGF&: *this);
1205 case NVPTX::BI__nvvm_fma_rn_oob_bf16x2:
1206 return MakeFMAOOB(IntrinsicID: Intrinsic::nvvm_fma_rn_oob,
1207 Ty: llvm::FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 2), E,
1208 CGF&: *this);
1209 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16:
1210 return MakeFMAOOB(IntrinsicID: Intrinsic::nvvm_fma_rn_oob_relu, Ty: Builder.getHalfTy(), E,
1211 CGF&: *this);
1212 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16x2:
1213 return MakeFMAOOB(IntrinsicID: Intrinsic::nvvm_fma_rn_oob_relu,
1214 Ty: llvm::FixedVectorType::get(ElementType: Builder.getHalfTy(), NumElts: 2), E,
1215 CGF&: *this);
1216 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16:
1217 return MakeFMAOOB(IntrinsicID: Intrinsic::nvvm_fma_rn_oob_relu, Ty: Builder.getBFloatTy(), E,
1218 CGF&: *this);
1219 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16x2:
1220 return MakeFMAOOB(IntrinsicID: Intrinsic::nvvm_fma_rn_oob_relu,
1221 Ty: llvm::FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 2), E,
1222 CGF&: *this);
1223 case NVPTX::BI__nvvm_fmax_f16:
1224 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_f16, BuiltinID, E, CGF&: *this);
1225 case NVPTX::BI__nvvm_fmax_f16x2:
1226 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_f16x2, BuiltinID, E, CGF&: *this);
1227 case NVPTX::BI__nvvm_fmax_ftz_f16:
1228 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_ftz_f16, BuiltinID, E, CGF&: *this);
1229 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
1230 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_ftz_f16x2, BuiltinID, E, CGF&: *this);
1231 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
1232 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_ftz_nan_f16, BuiltinID, E, CGF&: *this);
1233 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
1234 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_ftz_nan_f16x2, BuiltinID, E,
1235 CGF&: *this);
1236 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
1237 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16, BuiltinID,
1238 E, CGF&: *this);
1239 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
1240 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16x2,
1241 BuiltinID, E, CGF&: *this);
1242 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
1243 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16, BuiltinID, E,
1244 CGF&: *this);
1245 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
1246 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16x2, BuiltinID,
1247 E, CGF&: *this);
1248 case NVPTX::BI__nvvm_fmax_nan_f16:
1249 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_nan_f16, BuiltinID, E, CGF&: *this);
1250 case NVPTX::BI__nvvm_fmax_nan_f16x2:
1251 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_nan_f16x2, BuiltinID, E, CGF&: *this);
1252 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
1253 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_nan_xorsign_abs_f16, BuiltinID, E,
1254 CGF&: *this);
1255 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
1256 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_nan_xorsign_abs_f16x2, BuiltinID,
1257 E, CGF&: *this);
1258 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
1259 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_xorsign_abs_f16, BuiltinID, E,
1260 CGF&: *this);
1261 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
1262 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_xorsign_abs_f16x2, BuiltinID, E,
1263 CGF&: *this);
1264 case NVPTX::BI__nvvm_fmin_f16:
1265 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_f16, BuiltinID, E, CGF&: *this);
1266 case NVPTX::BI__nvvm_fmin_f16x2:
1267 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_f16x2, BuiltinID, E, CGF&: *this);
1268 case NVPTX::BI__nvvm_fmin_ftz_f16:
1269 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_ftz_f16, BuiltinID, E, CGF&: *this);
1270 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
1271 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_ftz_f16x2, BuiltinID, E, CGF&: *this);
1272 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
1273 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_ftz_nan_f16, BuiltinID, E, CGF&: *this);
1274 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
1275 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_ftz_nan_f16x2, BuiltinID, E,
1276 CGF&: *this);
1277 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
1278 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16, BuiltinID,
1279 E, CGF&: *this);
1280 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
1281 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16x2,
1282 BuiltinID, E, CGF&: *this);
1283 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
1284 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16, BuiltinID, E,
1285 CGF&: *this);
1286 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
1287 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16x2, BuiltinID,
1288 E, CGF&: *this);
1289 case NVPTX::BI__nvvm_fmin_nan_f16:
1290 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_nan_f16, BuiltinID, E, CGF&: *this);
1291 case NVPTX::BI__nvvm_fmin_nan_f16x2:
1292 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_nan_f16x2, BuiltinID, E, CGF&: *this);
1293 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
1294 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_nan_xorsign_abs_f16, BuiltinID, E,
1295 CGF&: *this);
1296 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
1297 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_nan_xorsign_abs_f16x2, BuiltinID,
1298 E, CGF&: *this);
1299 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
1300 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_xorsign_abs_f16, BuiltinID, E,
1301 CGF&: *this);
1302 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
1303 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_xorsign_abs_f16x2, BuiltinID, E,
1304 CGF&: *this);
1305 case NVPTX::BI__nvvm_fabs_f:
1306 case NVPTX::BI__nvvm_abs_bf16:
1307 case NVPTX::BI__nvvm_abs_bf16x2:
1308 case NVPTX::BI__nvvm_fabs_f16:
1309 case NVPTX::BI__nvvm_fabs_f16x2:
1310 return Builder.CreateUnaryIntrinsic(ID: Intrinsic::nvvm_fabs,
1311 Op: EmitScalarExpr(E: E->getArg(Arg: 0)));
1312 case NVPTX::BI__nvvm_fabs_ftz_f:
1313 case NVPTX::BI__nvvm_fabs_ftz_f16:
1314 case NVPTX::BI__nvvm_fabs_ftz_f16x2:
1315 return Builder.CreateUnaryIntrinsic(ID: Intrinsic::nvvm_fabs_ftz,
1316 Op: EmitScalarExpr(E: E->getArg(Arg: 0)));
1317 case NVPTX::BI__nvvm_fabs_d:
1318 return Builder.CreateFAbs(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
1319 case NVPTX::BI__nvvm_ex2_approx_d:
1320 case NVPTX::BI__nvvm_ex2_approx_f:
1321 return Builder.CreateUnaryIntrinsic(ID: Intrinsic::nvvm_ex2_approx,
1322 Op: EmitScalarExpr(E: E->getArg(Arg: 0)));
1323 case NVPTX::BI__nvvm_ex2_approx_ftz_f:
1324 return Builder.CreateUnaryIntrinsic(ID: Intrinsic::nvvm_ex2_approx_ftz,
1325 Op: EmitScalarExpr(E: E->getArg(Arg: 0)));
1326 case NVPTX::BI__nvvm_add_rn_f:
1327 case NVPTX::BI__nvvm_add_rn_d:
1328 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd, RM: APFloat::rmNearestTiesToEven,
1329 BuiltinID, E, CGF&: *this);
1330 case NVPTX::BI__nvvm_add_rz_f:
1331 case NVPTX::BI__nvvm_add_rz_d:
1332 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd, RM: APFloat::rmTowardZero, BuiltinID,
1333 E, CGF&: *this);
1334 case NVPTX::BI__nvvm_add_rm_f:
1335 case NVPTX::BI__nvvm_add_rm_d:
1336 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd, RM: APFloat::rmTowardNegative,
1337 BuiltinID, E, CGF&: *this);
1338 case NVPTX::BI__nvvm_add_rp_f:
1339 case NVPTX::BI__nvvm_add_rp_d:
1340 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd, RM: APFloat::rmTowardPositive,
1341 BuiltinID, E, CGF&: *this);
1342 case NVPTX::BI__nvvm_add_rn_ftz_f:
1343 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_ftz, RM: APFloat::rmNearestTiesToEven,
1344 BuiltinID, E, CGF&: *this);
1345 case NVPTX::BI__nvvm_add_rz_ftz_f:
1346 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_ftz, RM: APFloat::rmTowardZero,
1347 BuiltinID, E, CGF&: *this);
1348 case NVPTX::BI__nvvm_add_rm_ftz_f:
1349 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_ftz, RM: APFloat::rmTowardNegative,
1350 BuiltinID, E, CGF&: *this);
1351 case NVPTX::BI__nvvm_add_rp_ftz_f:
1352 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_ftz, RM: APFloat::rmTowardPositive,
1353 BuiltinID, E, CGF&: *this);
1354 case NVPTX::BI__nvvm_add_rn_sat_f:
1355 case NVPTX::BI__nvvm_add_rn_sat_f16:
1356 case NVPTX::BI__nvvm_add_rn_sat_v2f16:
1357 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_sat, RM: APFloat::rmNearestTiesToEven,
1358 BuiltinID, E, CGF&: *this);
1359 case NVPTX::BI__nvvm_add_rz_sat_f:
1360 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_sat, RM: APFloat::rmTowardZero,
1361 BuiltinID, E, CGF&: *this);
1362 case NVPTX::BI__nvvm_add_rm_sat_f:
1363 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_sat, RM: APFloat::rmTowardNegative,
1364 BuiltinID, E, CGF&: *this);
1365 case NVPTX::BI__nvvm_add_rp_sat_f:
1366 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_sat, RM: APFloat::rmTowardPositive,
1367 BuiltinID, E, CGF&: *this);
1368 case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
1369 case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
1370 case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
1371 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_ftz_sat,
1372 RM: APFloat::rmNearestTiesToEven, BuiltinID, E, CGF&: *this);
1373 case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
1374 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_ftz_sat, RM: APFloat::rmTowardZero,
1375 BuiltinID, E, CGF&: *this);
1376 case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
1377 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_ftz_sat, RM: APFloat::rmTowardNegative,
1378 BuiltinID, E, CGF&: *this);
1379 case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
1380 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_ftz_sat, RM: APFloat::rmTowardPositive,
1381 BuiltinID, E, CGF&: *this);
1382 case NVPTX::BI__nvvm_mul_rn_f:
1383 case NVPTX::BI__nvvm_mul_rn_d:
1384 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul, RM: APFloat::rmNearestTiesToEven,
1385 BuiltinID, E, CGF&: *this);
1386 case NVPTX::BI__nvvm_mul_rz_f:
1387 case NVPTX::BI__nvvm_mul_rz_d:
1388 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul, RM: APFloat::rmTowardZero, BuiltinID,
1389 E, CGF&: *this);
1390 case NVPTX::BI__nvvm_mul_rm_f:
1391 case NVPTX::BI__nvvm_mul_rm_d:
1392 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul, RM: APFloat::rmTowardNegative,
1393 BuiltinID, E, CGF&: *this);
1394 case NVPTX::BI__nvvm_mul_rp_f:
1395 case NVPTX::BI__nvvm_mul_rp_d:
1396 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul, RM: APFloat::rmTowardPositive,
1397 BuiltinID, E, CGF&: *this);
1398 case NVPTX::BI__nvvm_mul_rn_ftz_f:
1399 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul_ftz, RM: APFloat::rmNearestTiesToEven,
1400 BuiltinID, E, CGF&: *this);
1401 case NVPTX::BI__nvvm_mul_rz_ftz_f:
1402 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul_ftz, RM: APFloat::rmTowardZero,
1403 BuiltinID, E, CGF&: *this);
1404 case NVPTX::BI__nvvm_mul_rm_ftz_f:
1405 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul_ftz, RM: APFloat::rmTowardNegative,
1406 BuiltinID, E, CGF&: *this);
1407 case NVPTX::BI__nvvm_mul_rp_ftz_f:
1408 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul_ftz, RM: APFloat::rmTowardPositive,
1409 BuiltinID, E, CGF&: *this);
1410 case NVPTX::BI__nvvm_mul_rn_sat_f16:
1411 case NVPTX::BI__nvvm_mul_rn_sat_v2f16:
1412 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul_sat, RM: APFloat::rmNearestTiesToEven,
1413 BuiltinID, E, CGF&: *this);
1414 case NVPTX::BI__nvvm_mul_rn_ftz_sat_f16:
1415 case NVPTX::BI__nvvm_mul_rn_ftz_sat_v2f16:
1416 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul_ftz_sat,
1417 RM: APFloat::rmNearestTiesToEven, BuiltinID, E, CGF&: *this);
1418 case NVPTX::BI__nvvm_ldg_h:
1419 case NVPTX::BI__nvvm_ldg_h2:
1420 return MakeLdg(CGF&: *this, E);
1421 case NVPTX::BI__nvvm_ldu_h:
1422 case NVPTX::BI__nvvm_ldu_h2:
1423 return MakeLdu(IntrinsicID: Intrinsic::nvvm_ldu_global_f, CGF&: *this, E);
1424 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
1425 return MakeCpAsync(IntrinsicID: Intrinsic::nvvm_cp_async_ca_shared_global_4,
1426 IntrinsicIDS: Intrinsic::nvvm_cp_async_ca_shared_global_4_s, CGF&: *this, E,
1427 SrcSize: 4);
1428 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
1429 return MakeCpAsync(IntrinsicID: Intrinsic::nvvm_cp_async_ca_shared_global_8,
1430 IntrinsicIDS: Intrinsic::nvvm_cp_async_ca_shared_global_8_s, CGF&: *this, E,
1431 SrcSize: 8);
1432 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
1433 return MakeCpAsync(IntrinsicID: Intrinsic::nvvm_cp_async_ca_shared_global_16,
1434 IntrinsicIDS: Intrinsic::nvvm_cp_async_ca_shared_global_16_s, CGF&: *this, E,
1435 SrcSize: 16);
1436 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
1437 return MakeCpAsync(IntrinsicID: Intrinsic::nvvm_cp_async_cg_shared_global_16,
1438 IntrinsicIDS: Intrinsic::nvvm_cp_async_cg_shared_global_16_s, CGF&: *this, E,
1439 SrcSize: 16);
1440 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
1441 return Builder.CreateCall(
1442 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_clusterid_x));
1443 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
1444 return Builder.CreateCall(
1445 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_clusterid_y));
1446 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
1447 return Builder.CreateCall(
1448 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_clusterid_z));
1449 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
1450 return Builder.CreateCall(
1451 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_clusterid_w));
1452 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
1453 return Builder.CreateCall(
1454 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_nclusterid_x));
1455 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
1456 return Builder.CreateCall(
1457 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_nclusterid_y));
1458 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
1459 return Builder.CreateCall(
1460 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_nclusterid_z));
1461 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
1462 return Builder.CreateCall(
1463 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_nclusterid_w));
1464 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
1465 return Builder.CreateCall(
1466 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_x));
1467 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
1468 return Builder.CreateCall(
1469 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_y));
1470 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
1471 return Builder.CreateCall(
1472 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_z));
1473 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
1474 return Builder.CreateCall(
1475 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_w));
1476 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
1477 return Builder.CreateCall(
1478 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_x));
1479 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
1480 return Builder.CreateCall(
1481 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_y));
1482 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
1483 return Builder.CreateCall(
1484 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_z));
1485 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
1486 return Builder.CreateCall(
1487 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_w));
1488 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
1489 return Builder.CreateCall(
1490 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_ctarank));
1491 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
1492 return Builder.CreateCall(
1493 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_nctarank));
1494 case NVPTX::BI__nvvm_is_explicit_cluster:
1495 return Builder.CreateCall(
1496 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_is_explicit_cluster));
1497 case NVPTX::BI__nvvm_isspacep_shared_cluster:
1498 return Builder.CreateCall(
1499 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_isspacep_shared_cluster),
1500 Args: EmitScalarExpr(E: E->getArg(Arg: 0)));
1501 case NVPTX::BI__nvvm_mapa:
1502 return Builder.CreateCall(
1503 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_mapa),
1504 Args: {EmitScalarExpr(E: E->getArg(Arg: 0)), EmitScalarExpr(E: E->getArg(Arg: 1))});
1505 case NVPTX::BI__nvvm_mapa_shared_cluster:
1506 return Builder.CreateCall(
1507 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_mapa_shared_cluster),
1508 Args: {EmitScalarExpr(E: E->getArg(Arg: 0)), EmitScalarExpr(E: E->getArg(Arg: 1))});
1509 case NVPTX::BI__nvvm_getctarank:
1510 return Builder.CreateCall(
1511 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_getctarank),
1512 Args: EmitScalarExpr(E: E->getArg(Arg: 0)));
1513 case NVPTX::BI__nvvm_getctarank_shared_cluster:
1514 return Builder.CreateCall(
1515 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_getctarank_shared_cluster),
1516 Args: EmitScalarExpr(E: E->getArg(Arg: 0)));
1517 case NVPTX::BI__nvvm_barrier_cluster_arrive:
1518 return Builder.CreateCall(
1519 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_barrier_cluster_arrive));
1520 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
1521 return Builder.CreateCall(
1522 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_barrier_cluster_arrive_relaxed));
1523 case NVPTX::BI__nvvm_barrier_cluster_wait:
1524 return Builder.CreateCall(
1525 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_barrier_cluster_wait));
1526 case NVPTX::BI__nvvm_fence_sc_cluster:
1527 return Builder.CreateCall(
1528 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_fence_sc_cluster));
1529 case NVPTX::BI__nvvm_bar_sync:
1530 return Builder.CreateCall(
1531 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_barrier_cta_sync_aligned_all),
1532 Args: EmitScalarExpr(E: E->getArg(Arg: 0)));
1533 case NVPTX::BI__syncthreads:
1534 return Builder.CreateCall(
1535 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_barrier_cta_sync_aligned_all),
1536 Args: Builder.getInt32(C: 0));
1537 case NVPTX::BI__nvvm_barrier_sync:
1538 return Builder.CreateCall(
1539 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_barrier_cta_sync_all),
1540 Args: EmitScalarExpr(E: E->getArg(Arg: 0)));
1541 case NVPTX::BI__nvvm_barrier_sync_cnt:
1542 return Builder.CreateCall(
1543 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_barrier_cta_sync_count),
1544 Args: {EmitScalarExpr(E: E->getArg(Arg: 0)), EmitScalarExpr(E: E->getArg(Arg: 1))});
1545 case NVPTX::BI__nvvm_bar0_and:
1546 return Builder.CreateZExt(
1547 V: Builder.CreateIntrinsic(
1548 ID: Intrinsic::nvvm_barrier_cta_red_and_aligned_all, OverloadTypes: {},
1549 Args: {Builder.getInt32(C: 0),
1550 Builder.CreateICmpNE(LHS: EmitScalarExpr(E: E->getArg(Arg: 0)),
1551 RHS: Builder.getInt32(C: 0))}),
1552 DestTy: Builder.getInt32Ty());
1553 case NVPTX::BI__nvvm_bar0_or:
1554 return Builder.CreateZExt(
1555 V: Builder.CreateIntrinsic(
1556 ID: Intrinsic::nvvm_barrier_cta_red_or_aligned_all, OverloadTypes: {},
1557 Args: {Builder.getInt32(C: 0),
1558 Builder.CreateICmpNE(LHS: EmitScalarExpr(E: E->getArg(Arg: 0)),
1559 RHS: Builder.getInt32(C: 0))}),
1560 DestTy: Builder.getInt32Ty());
1561 case NVPTX::BI__nvvm_bar0_popc:
1562 return Builder.CreateIntrinsic(
1563 ID: Intrinsic::nvvm_barrier_cta_red_popc_aligned_all, OverloadTypes: {},
1564 Args: {Builder.getInt32(C: 0), Builder.CreateICmpNE(LHS: EmitScalarExpr(E: E->getArg(Arg: 0)),
1565 RHS: Builder.getInt32(C: 0))});
1566 case NVPTX::BI__nvvm_mbarrier_init:
1567 case NVPTX::BI__nvvm_mbarrier_init_shared: {
1568 // The intrinsic is overloaded on the pointer, so the two builtins differ
1569 // only in the address space of their first argument.
1570 Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 0));
1571 return Builder.CreateIntrinsic(
1572 ID: Intrinsic::nvvm_mbarrier_init, OverloadTypes: {Ptr->getType()},
1573 Args: {Ptr, EmitScalarExpr(E: E->getArg(Arg: 1)), Builder.getInt32(C: 0)});
1574 }
1575 default:
1576 return nullptr;
1577 }
1578}
1579