1//===-------- NVPTX.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "TargetInfo.h"
15#include "clang/Basic/TargetBuiltins.h"
16#include "llvm/IR/IntrinsicsNVPTX.h"
17#include "llvm/TargetParser/AtomicScope.h"
18
19using namespace clang;
20using namespace CodeGen;
21using namespace llvm;
22
23namespace {
24// Helper classes for mapping MMA builtins to particular LLVM intrinsic variant.
25struct NVPTXMmaLdstInfo {
26 unsigned NumResults; // Number of elements to load/store
27 // Intrinsic IDs for row/col variants. 0 if particular layout is unsupported.
28 unsigned IID_col;
29 unsigned IID_row;
30};
31
32#define MMA_INTR(geom_op_type, layout) \
33 Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
34#define MMA_LDST(n, geom_op_type) \
35 { n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row) }
36
37static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(unsigned BuiltinID) {
38 switch (BuiltinID) {
39 // FP MMA loads
40 case NVPTX::BI__hmma_m16n16k16_ld_a:
41 return MMA_LDST(8, m16n16k16_load_a_f16);
42 case NVPTX::BI__hmma_m16n16k16_ld_b:
43 return MMA_LDST(8, m16n16k16_load_b_f16);
44 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
45 return MMA_LDST(4, m16n16k16_load_c_f16);
46 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
47 return MMA_LDST(8, m16n16k16_load_c_f32);
48 case NVPTX::BI__hmma_m32n8k16_ld_a:
49 return MMA_LDST(8, m32n8k16_load_a_f16);
50 case NVPTX::BI__hmma_m32n8k16_ld_b:
51 return MMA_LDST(8, m32n8k16_load_b_f16);
52 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
53 return MMA_LDST(4, m32n8k16_load_c_f16);
54 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
55 return MMA_LDST(8, m32n8k16_load_c_f32);
56 case NVPTX::BI__hmma_m8n32k16_ld_a:
57 return MMA_LDST(8, m8n32k16_load_a_f16);
58 case NVPTX::BI__hmma_m8n32k16_ld_b:
59 return MMA_LDST(8, m8n32k16_load_b_f16);
60 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
61 return MMA_LDST(4, m8n32k16_load_c_f16);
62 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
63 return MMA_LDST(8, m8n32k16_load_c_f32);
64
65 // Integer MMA loads
66 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
67 return MMA_LDST(2, m16n16k16_load_a_s8);
68 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
69 return MMA_LDST(2, m16n16k16_load_a_u8);
70 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
71 return MMA_LDST(2, m16n16k16_load_b_s8);
72 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
73 return MMA_LDST(2, m16n16k16_load_b_u8);
74 case NVPTX::BI__imma_m16n16k16_ld_c:
75 return MMA_LDST(8, m16n16k16_load_c_s32);
76 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
77 return MMA_LDST(4, m32n8k16_load_a_s8);
78 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
79 return MMA_LDST(4, m32n8k16_load_a_u8);
80 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
81 return MMA_LDST(1, m32n8k16_load_b_s8);
82 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
83 return MMA_LDST(1, m32n8k16_load_b_u8);
84 case NVPTX::BI__imma_m32n8k16_ld_c:
85 return MMA_LDST(8, m32n8k16_load_c_s32);
86 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
87 return MMA_LDST(1, m8n32k16_load_a_s8);
88 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
89 return MMA_LDST(1, m8n32k16_load_a_u8);
90 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
91 return MMA_LDST(4, m8n32k16_load_b_s8);
92 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
93 return MMA_LDST(4, m8n32k16_load_b_u8);
94 case NVPTX::BI__imma_m8n32k16_ld_c:
95 return MMA_LDST(8, m8n32k16_load_c_s32);
96
97 // Sub-integer MMA loads.
98 // Only row/col layout is supported by A/B fragments.
99 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
100 return {.NumResults: 1, .IID_col: 0, MMA_INTR(m8n8k32_load_a_s4, row)};
101 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
102 return {.NumResults: 1, .IID_col: 0, MMA_INTR(m8n8k32_load_a_u4, row)};
103 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
104 return {.NumResults: 1, MMA_INTR(m8n8k32_load_b_s4, col), .IID_row: 0};
105 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
106 return {.NumResults: 1, MMA_INTR(m8n8k32_load_b_u4, col), .IID_row: 0};
107 case NVPTX::BI__imma_m8n8k32_ld_c:
108 return MMA_LDST(2, m8n8k32_load_c_s32);
109 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
110 return {.NumResults: 1, .IID_col: 0, MMA_INTR(m8n8k128_load_a_b1, row)};
111 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
112 return {.NumResults: 1, MMA_INTR(m8n8k128_load_b_b1, col), .IID_row: 0};
113 case NVPTX::BI__bmma_m8n8k128_ld_c:
114 return MMA_LDST(2, m8n8k128_load_c_s32);
115
116 // Double MMA loads
117 case NVPTX::BI__dmma_m8n8k4_ld_a:
118 return MMA_LDST(1, m8n8k4_load_a_f64);
119 case NVPTX::BI__dmma_m8n8k4_ld_b:
120 return MMA_LDST(1, m8n8k4_load_b_f64);
121 case NVPTX::BI__dmma_m8n8k4_ld_c:
122 return MMA_LDST(2, m8n8k4_load_c_f64);
123
124 // Alternate float MMA loads
125 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
126 return MMA_LDST(4, m16n16k16_load_a_bf16);
127 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
128 return MMA_LDST(4, m16n16k16_load_b_bf16);
129 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
130 return MMA_LDST(2, m8n32k16_load_a_bf16);
131 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
132 return MMA_LDST(8, m8n32k16_load_b_bf16);
133 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
134 return MMA_LDST(8, m32n8k16_load_a_bf16);
135 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
136 return MMA_LDST(2, m32n8k16_load_b_bf16);
137 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
138 return MMA_LDST(4, m16n16k8_load_a_tf32);
139 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
140 return MMA_LDST(4, m16n16k8_load_b_tf32);
141 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
142 return MMA_LDST(8, m16n16k8_load_c_f32);
143
144 // NOTE: We need to follow inconsitent naming scheme used by NVCC. Unlike
145 // PTX and LLVM IR where stores always use fragment D, NVCC builtins always
146 // use fragment C for both loads and stores.
147 // FP MMA stores.
148 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
149 return MMA_LDST(4, m16n16k16_store_d_f16);
150 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
151 return MMA_LDST(8, m16n16k16_store_d_f32);
152 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
153 return MMA_LDST(4, m32n8k16_store_d_f16);
154 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
155 return MMA_LDST(8, m32n8k16_store_d_f32);
156 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
157 return MMA_LDST(4, m8n32k16_store_d_f16);
158 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
159 return MMA_LDST(8, m8n32k16_store_d_f32);
160
161 // Integer and sub-integer MMA stores.
162 // Another naming quirk. Unlike other MMA builtins that use PTX types in the
163 // name, integer loads/stores use LLVM's i32.
164 case NVPTX::BI__imma_m16n16k16_st_c_i32:
165 return MMA_LDST(8, m16n16k16_store_d_s32);
166 case NVPTX::BI__imma_m32n8k16_st_c_i32:
167 return MMA_LDST(8, m32n8k16_store_d_s32);
168 case NVPTX::BI__imma_m8n32k16_st_c_i32:
169 return MMA_LDST(8, m8n32k16_store_d_s32);
170 case NVPTX::BI__imma_m8n8k32_st_c_i32:
171 return MMA_LDST(2, m8n8k32_store_d_s32);
172 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
173 return MMA_LDST(2, m8n8k128_store_d_s32);
174
175 // Double MMA store
176 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
177 return MMA_LDST(2, m8n8k4_store_d_f64);
178
179 // Alternate float MMA store
180 case NVPTX::BI__mma_m16n16k8_st_c_f32:
181 return MMA_LDST(8, m16n16k8_store_d_f32);
182
183 default:
184 llvm_unreachable("Unknown MMA builtin");
185 }
186}
187#undef MMA_LDST
188#undef MMA_INTR
189
190
191struct NVPTXMmaInfo {
192 unsigned NumEltsA;
193 unsigned NumEltsB;
194 unsigned NumEltsC;
195 unsigned NumEltsD;
196
197 // Variants are ordered by layout-A/layout-B/satf, where 'row' has priority
198 // over 'col' for layout. The index of non-satf variants is expected to match
199 // the undocumented layout constants used by CUDA's mma.hpp.
200 std::array<unsigned, 8> Variants;
201
202 unsigned getMMAIntrinsic(int Layout, bool Satf) {
203 unsigned Index = Layout + 4 * Satf;
204 if (Index >= Variants.size())
205 return 0;
206 return Variants[Index];
207 }
208};
209
210 // Returns an intrinsic that matches Layout and Satf for valid combinations of
211 // Layout and Satf, 0 otherwise.
212static NVPTXMmaInfo getNVPTXMmaInfo(unsigned BuiltinID) {
213 // clang-format off
214#define MMA_VARIANTS(geom, type) \
215 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type, \
216 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
217 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type, \
218 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type
219#define MMA_SATF_VARIANTS(geom, type) \
220 MMA_VARIANTS(geom, type), \
221 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \
222 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
223 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \
224 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite
225// Sub-integer MMA only supports row.col layout.
226#define MMA_VARIANTS_I4(geom, type) \
227 0, \
228 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
229 0, \
230 0, \
231 0, \
232 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
233 0, \
234 0
235// b1 MMA does not support .satfinite.
236#define MMA_VARIANTS_B1_XOR(geom, type) \
237 0, \
238 Intrinsic::nvvm_wmma_##geom##_mma_xor_popc_row_col_##type, \
239 0, \
240 0, \
241 0, \
242 0, \
243 0, \
244 0
245#define MMA_VARIANTS_B1_AND(geom, type) \
246 0, \
247 Intrinsic::nvvm_wmma_##geom##_mma_and_popc_row_col_##type, \
248 0, \
249 0, \
250 0, \
251 0, \
252 0, \
253 0
254 // clang-format on
255 switch (BuiltinID) {
256 // FP MMA
257 // Note that 'type' argument of MMA_SATF_VARIANTS uses D_C notation, while
258 // NumEltsN of return value are ordered as A,B,C,D.
259 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
260 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 4, .NumEltsD: 4, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m16n16k16, f16_f16)}}};
261 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
262 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 4, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m16n16k16, f32_f16)}}};
263 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
264 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 8, .NumEltsD: 4, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m16n16k16, f16_f32)}}};
265 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
266 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m16n16k16, f32_f32)}}};
267 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
268 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 4, .NumEltsD: 4, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m32n8k16, f16_f16)}}};
269 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
270 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 4, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m32n8k16, f32_f16)}}};
271 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
272 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 8, .NumEltsD: 4, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m32n8k16, f16_f32)}}};
273 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
274 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m32n8k16, f32_f32)}}};
275 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
276 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 4, .NumEltsD: 4, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m8n32k16, f16_f16)}}};
277 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
278 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 4, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m8n32k16, f32_f16)}}};
279 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
280 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 8, .NumEltsD: 4, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m8n32k16, f16_f32)}}};
281 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
282 return {.NumEltsA: 8, .NumEltsB: 8, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m8n32k16, f32_f32)}}};
283
284 // Integer MMA
285 case NVPTX::BI__imma_m16n16k16_mma_s8:
286 return {.NumEltsA: 2, .NumEltsB: 2, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m16n16k16, s8)}}};
287 case NVPTX::BI__imma_m16n16k16_mma_u8:
288 return {.NumEltsA: 2, .NumEltsB: 2, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m16n16k16, u8)}}};
289 case NVPTX::BI__imma_m32n8k16_mma_s8:
290 return {.NumEltsA: 4, .NumEltsB: 1, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m32n8k16, s8)}}};
291 case NVPTX::BI__imma_m32n8k16_mma_u8:
292 return {.NumEltsA: 4, .NumEltsB: 1, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m32n8k16, u8)}}};
293 case NVPTX::BI__imma_m8n32k16_mma_s8:
294 return {.NumEltsA: 1, .NumEltsB: 4, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m8n32k16, s8)}}};
295 case NVPTX::BI__imma_m8n32k16_mma_u8:
296 return {.NumEltsA: 1, .NumEltsB: 4, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_SATF_VARIANTS(m8n32k16, u8)}}};
297
298 // Sub-integer MMA
299 case NVPTX::BI__imma_m8n8k32_mma_s4:
300 return {.NumEltsA: 1, .NumEltsB: 1, .NumEltsC: 2, .NumEltsD: 2, .Variants: {._M_elems: {MMA_VARIANTS_I4(m8n8k32, s4)}}};
301 case NVPTX::BI__imma_m8n8k32_mma_u4:
302 return {.NumEltsA: 1, .NumEltsB: 1, .NumEltsC: 2, .NumEltsD: 2, .Variants: {._M_elems: {MMA_VARIANTS_I4(m8n8k32, u4)}}};
303 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
304 return {.NumEltsA: 1, .NumEltsB: 1, .NumEltsC: 2, .NumEltsD: 2, .Variants: {._M_elems: {MMA_VARIANTS_B1_XOR(m8n8k128, b1)}}};
305 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
306 return {.NumEltsA: 1, .NumEltsB: 1, .NumEltsC: 2, .NumEltsD: 2, .Variants: {._M_elems: {MMA_VARIANTS_B1_AND(m8n8k128, b1)}}};
307
308 // Double MMA
309 case NVPTX::BI__dmma_m8n8k4_mma_f64:
310 return {.NumEltsA: 1, .NumEltsB: 1, .NumEltsC: 2, .NumEltsD: 2, .Variants: {._M_elems: {MMA_VARIANTS(m8n8k4, f64)}}};
311
312 // Alternate FP MMA
313 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
314 return {.NumEltsA: 4, .NumEltsB: 4, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_VARIANTS(m16n16k16, bf16)}}};
315 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
316 return {.NumEltsA: 2, .NumEltsB: 8, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_VARIANTS(m8n32k16, bf16)}}};
317 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
318 return {.NumEltsA: 8, .NumEltsB: 2, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_VARIANTS(m32n8k16, bf16)}}};
319 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
320 return {.NumEltsA: 4, .NumEltsB: 4, .NumEltsC: 8, .NumEltsD: 8, .Variants: {._M_elems: {MMA_VARIANTS(m16n16k8, tf32)}}};
321 default:
322 llvm_unreachable("Unexpected builtin ID.");
323 }
324#undef MMA_VARIANTS
325#undef MMA_SATF_VARIANTS
326#undef MMA_VARIANTS_I4
327#undef MMA_VARIANTS_B1_AND
328#undef MMA_VARIANTS_B1_XOR
329}
330
331static Value *MakeLdu(unsigned IntrinsicID, CodeGenFunction &CGF,
332 const CallExpr *E) {
333 Value *Ptr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
334 QualType ArgType = E->getArg(Arg: 0)->getType();
335 clang::CharUnits Align = CGF.CGM.getNaturalPointeeTypeAlignment(T: ArgType);
336 llvm::Type *ElemTy = CGF.ConvertTypeForMem(T: ArgType->getPointeeType());
337 return CGF.Builder.CreateCall(
338 Callee: CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {ElemTy, Ptr->getType()}),
339 Args: {Ptr, ConstantInt::get(Ty: CGF.Builder.getInt32Ty(), V: Align.getQuantity())});
340}
341
342static Value *MakeLdg(CodeGenFunction &CGF, const CallExpr *E) {
343 Value *Ptr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
344 QualType ArgType = E->getArg(Arg: 0)->getType();
345 clang::CharUnits AlignV = CGF.CGM.getNaturalPointeeTypeAlignment(T: ArgType);
346 llvm::Type *ElemTy = CGF.ConvertTypeForMem(T: ArgType->getPointeeType());
347
348 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
349 auto *ASC = CGF.Builder.CreateAddrSpaceCast(V: Ptr, DestTy: CGF.Builder.getPtrTy(AddrSpace: 1));
350 auto *LD = CGF.Builder.CreateAlignedLoad(Ty: ElemTy, Ptr: ASC, Align: AlignV.getAsAlign());
351 MDNode *MD = MDNode::get(Context&: CGF.Builder.getContext(), MDs: {});
352 LD->setMetadata(KindID: LLVMContext::MD_invariant_load, Node: MD);
353
354 return LD;
355}
356
357// `Scope` is AtomicScope::Workgroup for _cta builtins and AtomicScope::System
358// for _sys builtins.
359static Value *MakeScopedAtomicRMW(CodeGenFunction &CGF, const CallExpr *E,
360 llvm::AtomicRMWInst::BinOp Kind,
361 llvm::AtomicScope Scope) {
362 Address Ptr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
363 Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
364 llvm::SyncScope::ID SSID = CGF.getLLVMContext().getOrInsertSyncScopeID(
365 SSN: *llvm::getAtomicScopeIRString(T: CGF.getTarget().getTriple(), S: Scope));
366 llvm::AtomicRMWInst *RMW = CGF.Builder.CreateAtomicRMW(
367 Op: Kind, Addr: Ptr, Val, Ordering: llvm::AtomicOrdering::Monotonic, SSID);
368
369 // CUDA's atomicAdd_block()/atomicAdd_system() reach this through the scoped
370 // add builtins, and are defined in terms of the native atom.add just as the
371 // unscoped atomicAdd() is. The helper ignores everything that is not a
372 // floating-point add, so the other scoped operations are unaffected.
373 addAtomicIgnoreDenormalModeMetadata(CGF, AtomicInst&: *RMW, /*AllowHalf=*/true);
374 return RMW;
375}
376
377// `Scope` is AtomicScope::Workgroup for _cta builtins and AtomicScope::System
378// for _sys builtins.
379static Value *MakeScopedAtomicCAS(CodeGenFunction &CGF, const CallExpr *E,
380 llvm::AtomicScope Scope) {
381 Address Ptr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
382 Value *Cmp = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
383 Value *New = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
384 llvm::SyncScope::ID SSID = CGF.getLLVMContext().getOrInsertSyncScopeID(
385 SSN: *llvm::getAtomicScopeIRString(T: CGF.getTarget().getTriple(), S: Scope));
386 Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
387 Addr: Ptr, Cmp, New, SuccessOrdering: llvm::AtomicOrdering::Monotonic,
388 FailureOrdering: llvm::AtomicOrdering::Monotonic, SSID);
389 return CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 0);
390}
391
392static Value *MakeCpAsync(unsigned IntrinsicID, unsigned IntrinsicIDS,
393 CodeGenFunction &CGF, const CallExpr *E,
394 int SrcSize) {
395 return E->getNumArgs() == 3
396 ? CGF.Builder.CreateCall(Callee: CGF.CGM.getIntrinsic(IID: IntrinsicIDS),
397 Args: {CGF.EmitScalarExpr(E: E->getArg(Arg: 0)),
398 CGF.EmitScalarExpr(E: E->getArg(Arg: 1)),
399 CGF.EmitScalarExpr(E: E->getArg(Arg: 2))})
400 : CGF.Builder.CreateCall(Callee: CGF.CGM.getIntrinsic(IID: IntrinsicID),
401 Args: {CGF.EmitScalarExpr(E: E->getArg(Arg: 0)),
402 CGF.EmitScalarExpr(E: E->getArg(Arg: 1))});
403}
404
405static Value *MakeHalfType(Function *Intrinsic, unsigned BuiltinID,
406 const CallExpr *E, CodeGenFunction &CGF,
407 ArrayRef<Value *> TrailingArgs = {}) {
408 SmallVector<Value *, 16> Args;
409 auto *FTy = Intrinsic->getFunctionType();
410 unsigned ICEArguments = 0;
411 ASTContext::GetBuiltinTypeError Error;
412 CGF.CGM.getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
413 assert(Error == ASTContext::GE_None && "Should not codegen an error");
414 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
415 assert((ICEArguments & (1 << i)) == 0);
416 auto *ArgValue = CGF.EmitScalarExpr(E: E->getArg(Arg: i));
417 auto *PTy = FTy->getParamType(i);
418 if (PTy != ArgValue->getType())
419 ArgValue = CGF.Builder.CreateBitCast(V: ArgValue, DestTy: PTy);
420 Args.push_back(Elt: ArgValue);
421 }
422
423 llvm::append_range(C&: Args, R&: TrailingArgs);
424 appendDefaultIntrinsicArgs(Args, F: Intrinsic);
425
426 return CGF.Builder.CreateCall(Callee: Intrinsic, Args);
427}
428
429static Value *MakeHalfType(unsigned IntrinsicID, unsigned BuiltinID,
430 const CallExpr *E, CodeGenFunction &CGF,
431 ArrayRef<Value *> TrailingArgs = {}) {
432 return MakeHalfType(Intrinsic: CGF.CGM.getIntrinsic(IID: IntrinsicID), BuiltinID, E, CGF,
433 TrailingArgs);
434}
435
436static Value *MakeFMAOOB(unsigned IntrinsicID, llvm::Type *Ty,
437 const CallExpr *E, CodeGenFunction &CGF) {
438 return CGF.Builder.CreateCall(Callee: CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Ty}),
439 Args: {CGF.EmitScalarExpr(E: E->getArg(Arg: 0)),
440 CGF.EmitScalarExpr(E: E->getArg(Arg: 1)),
441 CGF.EmitScalarExpr(E: E->getArg(Arg: 2))});
442}
443
444static Value *MakeFPArith(unsigned IntrinsicID, APFloat::roundingMode RM,
445 unsigned BuiltinID, const CallExpr *E,
446 CodeGenFunction &CGF) {
447 llvm::Type *Ty = CGF.ConvertType(T: E->getType());
448 return MakeHalfType(Intrinsic: CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Ty), BuiltinID, E, CGF,
449 TrailingArgs: {CGF.Builder.getInt32(C: static_cast<int>(RM))});
450}
451
452static Value *MakeIdp2a(unsigned IntrinsicID, bool IsHi, const CallExpr *E,
453 CodeGenFunction &CGF) {
454 return CGF.Builder.CreateCall(
455 Callee: CGF.CGM.getIntrinsic(IID: IntrinsicID),
456 Args: {CGF.EmitScalarExpr(E: E->getArg(Arg: 0)), CGF.EmitScalarExpr(E: E->getArg(Arg: 1)),
457 CGF.Builder.getInt1(V: IsHi), CGF.EmitScalarExpr(E: E->getArg(Arg: 2))});
458}
459
460} // namespace
461
462static llvm::SyncScope::ID getDeviceSyncScope(CodeGenFunction &CGF) {
463 return CGF.getLLVMContext().getOrInsertSyncScopeID(
464 SSN: *llvm::getAtomicScopeIRString(T: CGF.getTarget().getTriple(),
465 S: llvm::AtomicScope::Device));
466}
467
468Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
469 const CallExpr *E) {
470 switch (BuiltinID) {
471 case NVPTX::BI__nvvm_mulhi_i:
472 case NVPTX::BI__nvvm_mulhi_ui:
473 case NVPTX::BI__nvvm_mulhi_ll:
474 case NVPTX::BI__nvvm_mulhi_ull:
475 return Builder.CreateBinaryIntrinsic(
476 ID: E->getType()->hasSignedIntegerRepresentation() ? Intrinsic::smulh
477 : Intrinsic::umulh,
478 LHS: EmitScalarExpr(E: E->getArg(Arg: 0)), RHS: EmitScalarExpr(E: E->getArg(Arg: 1)));
479
480 // Unscoped __nvvm_atom_*_gen_* builtins (CUDA's atomicAdd() etc.) have device
481 // scope, but LLVM's default sync scope is system, so request device
482 // explicitly.
483 case NVPTX::BI__nvvm_atom_add_gen_i:
484 case NVPTX::BI__nvvm_atom_add_gen_l:
485 case NVPTX::BI__nvvm_atom_add_gen_ll:
486 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::Add, E,
487 Ordering: AtomicOrdering::Monotonic,
488 SSID: getDeviceSyncScope(CGF&: *this));
489
490 case NVPTX::BI__nvvm_atom_sub_gen_i:
491 case NVPTX::BI__nvvm_atom_sub_gen_l:
492 case NVPTX::BI__nvvm_atom_sub_gen_ll:
493 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::Sub, E,
494 Ordering: AtomicOrdering::Monotonic,
495 SSID: getDeviceSyncScope(CGF&: *this));
496
497 case NVPTX::BI__nvvm_atom_and_gen_i:
498 case NVPTX::BI__nvvm_atom_and_gen_l:
499 case NVPTX::BI__nvvm_atom_and_gen_ll:
500 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::And, E,
501 Ordering: AtomicOrdering::Monotonic,
502 SSID: getDeviceSyncScope(CGF&: *this));
503
504 case NVPTX::BI__nvvm_atom_or_gen_i:
505 case NVPTX::BI__nvvm_atom_or_gen_l:
506 case NVPTX::BI__nvvm_atom_or_gen_ll:
507 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::Or, E,
508 Ordering: AtomicOrdering::Monotonic,
509 SSID: getDeviceSyncScope(CGF&: *this));
510
511 case NVPTX::BI__nvvm_atom_xor_gen_i:
512 case NVPTX::BI__nvvm_atom_xor_gen_l:
513 case NVPTX::BI__nvvm_atom_xor_gen_ll:
514 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::Xor, E,
515 Ordering: AtomicOrdering::Monotonic,
516 SSID: getDeviceSyncScope(CGF&: *this));
517
518 case NVPTX::BI__nvvm_atom_xchg_gen_i:
519 case NVPTX::BI__nvvm_atom_xchg_gen_l:
520 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
521 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::Xchg, E,
522 Ordering: AtomicOrdering::Monotonic,
523 SSID: getDeviceSyncScope(CGF&: *this));
524
525 case NVPTX::BI__nvvm_atom_max_gen_i:
526 case NVPTX::BI__nvvm_atom_max_gen_l:
527 case NVPTX::BI__nvvm_atom_max_gen_ll:
528 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::Max, E,
529 Ordering: AtomicOrdering::Monotonic,
530 SSID: getDeviceSyncScope(CGF&: *this));
531
532 case NVPTX::BI__nvvm_atom_max_gen_ui:
533 case NVPTX::BI__nvvm_atom_max_gen_ul:
534 case NVPTX::BI__nvvm_atom_max_gen_ull:
535 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::UMax, E,
536 Ordering: AtomicOrdering::Monotonic,
537 SSID: getDeviceSyncScope(CGF&: *this));
538
539 case NVPTX::BI__nvvm_atom_min_gen_i:
540 case NVPTX::BI__nvvm_atom_min_gen_l:
541 case NVPTX::BI__nvvm_atom_min_gen_ll:
542 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::Min, E,
543 Ordering: AtomicOrdering::Monotonic,
544 SSID: getDeviceSyncScope(CGF&: *this));
545
546 case NVPTX::BI__nvvm_atom_min_gen_ui:
547 case NVPTX::BI__nvvm_atom_min_gen_ul:
548 case NVPTX::BI__nvvm_atom_min_gen_ull:
549 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::UMin, E,
550 Ordering: AtomicOrdering::Monotonic,
551 SSID: getDeviceSyncScope(CGF&: *this));
552
553 case NVPTX::BI__nvvm_atom_cas_gen_us:
554 case NVPTX::BI__nvvm_atom_cas_gen_i:
555 case NVPTX::BI__nvvm_atom_cas_gen_l:
556 case NVPTX::BI__nvvm_atom_cas_gen_ll:
557 // __nvvm_atom_cas_gen_* should return the old value rather than the
558 // success flag.
559 return MakeAtomicCmpXchgValue(
560 CGF&: *this, E, /*ReturnBool=*/false, SuccessOrdering: AtomicOrdering::Monotonic,
561 FailureOrdering: AtomicOrdering::Monotonic, SSID: getDeviceSyncScope(CGF&: *this));
562
563 case NVPTX::BI__nvvm_atom_add_gen_f:
564 case NVPTX::BI__nvvm_atom_add_gen_d: {
565 Address DestAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
566 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 1));
567
568 llvm::AtomicRMWInst *RMW = Builder.CreateAtomicRMW(
569 Op: llvm::AtomicRMWInst::FAdd, Addr: DestAddr, Val, Ordering: AtomicOrdering::Monotonic,
570 SSID: getDeviceSyncScope(CGF&: *this));
571
572 // CUDA's atomicAdd() is defined in terms of the native atom.add
573 // instruction, whose denormal behavior is fixed by the hardware and need
574 // not agree with the function's denormal mode. Without this the atomic is
575 // expanded into a CAS loop whenever the two disagree.
576 addAtomicIgnoreDenormalModeMetadata(CGF&: *this, AtomicInst&: *RMW, /*AllowHalf=*/true);
577 return RMW;
578 }
579
580 case NVPTX::BI__nvvm_atom_inc_gen_ui:
581 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::UIncWrap, E,
582 Ordering: AtomicOrdering::Monotonic,
583 SSID: getDeviceSyncScope(CGF&: *this));
584
585 case NVPTX::BI__nvvm_atom_dec_gen_ui:
586 return MakeBinaryAtomicValue(CGF&: *this, Kind: llvm::AtomicRMWInst::UDecWrap, E,
587 Ordering: AtomicOrdering::Monotonic,
588 SSID: getDeviceSyncScope(CGF&: *this));
589
590 case NVPTX::BI__nvvm_ldg_c:
591 case NVPTX::BI__nvvm_ldg_sc:
592 case NVPTX::BI__nvvm_ldg_c2:
593 case NVPTX::BI__nvvm_ldg_sc2:
594 case NVPTX::BI__nvvm_ldg_c4:
595 case NVPTX::BI__nvvm_ldg_sc4:
596 case NVPTX::BI__nvvm_ldg_s:
597 case NVPTX::BI__nvvm_ldg_s2:
598 case NVPTX::BI__nvvm_ldg_s4:
599 case NVPTX::BI__nvvm_ldg_i:
600 case NVPTX::BI__nvvm_ldg_i2:
601 case NVPTX::BI__nvvm_ldg_i4:
602 case NVPTX::BI__nvvm_ldg_l:
603 case NVPTX::BI__nvvm_ldg_l2:
604 case NVPTX::BI__nvvm_ldg_ll:
605 case NVPTX::BI__nvvm_ldg_ll2:
606 case NVPTX::BI__nvvm_ldg_uc:
607 case NVPTX::BI__nvvm_ldg_uc2:
608 case NVPTX::BI__nvvm_ldg_uc4:
609 case NVPTX::BI__nvvm_ldg_us:
610 case NVPTX::BI__nvvm_ldg_us2:
611 case NVPTX::BI__nvvm_ldg_us4:
612 case NVPTX::BI__nvvm_ldg_ui:
613 case NVPTX::BI__nvvm_ldg_ui2:
614 case NVPTX::BI__nvvm_ldg_ui4:
615 case NVPTX::BI__nvvm_ldg_ul:
616 case NVPTX::BI__nvvm_ldg_ul2:
617 case NVPTX::BI__nvvm_ldg_ull:
618 case NVPTX::BI__nvvm_ldg_ull2:
619 case NVPTX::BI__nvvm_ldg_f:
620 case NVPTX::BI__nvvm_ldg_f2:
621 case NVPTX::BI__nvvm_ldg_f4:
622 case NVPTX::BI__nvvm_ldg_d:
623 case NVPTX::BI__nvvm_ldg_d2:
624 // PTX Interoperability section 2.2: "For a vector with an even number of
625 // elements, its alignment is set to number of elements times the alignment
626 // of its member: n*alignof(t)."
627 return MakeLdg(CGF&: *this, E);
628
629 case NVPTX::BI__nvvm_ldu_c:
630 case NVPTX::BI__nvvm_ldu_sc:
631 case NVPTX::BI__nvvm_ldu_c2:
632 case NVPTX::BI__nvvm_ldu_sc2:
633 case NVPTX::BI__nvvm_ldu_c4:
634 case NVPTX::BI__nvvm_ldu_sc4:
635 case NVPTX::BI__nvvm_ldu_s:
636 case NVPTX::BI__nvvm_ldu_s2:
637 case NVPTX::BI__nvvm_ldu_s4:
638 case NVPTX::BI__nvvm_ldu_i:
639 case NVPTX::BI__nvvm_ldu_i2:
640 case NVPTX::BI__nvvm_ldu_i4:
641 case NVPTX::BI__nvvm_ldu_l:
642 case NVPTX::BI__nvvm_ldu_l2:
643 case NVPTX::BI__nvvm_ldu_ll:
644 case NVPTX::BI__nvvm_ldu_ll2:
645 case NVPTX::BI__nvvm_ldu_uc:
646 case NVPTX::BI__nvvm_ldu_uc2:
647 case NVPTX::BI__nvvm_ldu_uc4:
648 case NVPTX::BI__nvvm_ldu_us:
649 case NVPTX::BI__nvvm_ldu_us2:
650 case NVPTX::BI__nvvm_ldu_us4:
651 case NVPTX::BI__nvvm_ldu_ui:
652 case NVPTX::BI__nvvm_ldu_ui2:
653 case NVPTX::BI__nvvm_ldu_ui4:
654 case NVPTX::BI__nvvm_ldu_ul:
655 case NVPTX::BI__nvvm_ldu_ul2:
656 case NVPTX::BI__nvvm_ldu_ull:
657 case NVPTX::BI__nvvm_ldu_ull2:
658 return MakeLdu(IntrinsicID: Intrinsic::nvvm_ldu_global_i, CGF&: *this, E);
659 case NVPTX::BI__nvvm_ldu_f:
660 case NVPTX::BI__nvvm_ldu_f2:
661 case NVPTX::BI__nvvm_ldu_f4:
662 case NVPTX::BI__nvvm_ldu_d:
663 case NVPTX::BI__nvvm_ldu_d2:
664 return MakeLdu(IntrinsicID: Intrinsic::nvvm_ldu_global_f, CGF&: *this, E);
665
666 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
667 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
668 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
669 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Add,
670 Scope: llvm::AtomicScope::Workgroup);
671 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
672 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
673 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
674 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Add,
675 Scope: llvm::AtomicScope::System);
676 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
677 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
678 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::FAdd,
679 Scope: llvm::AtomicScope::Workgroup);
680 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
681 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
682 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::FAdd,
683 Scope: llvm::AtomicScope::System);
684 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
685 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
686 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
687 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Xchg,
688 Scope: llvm::AtomicScope::Workgroup);
689 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
690 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
691 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
692 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Xchg,
693 Scope: llvm::AtomicScope::System);
694 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
695 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
696 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
697 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Max,
698 Scope: llvm::AtomicScope::Workgroup);
699 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
700 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
701 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
702 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::UMax,
703 Scope: llvm::AtomicScope::Workgroup);
704 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
705 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
706 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
707 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Max,
708 Scope: llvm::AtomicScope::System);
709 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
710 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
711 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
712 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::UMax,
713 Scope: llvm::AtomicScope::System);
714 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
715 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
716 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
717 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Min,
718 Scope: llvm::AtomicScope::Workgroup);
719 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
720 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
721 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
722 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::UMin,
723 Scope: llvm::AtomicScope::Workgroup);
724 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
725 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
726 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
727 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Min,
728 Scope: llvm::AtomicScope::System);
729 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
730 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
731 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
732 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::UMin,
733 Scope: llvm::AtomicScope::System);
734 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
735 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::UIncWrap,
736 Scope: llvm::AtomicScope::Workgroup);
737 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
738 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::UDecWrap,
739 Scope: llvm::AtomicScope::Workgroup);
740 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
741 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::UIncWrap,
742 Scope: llvm::AtomicScope::System);
743 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
744 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::UDecWrap,
745 Scope: llvm::AtomicScope::System);
746 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
747 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
748 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
749 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::And,
750 Scope: llvm::AtomicScope::Workgroup);
751 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
752 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
753 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
754 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::And,
755 Scope: llvm::AtomicScope::System);
756 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
757 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
758 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
759 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Or,
760 Scope: llvm::AtomicScope::Workgroup);
761 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
762 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
763 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
764 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Or,
765 Scope: llvm::AtomicScope::System);
766 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
767 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
768 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
769 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Xor,
770 Scope: llvm::AtomicScope::Workgroup);
771 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
772 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
773 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
774 return MakeScopedAtomicRMW(CGF&: *this, E, Kind: llvm::AtomicRMWInst::Xor,
775 Scope: llvm::AtomicScope::System);
776 case NVPTX::BI__nvvm_atom_cta_cas_gen_us:
777 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
778 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
779 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll:
780 return MakeScopedAtomicCAS(CGF&: *this, E, Scope: llvm::AtomicScope::Workgroup);
781 case NVPTX::BI__nvvm_atom_sys_cas_gen_us:
782 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
783 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
784 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll:
785 return MakeScopedAtomicCAS(CGF&: *this, E, Scope: llvm::AtomicScope::System);
786 case NVPTX::BI__nvvm_match_all_sync_i32p:
787 case NVPTX::BI__nvvm_match_all_sync_i64p: {
788 Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
789 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 1));
790 Address PredOutPtr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 2));
791 Value *ResultPair = Builder.CreateCall(
792 Callee: CGM.getIntrinsic(IID: BuiltinID == NVPTX::BI__nvvm_match_all_sync_i32p
793 ? Intrinsic::nvvm_match_all_sync_i32p
794 : Intrinsic::nvvm_match_all_sync_i64p),
795 Args: {Mask, Val});
796 Value *Pred = Builder.CreateZExt(V: Builder.CreateExtractValue(Agg: ResultPair, Idxs: 1),
797 DestTy: PredOutPtr.getElementType());
798 Builder.CreateStore(Val: Pred, Addr: PredOutPtr);
799 return Builder.CreateExtractValue(Agg: ResultPair, Idxs: 0);
800 }
801
802 // FP MMA loads
803 case NVPTX::BI__hmma_m16n16k16_ld_a:
804 case NVPTX::BI__hmma_m16n16k16_ld_b:
805 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
806 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
807 case NVPTX::BI__hmma_m32n8k16_ld_a:
808 case NVPTX::BI__hmma_m32n8k16_ld_b:
809 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
810 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
811 case NVPTX::BI__hmma_m8n32k16_ld_a:
812 case NVPTX::BI__hmma_m8n32k16_ld_b:
813 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
814 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
815 // Integer MMA loads.
816 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
817 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
818 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
819 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
820 case NVPTX::BI__imma_m16n16k16_ld_c:
821 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
822 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
823 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
824 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
825 case NVPTX::BI__imma_m32n8k16_ld_c:
826 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
827 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
828 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
829 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
830 case NVPTX::BI__imma_m8n32k16_ld_c:
831 // Sub-integer MMA loads.
832 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
833 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
834 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
835 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
836 case NVPTX::BI__imma_m8n8k32_ld_c:
837 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
838 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
839 case NVPTX::BI__bmma_m8n8k128_ld_c:
840 // Double MMA loads.
841 case NVPTX::BI__dmma_m8n8k4_ld_a:
842 case NVPTX::BI__dmma_m8n8k4_ld_b:
843 case NVPTX::BI__dmma_m8n8k4_ld_c:
844 // Alternate float MMA loads.
845 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
846 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
847 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
848 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
849 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
850 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
851 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
852 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
853 case NVPTX::BI__mma_tf32_m16n16k8_ld_c: {
854 Address Dst = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
855 Value *Src = EmitScalarExpr(E: E->getArg(Arg: 1));
856 Value *Ldm = EmitScalarExpr(E: E->getArg(Arg: 2));
857 std::optional<llvm::APSInt> isColMajorArg =
858 E->getArg(Arg: 3)->getIntegerConstantExpr(Ctx: getContext());
859 if (!isColMajorArg)
860 return nullptr;
861 bool isColMajor = isColMajorArg->getSExtValue();
862 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
863 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
864 if (IID == 0)
865 return nullptr;
866
867 Value *Result =
868 Builder.CreateCall(Callee: CGM.getIntrinsic(IID, Tys: Src->getType()), Args: {Src, Ldm});
869
870 // Save returned values.
871 assert(II.NumResults);
872 if (II.NumResults == 1) {
873 Builder.CreateAlignedStore(Val: Result, Addr: Dst.emitRawPointer(CGF&: *this),
874 Align: CharUnits::fromQuantity(Quantity: 4));
875 } else {
876 for (unsigned i = 0; i < II.NumResults; ++i) {
877 Builder.CreateAlignedStore(
878 Val: Builder.CreateBitCast(V: Builder.CreateExtractValue(Agg: Result, Idxs: i),
879 DestTy: Dst.getElementType()),
880 Addr: Builder.CreateGEP(Ty: Dst.getElementType(), Ptr: Dst.emitRawPointer(CGF&: *this),
881 IdxList: llvm::ConstantInt::get(Ty: IntTy, V: i)),
882 Align: CharUnits::fromQuantity(Quantity: 4));
883 }
884 }
885 return Result;
886 }
887
888 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
889 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
890 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
891 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
892 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
893 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
894 case NVPTX::BI__imma_m16n16k16_st_c_i32:
895 case NVPTX::BI__imma_m32n8k16_st_c_i32:
896 case NVPTX::BI__imma_m8n32k16_st_c_i32:
897 case NVPTX::BI__imma_m8n8k32_st_c_i32:
898 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
899 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
900 case NVPTX::BI__mma_m16n16k8_st_c_f32: {
901 Value *Dst = EmitScalarExpr(E: E->getArg(Arg: 0));
902 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
903 Value *Ldm = EmitScalarExpr(E: E->getArg(Arg: 2));
904 std::optional<llvm::APSInt> isColMajorArg =
905 E->getArg(Arg: 3)->getIntegerConstantExpr(Ctx: getContext());
906 if (!isColMajorArg)
907 return nullptr;
908 bool isColMajor = isColMajorArg->getSExtValue();
909 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
910 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
911 if (IID == 0)
912 return nullptr;
913 Function *Intrinsic =
914 CGM.getIntrinsic(IID, Tys: Dst->getType());
915 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(i: 1);
916 SmallVector<Value *, 10> Values = {Dst};
917 for (unsigned i = 0; i < II.NumResults; ++i) {
918 Value *V = Builder.CreateAlignedLoad(
919 Ty: Src.getElementType(),
920 Addr: Builder.CreateGEP(Ty: Src.getElementType(), Ptr: Src.emitRawPointer(CGF&: *this),
921 IdxList: llvm::ConstantInt::get(Ty: IntTy, V: i)),
922 Align: CharUnits::fromQuantity(Quantity: 4));
923 Values.push_back(Elt: Builder.CreateBitCast(V, DestTy: ParamType));
924 }
925 Values.push_back(Elt: Ldm);
926 Value *Result = Builder.CreateCall(Callee: Intrinsic, Args: Values);
927 return Result;
928 }
929
930 // BI__hmma_m16n16k16_mma_<Dtype><CType>(d, a, b, c, layout, satf) -->
931 // Intrinsic::nvvm_wmma_m16n16k16_mma_sync<layout A,B><DType><CType><Satf>
932 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
933 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
934 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
935 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
936 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
937 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
938 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
939 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
940 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
941 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
942 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
943 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
944 case NVPTX::BI__imma_m16n16k16_mma_s8:
945 case NVPTX::BI__imma_m16n16k16_mma_u8:
946 case NVPTX::BI__imma_m32n8k16_mma_s8:
947 case NVPTX::BI__imma_m32n8k16_mma_u8:
948 case NVPTX::BI__imma_m8n32k16_mma_s8:
949 case NVPTX::BI__imma_m8n32k16_mma_u8:
950 case NVPTX::BI__imma_m8n8k32_mma_s4:
951 case NVPTX::BI__imma_m8n8k32_mma_u4:
952 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
953 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
954 case NVPTX::BI__dmma_m8n8k4_mma_f64:
955 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
956 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
957 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
958 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32: {
959 Address Dst = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
960 Address SrcA = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
961 Address SrcB = EmitPointerWithAlignment(Addr: E->getArg(Arg: 2));
962 Address SrcC = EmitPointerWithAlignment(Addr: E->getArg(Arg: 3));
963 std::optional<llvm::APSInt> LayoutArg =
964 E->getArg(Arg: 4)->getIntegerConstantExpr(Ctx: getContext());
965 if (!LayoutArg)
966 return nullptr;
967 int Layout = LayoutArg->getSExtValue();
968 if (Layout < 0 || Layout > 3)
969 return nullptr;
970 llvm::APSInt SatfArg;
971 if (BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1 ||
972 BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1)
973 SatfArg = 0; // .b1 does not have satf argument.
974 else if (std::optional<llvm::APSInt> OptSatfArg =
975 E->getArg(Arg: 5)->getIntegerConstantExpr(Ctx: getContext()))
976 SatfArg = *OptSatfArg;
977 else
978 return nullptr;
979 bool Satf = SatfArg.getSExtValue();
980 NVPTXMmaInfo MI = getNVPTXMmaInfo(BuiltinID);
981 unsigned IID = MI.getMMAIntrinsic(Layout, Satf);
982 if (IID == 0) // Unsupported combination of Layout/Satf.
983 return nullptr;
984
985 SmallVector<Value *, 24> Values;
986 Function *Intrinsic = CGM.getIntrinsic(IID);
987 llvm::Type *AType = Intrinsic->getFunctionType()->getParamType(i: 0);
988 // Load A
989 for (unsigned i = 0; i < MI.NumEltsA; ++i) {
990 Value *V = Builder.CreateAlignedLoad(
991 Ty: SrcA.getElementType(),
992 Addr: Builder.CreateGEP(Ty: SrcA.getElementType(), Ptr: SrcA.emitRawPointer(CGF&: *this),
993 IdxList: llvm::ConstantInt::get(Ty: IntTy, V: i)),
994 Align: CharUnits::fromQuantity(Quantity: 4));
995 Values.push_back(Elt: Builder.CreateBitCast(V, DestTy: AType));
996 }
997 // Load B
998 llvm::Type *BType = Intrinsic->getFunctionType()->getParamType(i: MI.NumEltsA);
999 for (unsigned i = 0; i < MI.NumEltsB; ++i) {
1000 Value *V = Builder.CreateAlignedLoad(
1001 Ty: SrcB.getElementType(),
1002 Addr: Builder.CreateGEP(Ty: SrcB.getElementType(), Ptr: SrcB.emitRawPointer(CGF&: *this),
1003 IdxList: llvm::ConstantInt::get(Ty: IntTy, V: i)),
1004 Align: CharUnits::fromQuantity(Quantity: 4));
1005 Values.push_back(Elt: Builder.CreateBitCast(V, DestTy: BType));
1006 }
1007 // Load C
1008 llvm::Type *CType =
1009 Intrinsic->getFunctionType()->getParamType(i: MI.NumEltsA + MI.NumEltsB);
1010 for (unsigned i = 0; i < MI.NumEltsC; ++i) {
1011 Value *V = Builder.CreateAlignedLoad(
1012 Ty: SrcC.getElementType(),
1013 Addr: Builder.CreateGEP(Ty: SrcC.getElementType(), Ptr: SrcC.emitRawPointer(CGF&: *this),
1014 IdxList: llvm::ConstantInt::get(Ty: IntTy, V: i)),
1015 Align: CharUnits::fromQuantity(Quantity: 4));
1016 Values.push_back(Elt: Builder.CreateBitCast(V, DestTy: CType));
1017 }
1018 Value *Result = Builder.CreateCall(Callee: Intrinsic, Args: Values);
1019 llvm::Type *DType = Dst.getElementType();
1020 for (unsigned i = 0; i < MI.NumEltsD; ++i)
1021 Builder.CreateAlignedStore(
1022 Val: Builder.CreateBitCast(V: Builder.CreateExtractValue(Agg: Result, Idxs: i), DestTy: DType),
1023 Addr: Builder.CreateGEP(Ty: Dst.getElementType(), Ptr: Dst.emitRawPointer(CGF&: *this),
1024 IdxList: llvm::ConstantInt::get(Ty: IntTy, V: i)),
1025 Align: CharUnits::fromQuantity(Quantity: 4));
1026 return Result;
1027 }
1028 // The following builtins require half type support
1029 case NVPTX::BI__nvvm_ex2_approx_f16:
1030 return MakeHalfType(
1031 Intrinsic: CGM.getIntrinsic(IID: Intrinsic::nvvm_ex2_approx, Tys: Builder.getHalfTy()),
1032 BuiltinID, E, CGF&: *this);
1033 case NVPTX::BI__nvvm_ex2_approx_f16x2:
1034 return MakeHalfType(
1035 Intrinsic: CGM.getIntrinsic(IID: Intrinsic::nvvm_ex2_approx,
1036 Tys: FixedVectorType::get(ElementType: Builder.getHalfTy(), NumElts: 2)),
1037 BuiltinID, E, CGF&: *this);
1038 case NVPTX::BI__nvvm_ff2f16x2_rn:
1039 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_ff2f16x2_rn, BuiltinID, E, CGF&: *this);
1040 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
1041 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_ff2f16x2_rn_relu, BuiltinID, E, CGF&: *this);
1042 case NVPTX::BI__nvvm_ff2f16x2_rz:
1043 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_ff2f16x2_rz, BuiltinID, E, CGF&: *this);
1044 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
1045 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_ff2f16x2_rz_relu, BuiltinID, E, CGF&: *this);
1046
1047#define PZO_CVT(cvt) \
1048 case NVPTX::BI__nvvm_##cvt##_pzo: \
1049 return MakeHalfType(Intrinsic::nvvm_##cvt, BuiltinID, E, *this, \
1050 {Builder.getTrue()})
1051
1052 PZO_CVT(ff2f16x2_rn);
1053 PZO_CVT(ff2f16x2_rn_relu);
1054 PZO_CVT(ff2f16x2_rz);
1055 PZO_CVT(ff2f16x2_rz_relu);
1056 PZO_CVT(ff2f16x2_rn_satfinite);
1057 PZO_CVT(ff2f16x2_rn_relu_satfinite);
1058 PZO_CVT(ff2f16x2_rz_satfinite);
1059 PZO_CVT(ff2f16x2_rz_relu_satfinite);
1060 PZO_CVT(ff2bf16x2_rn);
1061 PZO_CVT(ff2bf16x2_rn_relu);
1062 PZO_CVT(ff2bf16x2_rz);
1063 PZO_CVT(ff2bf16x2_rz_relu);
1064 PZO_CVT(ff2bf16x2_rn_satfinite);
1065 PZO_CVT(ff2bf16x2_rn_relu_satfinite);
1066 PZO_CVT(ff2bf16x2_rz_satfinite);
1067 PZO_CVT(ff2bf16x2_rz_relu_satfinite);
1068 PZO_CVT(f2f16_rn);
1069 PZO_CVT(f2f16_rn_relu);
1070 PZO_CVT(f2f16_rz);
1071 PZO_CVT(f2f16_rz_relu);
1072 PZO_CVT(f2f16_rn_satfinite);
1073 PZO_CVT(f2f16_rn_relu_satfinite);
1074 PZO_CVT(f2f16_rz_satfinite);
1075 PZO_CVT(f2f16_rz_relu_satfinite);
1076 PZO_CVT(f2bf16_rn);
1077 PZO_CVT(f2bf16_rn_relu);
1078 PZO_CVT(f2bf16_rz);
1079 PZO_CVT(f2bf16_rz_relu);
1080 PZO_CVT(f2bf16_rn_satfinite);
1081 PZO_CVT(f2bf16_rn_relu_satfinite);
1082 PZO_CVT(f2bf16_rz_satfinite);
1083 PZO_CVT(f2bf16_rz_relu_satfinite);
1084
1085 PZO_CVT(ff_to_e4m3x2_rn);
1086 PZO_CVT(ff_to_e4m3x2_rn_relu);
1087 PZO_CVT(ff_to_e4m3x2_rz);
1088 PZO_CVT(ff_to_e4m3x2_rz_relu);
1089 PZO_CVT(ff_to_e5m2x2_rn);
1090 PZO_CVT(ff_to_e5m2x2_rn_relu);
1091 PZO_CVT(ff_to_e5m2x2_rz);
1092 PZO_CVT(ff_to_e5m2x2_rz_relu);
1093 PZO_CVT(f16x2_to_e4m3x2_rn);
1094 PZO_CVT(f16x2_to_e4m3x2_rn_relu);
1095 PZO_CVT(f16x2_to_e4m3x2_rz);
1096 PZO_CVT(f16x2_to_e4m3x2_rz_relu);
1097 PZO_CVT(f16x2_to_e5m2x2_rn);
1098 PZO_CVT(f16x2_to_e5m2x2_rn_relu);
1099 PZO_CVT(f16x2_to_e5m2x2_rz);
1100 PZO_CVT(f16x2_to_e5m2x2_rz_relu);
1101 PZO_CVT(bf16x2_to_e4m3x2_rn_satfinite);
1102 PZO_CVT(bf16x2_to_e4m3x2_rn_relu_satfinite);
1103 PZO_CVT(bf16x2_to_e4m3x2_rz_satfinite);
1104 PZO_CVT(bf16x2_to_e4m3x2_rz_relu_satfinite);
1105 PZO_CVT(bf16x2_to_e5m2x2_rn_satfinite);
1106 PZO_CVT(bf16x2_to_e5m2x2_rn_relu_satfinite);
1107 PZO_CVT(bf16x2_to_e5m2x2_rz_satfinite);
1108 PZO_CVT(bf16x2_to_e5m2x2_rz_relu_satfinite);
1109
1110 PZO_CVT(ff_to_e4m3x2_rn_scale_n1_ue8m0);
1111 PZO_CVT(ff_to_e4m3x2_rn_relu_scale_n1_ue8m0);
1112 PZO_CVT(ff_to_e4m3x2_rz_scale_n1_ue8m0);
1113 PZO_CVT(ff_to_e4m3x2_rz_relu_scale_n1_ue8m0);
1114 PZO_CVT(ff_to_e5m2x2_rn_scale_n1_ue8m0);
1115 PZO_CVT(ff_to_e5m2x2_rn_relu_scale_n1_ue8m0);
1116 PZO_CVT(ff_to_e5m2x2_rz_scale_n1_ue8m0);
1117 PZO_CVT(ff_to_e5m2x2_rz_relu_scale_n1_ue8m0);
1118 PZO_CVT(f16x2_to_e4m3x2_rn_scale_n1_ue8m0);
1119 PZO_CVT(f16x2_to_e4m3x2_rn_relu_scale_n1_ue8m0);
1120 PZO_CVT(f16x2_to_e4m3x2_rz_scale_n1_ue8m0);
1121 PZO_CVT(f16x2_to_e4m3x2_rz_relu_scale_n1_ue8m0);
1122 PZO_CVT(f16x2_to_e5m2x2_rn_scale_n1_ue8m0);
1123 PZO_CVT(f16x2_to_e5m2x2_rn_relu_scale_n1_ue8m0);
1124 PZO_CVT(f16x2_to_e5m2x2_rz_scale_n1_ue8m0);
1125 PZO_CVT(f16x2_to_e5m2x2_rz_relu_scale_n1_ue8m0);
1126 PZO_CVT(bf16x2_to_e4m3x2_rn_satfinite_scale_n1_ue8m0);
1127 PZO_CVT(bf16x2_to_e4m3x2_rn_relu_satfinite_scale_n1_ue8m0);
1128 PZO_CVT(bf16x2_to_e4m3x2_rz_satfinite_scale_n1_ue8m0);
1129 PZO_CVT(bf16x2_to_e4m3x2_rz_relu_satfinite_scale_n1_ue8m0);
1130 PZO_CVT(bf16x2_to_e5m2x2_rn_satfinite_scale_n1_ue8m0);
1131 PZO_CVT(bf16x2_to_e5m2x2_rn_relu_satfinite_scale_n1_ue8m0);
1132 PZO_CVT(bf16x2_to_e5m2x2_rz_satfinite_scale_n1_ue8m0);
1133 PZO_CVT(bf16x2_to_e5m2x2_rz_relu_satfinite_scale_n1_ue8m0);
1134
1135 PZO_CVT(ff_to_e2m3x2_rn_satfinite);
1136 PZO_CVT(ff_to_e2m3x2_rn_relu_satfinite);
1137 PZO_CVT(ff_to_e2m3x2_rz_satfinite);
1138 PZO_CVT(ff_to_e2m3x2_rz_relu_satfinite);
1139 PZO_CVT(ff_to_e3m2x2_rn_satfinite);
1140 PZO_CVT(ff_to_e3m2x2_rn_relu_satfinite);
1141 PZO_CVT(ff_to_e3m2x2_rz_satfinite);
1142 PZO_CVT(ff_to_e3m2x2_rz_relu_satfinite);
1143 PZO_CVT(f16x2_to_e2m3x2_rn_satfinite);
1144 PZO_CVT(f16x2_to_e2m3x2_rn_relu_satfinite);
1145 PZO_CVT(f16x2_to_e2m3x2_rz_satfinite);
1146 PZO_CVT(f16x2_to_e2m3x2_rz_relu_satfinite);
1147 PZO_CVT(f16x2_to_e3m2x2_rn_satfinite);
1148 PZO_CVT(f16x2_to_e3m2x2_rn_relu_satfinite);
1149 PZO_CVT(f16x2_to_e3m2x2_rz_satfinite);
1150 PZO_CVT(f16x2_to_e3m2x2_rz_relu_satfinite);
1151 PZO_CVT(bf16x2_to_e2m3x2_rn_satfinite);
1152 PZO_CVT(bf16x2_to_e2m3x2_rn_relu_satfinite);
1153 PZO_CVT(bf16x2_to_e2m3x2_rz_satfinite);
1154 PZO_CVT(bf16x2_to_e2m3x2_rz_relu_satfinite);
1155 PZO_CVT(bf16x2_to_e3m2x2_rn_satfinite);
1156 PZO_CVT(bf16x2_to_e3m2x2_rn_relu_satfinite);
1157 PZO_CVT(bf16x2_to_e3m2x2_rz_satfinite);
1158 PZO_CVT(bf16x2_to_e3m2x2_rz_relu_satfinite);
1159
1160 PZO_CVT(ff_to_e2m3x2_rn_satfinite_scale_n1_ue8m0);
1161 PZO_CVT(ff_to_e2m3x2_rn_relu_satfinite_scale_n1_ue8m0);
1162 PZO_CVT(ff_to_e2m3x2_rz_satfinite_scale_n1_ue8m0);
1163 PZO_CVT(ff_to_e2m3x2_rz_relu_satfinite_scale_n1_ue8m0);
1164 PZO_CVT(ff_to_e3m2x2_rn_satfinite_scale_n1_ue8m0);
1165 PZO_CVT(ff_to_e3m2x2_rn_relu_satfinite_scale_n1_ue8m0);
1166 PZO_CVT(ff_to_e3m2x2_rz_satfinite_scale_n1_ue8m0);
1167 PZO_CVT(ff_to_e3m2x2_rz_relu_satfinite_scale_n1_ue8m0);
1168 PZO_CVT(f16x2_to_e2m3x2_rn_satfinite_scale_n1_ue8m0);
1169 PZO_CVT(f16x2_to_e2m3x2_rn_relu_satfinite_scale_n1_ue8m0);
1170 PZO_CVT(f16x2_to_e2m3x2_rz_satfinite_scale_n1_ue8m0);
1171 PZO_CVT(f16x2_to_e2m3x2_rz_relu_satfinite_scale_n1_ue8m0);
1172 PZO_CVT(f16x2_to_e3m2x2_rn_satfinite_scale_n1_ue8m0);
1173 PZO_CVT(f16x2_to_e3m2x2_rn_relu_satfinite_scale_n1_ue8m0);
1174 PZO_CVT(f16x2_to_e3m2x2_rz_satfinite_scale_n1_ue8m0);
1175 PZO_CVT(f16x2_to_e3m2x2_rz_relu_satfinite_scale_n1_ue8m0);
1176 PZO_CVT(bf16x2_to_e2m3x2_rn_satfinite_scale_n1_ue8m0);
1177 PZO_CVT(bf16x2_to_e2m3x2_rn_relu_satfinite_scale_n1_ue8m0);
1178 PZO_CVT(bf16x2_to_e2m3x2_rz_satfinite_scale_n1_ue8m0);
1179 PZO_CVT(bf16x2_to_e2m3x2_rz_relu_satfinite_scale_n1_ue8m0);
1180 PZO_CVT(bf16x2_to_e3m2x2_rn_satfinite_scale_n1_ue8m0);
1181 PZO_CVT(bf16x2_to_e3m2x2_rn_relu_satfinite_scale_n1_ue8m0);
1182 PZO_CVT(bf16x2_to_e3m2x2_rz_satfinite_scale_n1_ue8m0);
1183 PZO_CVT(bf16x2_to_e3m2x2_rz_relu_satfinite_scale_n1_ue8m0);
1184
1185 PZO_CVT(ff_to_e2m1x2_rn_satfinite);
1186 PZO_CVT(ff_to_e2m1x2_rn_relu_satfinite);
1187 PZO_CVT(ff_to_e2m1x2_rz_satfinite);
1188 PZO_CVT(ff_to_e2m1x2_rz_relu_satfinite);
1189 PZO_CVT(f16x2_to_e2m1x2_rn_satfinite);
1190 PZO_CVT(f16x2_to_e2m1x2_rn_relu_satfinite);
1191 PZO_CVT(f16x2_to_e2m1x2_rz_satfinite);
1192 PZO_CVT(f16x2_to_e2m1x2_rz_relu_satfinite);
1193 PZO_CVT(bf16x2_to_e2m1x2_rn_satfinite);
1194 PZO_CVT(bf16x2_to_e2m1x2_rn_relu_satfinite);
1195 PZO_CVT(bf16x2_to_e2m1x2_rz_satfinite);
1196 PZO_CVT(bf16x2_to_e2m1x2_rz_relu_satfinite);
1197
1198 PZO_CVT(ff_to_e2m1x2_rn_satfinite_scale_n1_ue8m0);
1199 PZO_CVT(ff_to_e2m1x2_rn_relu_satfinite_scale_n1_ue8m0);
1200 PZO_CVT(ff_to_e2m1x2_rz_satfinite_scale_n1_ue8m0);
1201 PZO_CVT(ff_to_e2m1x2_rz_relu_satfinite_scale_n1_ue8m0);
1202 PZO_CVT(f16x2_to_e2m1x2_rn_satfinite_scale_n1_ue8m0);
1203 PZO_CVT(f16x2_to_e2m1x2_rn_relu_satfinite_scale_n1_ue8m0);
1204 PZO_CVT(f16x2_to_e2m1x2_rz_satfinite_scale_n1_ue8m0);
1205 PZO_CVT(f16x2_to_e2m1x2_rz_relu_satfinite_scale_n1_ue8m0);
1206 PZO_CVT(bf16x2_to_e2m1x2_rn_satfinite_scale_n1_ue8m0);
1207 PZO_CVT(bf16x2_to_e2m1x2_rn_relu_satfinite_scale_n1_ue8m0);
1208 PZO_CVT(bf16x2_to_e2m1x2_rz_satfinite_scale_n1_ue8m0);
1209 PZO_CVT(bf16x2_to_e2m1x2_rz_relu_satfinite_scale_n1_ue8m0);
1210
1211#undef PZO_CVT
1212
1213 case NVPTX::BI__nvvm_fma_rn_f16:
1214 case NVPTX::BI__nvvm_fma_rn_f16x2:
1215 case NVPTX::BI__nvvm_fma_rn_bf16:
1216 case NVPTX::BI__nvvm_fma_rn_bf16x2:
1217 case NVPTX::BI__nvvm_fma_rn_f:
1218 case NVPTX::BI__nvvm_fma_rn_d:
1219 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma, RM: APFloat::rmNearestTiesToEven,
1220 BuiltinID, E, CGF&: *this);
1221 case NVPTX::BI__nvvm_fma_rz_f:
1222 case NVPTX::BI__nvvm_fma_rz_d:
1223 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma, RM: APFloat::rmTowardZero, BuiltinID, E,
1224 CGF&: *this);
1225 case NVPTX::BI__nvvm_fma_rm_f:
1226 case NVPTX::BI__nvvm_fma_rm_d:
1227 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma, RM: APFloat::rmTowardNegative,
1228 BuiltinID, E, CGF&: *this);
1229 case NVPTX::BI__nvvm_fma_rp_f:
1230 case NVPTX::BI__nvvm_fma_rp_d:
1231 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma, RM: APFloat::rmTowardPositive,
1232 BuiltinID, E, CGF&: *this);
1233 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
1234 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
1235 case NVPTX::BI__nvvm_fma_rn_ftz_f:
1236 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma_ftz, RM: APFloat::rmNearestTiesToEven,
1237 BuiltinID, E, CGF&: *this);
1238 case NVPTX::BI__nvvm_fma_rz_ftz_f:
1239 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma_ftz, RM: APFloat::rmTowardZero,
1240 BuiltinID, E, CGF&: *this);
1241 case NVPTX::BI__nvvm_fma_rm_ftz_f:
1242 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma_ftz, RM: APFloat::rmTowardNegative,
1243 BuiltinID, E, CGF&: *this);
1244 case NVPTX::BI__nvvm_fma_rp_ftz_f:
1245 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma_ftz, RM: APFloat::rmTowardPositive,
1246 BuiltinID, E, CGF&: *this);
1247 case NVPTX::BI__nvvm_fma_rn_sat_f16:
1248 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
1249 case NVPTX::BI__nvvm_fma_rn_sat_f:
1250 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma_sat, RM: APFloat::rmNearestTiesToEven,
1251 BuiltinID, E, CGF&: *this);
1252 case NVPTX::BI__nvvm_fma_rz_sat_f:
1253 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma_sat, RM: APFloat::rmTowardZero,
1254 BuiltinID, E, CGF&: *this);
1255 case NVPTX::BI__nvvm_fma_rm_sat_f:
1256 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma_sat, RM: APFloat::rmTowardNegative,
1257 BuiltinID, E, CGF&: *this);
1258 case NVPTX::BI__nvvm_fma_rp_sat_f:
1259 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma_sat, RM: APFloat::rmTowardPositive,
1260 BuiltinID, E, CGF&: *this);
1261 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
1262 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
1263 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f:
1264 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma_ftz_sat,
1265 RM: APFloat::rmNearestTiesToEven, BuiltinID, E, CGF&: *this);
1266 case NVPTX::BI__nvvm_fma_rz_ftz_sat_f:
1267 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma_ftz_sat, RM: APFloat::rmTowardZero,
1268 BuiltinID, E, CGF&: *this);
1269 case NVPTX::BI__nvvm_fma_rm_ftz_sat_f:
1270 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma_ftz_sat, RM: APFloat::rmTowardNegative,
1271 BuiltinID, E, CGF&: *this);
1272 case NVPTX::BI__nvvm_fma_rp_ftz_sat_f:
1273 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma_ftz_sat, RM: APFloat::rmTowardPositive,
1274 BuiltinID, E, CGF&: *this);
1275 case NVPTX::BI__nvvm_fma_rn_relu_f16:
1276 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
1277 case NVPTX::BI__nvvm_fma_rn_relu_bf16:
1278 case NVPTX::BI__nvvm_fma_rn_relu_bf16x2:
1279 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma_relu, RM: APFloat::rmNearestTiesToEven,
1280 BuiltinID, E, CGF&: *this);
1281 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
1282 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
1283 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fma_ftz_relu,
1284 RM: APFloat::rmNearestTiesToEven, BuiltinID, E, CGF&: *this);
1285 case NVPTX::BI__nvvm_fma_rn_oob_f16:
1286 return MakeFMAOOB(IntrinsicID: Intrinsic::nvvm_fma_rn_oob, Ty: Builder.getHalfTy(), E,
1287 CGF&: *this);
1288 case NVPTX::BI__nvvm_fma_rn_oob_f16x2:
1289 return MakeFMAOOB(IntrinsicID: Intrinsic::nvvm_fma_rn_oob,
1290 Ty: llvm::FixedVectorType::get(ElementType: Builder.getHalfTy(), NumElts: 2), E,
1291 CGF&: *this);
1292 case NVPTX::BI__nvvm_fma_rn_oob_bf16:
1293 return MakeFMAOOB(IntrinsicID: Intrinsic::nvvm_fma_rn_oob, Ty: Builder.getBFloatTy(), E,
1294 CGF&: *this);
1295 case NVPTX::BI__nvvm_fma_rn_oob_bf16x2:
1296 return MakeFMAOOB(IntrinsicID: Intrinsic::nvvm_fma_rn_oob,
1297 Ty: llvm::FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 2), E,
1298 CGF&: *this);
1299 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16:
1300 return MakeFMAOOB(IntrinsicID: Intrinsic::nvvm_fma_rn_oob_relu, Ty: Builder.getHalfTy(), E,
1301 CGF&: *this);
1302 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16x2:
1303 return MakeFMAOOB(IntrinsicID: Intrinsic::nvvm_fma_rn_oob_relu,
1304 Ty: llvm::FixedVectorType::get(ElementType: Builder.getHalfTy(), NumElts: 2), E,
1305 CGF&: *this);
1306 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16:
1307 return MakeFMAOOB(IntrinsicID: Intrinsic::nvvm_fma_rn_oob_relu, Ty: Builder.getBFloatTy(), E,
1308 CGF&: *this);
1309 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16x2:
1310 return MakeFMAOOB(IntrinsicID: Intrinsic::nvvm_fma_rn_oob_relu,
1311 Ty: llvm::FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 2), E,
1312 CGF&: *this);
1313 case NVPTX::BI__nvvm_fmax_f16:
1314 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_f16, BuiltinID, E, CGF&: *this);
1315 case NVPTX::BI__nvvm_fmax_f16x2:
1316 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_f16x2, BuiltinID, E, CGF&: *this);
1317 case NVPTX::BI__nvvm_fmax_ftz_f16:
1318 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_ftz_f16, BuiltinID, E, CGF&: *this);
1319 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
1320 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_ftz_f16x2, BuiltinID, E, CGF&: *this);
1321 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
1322 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_ftz_nan_f16, BuiltinID, E, CGF&: *this);
1323 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
1324 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_ftz_nan_f16x2, BuiltinID, E,
1325 CGF&: *this);
1326 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
1327 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16, BuiltinID,
1328 E, CGF&: *this);
1329 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
1330 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16x2,
1331 BuiltinID, E, CGF&: *this);
1332 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
1333 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16, BuiltinID, E,
1334 CGF&: *this);
1335 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
1336 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16x2, BuiltinID,
1337 E, CGF&: *this);
1338 case NVPTX::BI__nvvm_fmax_nan_f16:
1339 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_nan_f16, BuiltinID, E, CGF&: *this);
1340 case NVPTX::BI__nvvm_fmax_nan_f16x2:
1341 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_nan_f16x2, BuiltinID, E, CGF&: *this);
1342 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
1343 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_nan_xorsign_abs_f16, BuiltinID, E,
1344 CGF&: *this);
1345 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
1346 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_nan_xorsign_abs_f16x2, BuiltinID,
1347 E, CGF&: *this);
1348 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
1349 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_xorsign_abs_f16, BuiltinID, E,
1350 CGF&: *this);
1351 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
1352 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmax_xorsign_abs_f16x2, BuiltinID, E,
1353 CGF&: *this);
1354 case NVPTX::BI__nvvm_fmin_f16:
1355 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_f16, BuiltinID, E, CGF&: *this);
1356 case NVPTX::BI__nvvm_fmin_f16x2:
1357 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_f16x2, BuiltinID, E, CGF&: *this);
1358 case NVPTX::BI__nvvm_fmin_ftz_f16:
1359 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_ftz_f16, BuiltinID, E, CGF&: *this);
1360 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
1361 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_ftz_f16x2, BuiltinID, E, CGF&: *this);
1362 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
1363 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_ftz_nan_f16, BuiltinID, E, CGF&: *this);
1364 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
1365 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_ftz_nan_f16x2, BuiltinID, E,
1366 CGF&: *this);
1367 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
1368 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16, BuiltinID,
1369 E, CGF&: *this);
1370 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
1371 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16x2,
1372 BuiltinID, E, CGF&: *this);
1373 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
1374 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16, BuiltinID, E,
1375 CGF&: *this);
1376 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
1377 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16x2, BuiltinID,
1378 E, CGF&: *this);
1379 case NVPTX::BI__nvvm_fmin_nan_f16:
1380 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_nan_f16, BuiltinID, E, CGF&: *this);
1381 case NVPTX::BI__nvvm_fmin_nan_f16x2:
1382 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_nan_f16x2, BuiltinID, E, CGF&: *this);
1383 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
1384 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_nan_xorsign_abs_f16, BuiltinID, E,
1385 CGF&: *this);
1386 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
1387 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_nan_xorsign_abs_f16x2, BuiltinID,
1388 E, CGF&: *this);
1389 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
1390 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_xorsign_abs_f16, BuiltinID, E,
1391 CGF&: *this);
1392 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
1393 return MakeHalfType(IntrinsicID: Intrinsic::nvvm_fmin_xorsign_abs_f16x2, BuiltinID, E,
1394 CGF&: *this);
1395 case NVPTX::BI__nvvm_fabs_f:
1396 case NVPTX::BI__nvvm_abs_bf16:
1397 case NVPTX::BI__nvvm_abs_bf16x2:
1398 case NVPTX::BI__nvvm_fabs_f16:
1399 case NVPTX::BI__nvvm_fabs_f16x2:
1400 return Builder.CreateUnaryIntrinsic(ID: Intrinsic::nvvm_fabs,
1401 Op: EmitScalarExpr(E: E->getArg(Arg: 0)));
1402 case NVPTX::BI__nvvm_fabs_ftz_f:
1403 case NVPTX::BI__nvvm_fabs_ftz_f16:
1404 case NVPTX::BI__nvvm_fabs_ftz_f16x2:
1405 return Builder.CreateUnaryIntrinsic(ID: Intrinsic::nvvm_fabs_ftz,
1406 Op: EmitScalarExpr(E: E->getArg(Arg: 0)));
1407 case NVPTX::BI__nvvm_fabs_d:
1408 return Builder.CreateFAbs(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
1409 case NVPTX::BI__nvvm_ex2_approx_d:
1410 case NVPTX::BI__nvvm_ex2_approx_f:
1411 return Builder.CreateUnaryIntrinsic(ID: Intrinsic::nvvm_ex2_approx,
1412 Op: EmitScalarExpr(E: E->getArg(Arg: 0)));
1413 case NVPTX::BI__nvvm_ex2_approx_ftz_f:
1414 return Builder.CreateUnaryIntrinsic(ID: Intrinsic::nvvm_ex2_approx_ftz,
1415 Op: EmitScalarExpr(E: E->getArg(Arg: 0)));
1416 case NVPTX::BI__nvvm_add_rn_f:
1417 case NVPTX::BI__nvvm_add_rn_d:
1418 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd, RM: APFloat::rmNearestTiesToEven,
1419 BuiltinID, E, CGF&: *this);
1420 case NVPTX::BI__nvvm_add_rz_f:
1421 case NVPTX::BI__nvvm_add_rz_d:
1422 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd, RM: APFloat::rmTowardZero, BuiltinID,
1423 E, CGF&: *this);
1424 case NVPTX::BI__nvvm_add_rm_f:
1425 case NVPTX::BI__nvvm_add_rm_d:
1426 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd, RM: APFloat::rmTowardNegative,
1427 BuiltinID, E, CGF&: *this);
1428 case NVPTX::BI__nvvm_add_rp_f:
1429 case NVPTX::BI__nvvm_add_rp_d:
1430 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd, RM: APFloat::rmTowardPositive,
1431 BuiltinID, E, CGF&: *this);
1432 case NVPTX::BI__nvvm_add_rn_ftz_f:
1433 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_ftz, RM: APFloat::rmNearestTiesToEven,
1434 BuiltinID, E, CGF&: *this);
1435 case NVPTX::BI__nvvm_add_rz_ftz_f:
1436 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_ftz, RM: APFloat::rmTowardZero,
1437 BuiltinID, E, CGF&: *this);
1438 case NVPTX::BI__nvvm_add_rm_ftz_f:
1439 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_ftz, RM: APFloat::rmTowardNegative,
1440 BuiltinID, E, CGF&: *this);
1441 case NVPTX::BI__nvvm_add_rp_ftz_f:
1442 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_ftz, RM: APFloat::rmTowardPositive,
1443 BuiltinID, E, CGF&: *this);
1444 case NVPTX::BI__nvvm_add_rn_sat_f:
1445 case NVPTX::BI__nvvm_add_rn_sat_f16:
1446 case NVPTX::BI__nvvm_add_rn_sat_v2f16:
1447 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_sat, RM: APFloat::rmNearestTiesToEven,
1448 BuiltinID, E, CGF&: *this);
1449 case NVPTX::BI__nvvm_add_rz_sat_f:
1450 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_sat, RM: APFloat::rmTowardZero,
1451 BuiltinID, E, CGF&: *this);
1452 case NVPTX::BI__nvvm_add_rm_sat_f:
1453 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_sat, RM: APFloat::rmTowardNegative,
1454 BuiltinID, E, CGF&: *this);
1455 case NVPTX::BI__nvvm_add_rp_sat_f:
1456 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_sat, RM: APFloat::rmTowardPositive,
1457 BuiltinID, E, CGF&: *this);
1458 case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
1459 case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
1460 case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
1461 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_ftz_sat,
1462 RM: APFloat::rmNearestTiesToEven, BuiltinID, E, CGF&: *this);
1463 case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
1464 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_ftz_sat, RM: APFloat::rmTowardZero,
1465 BuiltinID, E, CGF&: *this);
1466 case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
1467 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_ftz_sat, RM: APFloat::rmTowardNegative,
1468 BuiltinID, E, CGF&: *this);
1469 case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
1470 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fadd_ftz_sat, RM: APFloat::rmTowardPositive,
1471 BuiltinID, E, CGF&: *this);
1472 case NVPTX::BI__nvvm_mul_rn_f:
1473 case NVPTX::BI__nvvm_mul_rn_d:
1474 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul, RM: APFloat::rmNearestTiesToEven,
1475 BuiltinID, E, CGF&: *this);
1476 case NVPTX::BI__nvvm_mul_rz_f:
1477 case NVPTX::BI__nvvm_mul_rz_d:
1478 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul, RM: APFloat::rmTowardZero, BuiltinID,
1479 E, CGF&: *this);
1480 case NVPTX::BI__nvvm_mul_rm_f:
1481 case NVPTX::BI__nvvm_mul_rm_d:
1482 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul, RM: APFloat::rmTowardNegative,
1483 BuiltinID, E, CGF&: *this);
1484 case NVPTX::BI__nvvm_mul_rp_f:
1485 case NVPTX::BI__nvvm_mul_rp_d:
1486 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul, RM: APFloat::rmTowardPositive,
1487 BuiltinID, E, CGF&: *this);
1488 case NVPTX::BI__nvvm_mul_rn_ftz_f:
1489 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul_ftz, RM: APFloat::rmNearestTiesToEven,
1490 BuiltinID, E, CGF&: *this);
1491 case NVPTX::BI__nvvm_mul_rz_ftz_f:
1492 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul_ftz, RM: APFloat::rmTowardZero,
1493 BuiltinID, E, CGF&: *this);
1494 case NVPTX::BI__nvvm_mul_rm_ftz_f:
1495 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul_ftz, RM: APFloat::rmTowardNegative,
1496 BuiltinID, E, CGF&: *this);
1497 case NVPTX::BI__nvvm_mul_rp_ftz_f:
1498 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul_ftz, RM: APFloat::rmTowardPositive,
1499 BuiltinID, E, CGF&: *this);
1500 case NVPTX::BI__nvvm_mul_rn_sat_f16:
1501 case NVPTX::BI__nvvm_mul_rn_sat_v2f16:
1502 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul_sat, RM: APFloat::rmNearestTiesToEven,
1503 BuiltinID, E, CGF&: *this);
1504 case NVPTX::BI__nvvm_mul_rn_ftz_sat_f16:
1505 case NVPTX::BI__nvvm_mul_rn_ftz_sat_v2f16:
1506 return MakeFPArith(IntrinsicID: Intrinsic::nvvm_fmul_ftz_sat,
1507 RM: APFloat::rmNearestTiesToEven, BuiltinID, E, CGF&: *this);
1508 case NVPTX::BI__nvvm_ldg_h:
1509 case NVPTX::BI__nvvm_ldg_h2:
1510 return MakeLdg(CGF&: *this, E);
1511 case NVPTX::BI__nvvm_ldu_h:
1512 case NVPTX::BI__nvvm_ldu_h2:
1513 return MakeLdu(IntrinsicID: Intrinsic::nvvm_ldu_global_f, CGF&: *this, E);
1514 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
1515 return MakeCpAsync(IntrinsicID: Intrinsic::nvvm_cp_async_ca_shared_global_4,
1516 IntrinsicIDS: Intrinsic::nvvm_cp_async_ca_shared_global_4_s, CGF&: *this, E,
1517 SrcSize: 4);
1518 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
1519 return MakeCpAsync(IntrinsicID: Intrinsic::nvvm_cp_async_ca_shared_global_8,
1520 IntrinsicIDS: Intrinsic::nvvm_cp_async_ca_shared_global_8_s, CGF&: *this, E,
1521 SrcSize: 8);
1522 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
1523 return MakeCpAsync(IntrinsicID: Intrinsic::nvvm_cp_async_ca_shared_global_16,
1524 IntrinsicIDS: Intrinsic::nvvm_cp_async_ca_shared_global_16_s, CGF&: *this, E,
1525 SrcSize: 16);
1526 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
1527 return MakeCpAsync(IntrinsicID: Intrinsic::nvvm_cp_async_cg_shared_global_16,
1528 IntrinsicIDS: Intrinsic::nvvm_cp_async_cg_shared_global_16_s, CGF&: *this, E,
1529 SrcSize: 16);
1530 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
1531 return Builder.CreateCall(
1532 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_clusterid_x));
1533 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
1534 return Builder.CreateCall(
1535 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_clusterid_y));
1536 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
1537 return Builder.CreateCall(
1538 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_clusterid_z));
1539 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
1540 return Builder.CreateCall(
1541 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_clusterid_w));
1542 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
1543 return Builder.CreateCall(
1544 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_nclusterid_x));
1545 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
1546 return Builder.CreateCall(
1547 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_nclusterid_y));
1548 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
1549 return Builder.CreateCall(
1550 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_nclusterid_z));
1551 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
1552 return Builder.CreateCall(
1553 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_nclusterid_w));
1554 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
1555 return Builder.CreateCall(
1556 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_x));
1557 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
1558 return Builder.CreateCall(
1559 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_y));
1560 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
1561 return Builder.CreateCall(
1562 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_z));
1563 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
1564 return Builder.CreateCall(
1565 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_w));
1566 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
1567 return Builder.CreateCall(
1568 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_x));
1569 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
1570 return Builder.CreateCall(
1571 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_y));
1572 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
1573 return Builder.CreateCall(
1574 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_z));
1575 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
1576 return Builder.CreateCall(
1577 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_w));
1578 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
1579 return Builder.CreateCall(
1580 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_ctarank));
1581 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
1582 return Builder.CreateCall(
1583 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_read_ptx_sreg_cluster_nctarank));
1584 case NVPTX::BI__nvvm_is_explicit_cluster:
1585 return Builder.CreateCall(
1586 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_is_explicit_cluster));
1587 case NVPTX::BI__nvvm_isspacep_shared_cluster:
1588 return Builder.CreateCall(
1589 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_isspacep_shared_cluster),
1590 Args: EmitScalarExpr(E: E->getArg(Arg: 0)));
1591 case NVPTX::BI__nvvm_mapa:
1592 return Builder.CreateCall(
1593 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_mapa),
1594 Args: {EmitScalarExpr(E: E->getArg(Arg: 0)), EmitScalarExpr(E: E->getArg(Arg: 1))});
1595 case NVPTX::BI__nvvm_mapa_shared_cluster:
1596 return Builder.CreateCall(
1597 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_mapa_shared_cluster),
1598 Args: {EmitScalarExpr(E: E->getArg(Arg: 0)), EmitScalarExpr(E: E->getArg(Arg: 1))});
1599 case NVPTX::BI__nvvm_getctarank:
1600 return Builder.CreateCall(
1601 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_getctarank),
1602 Args: EmitScalarExpr(E: E->getArg(Arg: 0)));
1603 case NVPTX::BI__nvvm_getctarank_shared_cluster:
1604 return Builder.CreateCall(
1605 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_getctarank_shared_cluster),
1606 Args: EmitScalarExpr(E: E->getArg(Arg: 0)));
1607 case NVPTX::BI__nvvm_barrier_cluster_arrive:
1608 return Builder.CreateCall(
1609 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_barrier_cluster_arrive));
1610 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
1611 return Builder.CreateCall(
1612 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_barrier_cluster_arrive_relaxed));
1613 case NVPTX::BI__nvvm_barrier_cluster_wait:
1614 return Builder.CreateCall(
1615 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_barrier_cluster_wait));
1616 case NVPTX::BI__nvvm_fence_sc_cluster:
1617 return Builder.CreateCall(
1618 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_fence_sc_cluster));
1619 case NVPTX::BI__nvvm_bar_sync:
1620 return Builder.CreateCall(
1621 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_barrier_cta_sync_aligned_all),
1622 Args: EmitScalarExpr(E: E->getArg(Arg: 0)));
1623 case NVPTX::BI__syncthreads:
1624 return Builder.CreateCall(
1625 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_barrier_cta_sync_aligned_all),
1626 Args: Builder.getInt32(C: 0));
1627 case NVPTX::BI__nvvm_barrier_sync:
1628 return Builder.CreateCall(
1629 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_barrier_cta_sync_all),
1630 Args: EmitScalarExpr(E: E->getArg(Arg: 0)));
1631 case NVPTX::BI__nvvm_barrier_sync_cnt:
1632 return Builder.CreateCall(
1633 Callee: CGM.getIntrinsic(IID: Intrinsic::nvvm_barrier_cta_sync_count),
1634 Args: {EmitScalarExpr(E: E->getArg(Arg: 0)), EmitScalarExpr(E: E->getArg(Arg: 1))});
1635 case NVPTX::BI__nvvm_bar0_and:
1636 return Builder.CreateZExt(
1637 V: Builder.CreateIntrinsic(
1638 ID: Intrinsic::nvvm_barrier_cta_red_and_aligned_all, OverloadTypes: {},
1639 Args: {Builder.getInt32(C: 0),
1640 Builder.CreateICmpNE(LHS: EmitScalarExpr(E: E->getArg(Arg: 0)),
1641 RHS: Builder.getInt32(C: 0))}),
1642 DestTy: Builder.getInt32Ty());
1643 case NVPTX::BI__nvvm_bar0_or:
1644 return Builder.CreateZExt(
1645 V: Builder.CreateIntrinsic(
1646 ID: Intrinsic::nvvm_barrier_cta_red_or_aligned_all, OverloadTypes: {},
1647 Args: {Builder.getInt32(C: 0),
1648 Builder.CreateICmpNE(LHS: EmitScalarExpr(E: E->getArg(Arg: 0)),
1649 RHS: Builder.getInt32(C: 0))}),
1650 DestTy: Builder.getInt32Ty());
1651 case NVPTX::BI__nvvm_bar0_popc:
1652 return Builder.CreateIntrinsic(
1653 ID: Intrinsic::nvvm_barrier_cta_red_popc_aligned_all, OverloadTypes: {},
1654 Args: {Builder.getInt32(C: 0), Builder.CreateICmpNE(LHS: EmitScalarExpr(E: E->getArg(Arg: 0)),
1655 RHS: Builder.getInt32(C: 0))});
1656 case NVPTX::BI__nvvm_mbarrier_init:
1657 case NVPTX::BI__nvvm_mbarrier_init_shared: {
1658 // The intrinsic is overloaded on the pointer, so the two builtins differ
1659 // only in the address space of their first argument.
1660 Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 0));
1661 return Builder.CreateIntrinsic(
1662 ID: Intrinsic::nvvm_mbarrier_init, OverloadTypes: {Ptr->getType()},
1663 Args: {Ptr, EmitScalarExpr(E: E->getArg(Arg: 1)), Builder.getInt32(C: 0)});
1664 }
1665 case NVPTX::BI__nvvm_idp2a_lo_s_s:
1666 return MakeIdp2a(IntrinsicID: Intrinsic::nvvm_idp2a_s_s, /*IsHi=*/false, E, CGF&: *this);
1667 case NVPTX::BI__nvvm_idp2a_lo_s_u:
1668 return MakeIdp2a(IntrinsicID: Intrinsic::nvvm_idp2a_s_u, /*IsHi=*/false, E, CGF&: *this);
1669 case NVPTX::BI__nvvm_idp2a_lo_u_s:
1670 return MakeIdp2a(IntrinsicID: Intrinsic::nvvm_idp2a_u_s, /*IsHi=*/false, E, CGF&: *this);
1671 case NVPTX::BI__nvvm_idp2a_lo_u_u:
1672 return MakeIdp2a(IntrinsicID: Intrinsic::nvvm_idp2a_u_u, /*IsHi=*/false, E, CGF&: *this);
1673 case NVPTX::BI__nvvm_idp2a_hi_s_s:
1674 return MakeIdp2a(IntrinsicID: Intrinsic::nvvm_idp2a_s_s, /*IsHi=*/true, E, CGF&: *this);
1675 case NVPTX::BI__nvvm_idp2a_hi_s_u:
1676 return MakeIdp2a(IntrinsicID: Intrinsic::nvvm_idp2a_s_u, /*IsHi=*/true, E, CGF&: *this);
1677 case NVPTX::BI__nvvm_idp2a_hi_u_s:
1678 return MakeIdp2a(IntrinsicID: Intrinsic::nvvm_idp2a_u_s, /*IsHi=*/true, E, CGF&: *this);
1679 case NVPTX::BI__nvvm_idp2a_hi_u_u:
1680 return MakeIdp2a(IntrinsicID: Intrinsic::nvvm_idp2a_u_u, /*IsHi=*/true, E, CGF&: *this);
1681 default:
1682 return nullptr;
1683 }
1684}
1685