1//===- NVVMToLLVMIRTranslation.cpp - Translate NVVM to LLVM IR ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a translation between the MLIR NVVM dialect and
10// LLVM IR.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.h"
15#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
16#include "mlir/Dialect/LLVMIR/NVVMDialect.h"
17#include "mlir/IR/Operation.h"
18#include "mlir/Target/LLVMIR/ModuleTranslation.h"
19
20#include "llvm/ADT/StringExtras.h"
21#include "llvm/ADT/iterator_range.h"
22#include "llvm/IR/IRBuilder.h"
23#include "llvm/IR/IntrinsicsNVPTX.h"
24#include "llvm/Support/FormatVariadic.h"
25#include "llvm/Support/NVVMAttributes.h"
26#include <cmath>
27
28using namespace mlir;
29using namespace mlir::LLVM;
30using mlir::LLVM::detail::createIntrinsicCall;
31
32#define REDUX_F32_ID_IMPL(op, abs, hasNaN) \
33 hasNaN ? llvm::Intrinsic::nvvm_redux_sync_f##op##abs##_NaN \
34 : llvm::Intrinsic::nvvm_redux_sync_f##op##abs
35
36#define GET_REDUX_F32_ID(op, hasAbs, hasNaN) \
37 hasAbs ? REDUX_F32_ID_IMPL(op, _abs, hasNaN) : REDUX_F32_ID_IMPL(op, , hasNaN)
38
39static llvm::Intrinsic::ID getReduxIntrinsicId(llvm::Type *resultType,
40 NVVM::ReductionKind kind,
41 bool hasAbs, bool hasNaN) {
42 switch (kind) {
43 case NVVM::ReductionKind::ADD:
44 return llvm::Intrinsic::nvvm_redux_sync_add;
45 case NVVM::ReductionKind::UMAX:
46 return llvm::Intrinsic::nvvm_redux_sync_umax;
47 case NVVM::ReductionKind::UMIN:
48 return llvm::Intrinsic::nvvm_redux_sync_umin;
49 case NVVM::ReductionKind::AND:
50 return llvm::Intrinsic::nvvm_redux_sync_and;
51 case NVVM::ReductionKind::OR:
52 return llvm::Intrinsic::nvvm_redux_sync_or;
53 case NVVM::ReductionKind::XOR:
54 return llvm::Intrinsic::nvvm_redux_sync_xor;
55 case NVVM::ReductionKind::MAX:
56 return llvm::Intrinsic::nvvm_redux_sync_max;
57 case NVVM::ReductionKind::MIN:
58 return llvm::Intrinsic::nvvm_redux_sync_min;
59 case NVVM::ReductionKind::FMIN:
60 return GET_REDUX_F32_ID(min, hasAbs, hasNaN);
61 case NVVM::ReductionKind::FMAX:
62 return GET_REDUX_F32_ID(max, hasAbs, hasNaN);
63 }
64 llvm_unreachable("unknown reduction kind");
65}
66
67static llvm::Intrinsic::ID getShflIntrinsicId(llvm::Type *resultType,
68 NVVM::ShflKind kind,
69 bool withPredicate) {
70
71 if (withPredicate) {
72 resultType = cast<llvm::StructType>(Val: resultType)->getElementType(N: 0);
73 switch (kind) {
74 case NVVM::ShflKind::bfly:
75 return resultType->isFloatTy()
76 ? llvm::Intrinsic::nvvm_shfl_sync_bfly_f32p
77 : llvm::Intrinsic::nvvm_shfl_sync_bfly_i32p;
78 case NVVM::ShflKind::up:
79 return resultType->isFloatTy() ? llvm::Intrinsic::nvvm_shfl_sync_up_f32p
80 : llvm::Intrinsic::nvvm_shfl_sync_up_i32p;
81 case NVVM::ShflKind::down:
82 return resultType->isFloatTy()
83 ? llvm::Intrinsic::nvvm_shfl_sync_down_f32p
84 : llvm::Intrinsic::nvvm_shfl_sync_down_i32p;
85 case NVVM::ShflKind::idx:
86 return resultType->isFloatTy() ? llvm::Intrinsic::nvvm_shfl_sync_idx_f32p
87 : llvm::Intrinsic::nvvm_shfl_sync_idx_i32p;
88 }
89 } else {
90 switch (kind) {
91 case NVVM::ShflKind::bfly:
92 return resultType->isFloatTy() ? llvm::Intrinsic::nvvm_shfl_sync_bfly_f32
93 : llvm::Intrinsic::nvvm_shfl_sync_bfly_i32;
94 case NVVM::ShflKind::up:
95 return resultType->isFloatTy() ? llvm::Intrinsic::nvvm_shfl_sync_up_f32
96 : llvm::Intrinsic::nvvm_shfl_sync_up_i32;
97 case NVVM::ShflKind::down:
98 return resultType->isFloatTy() ? llvm::Intrinsic::nvvm_shfl_sync_down_f32
99 : llvm::Intrinsic::nvvm_shfl_sync_down_i32;
100 case NVVM::ShflKind::idx:
101 return resultType->isFloatTy() ? llvm::Intrinsic::nvvm_shfl_sync_idx_f32
102 : llvm::Intrinsic::nvvm_shfl_sync_idx_i32;
103 }
104 }
105 llvm_unreachable("unknown shuffle kind");
106}
107
108static llvm::Intrinsic::ID getMatchSyncIntrinsicId(Type valType,
109 NVVM::MatchSyncKind kind) {
110 switch (kind) {
111 case NVVM::MatchSyncKind::any:
112 return valType.isInteger(width: 32) ? llvm::Intrinsic::nvvm_match_any_sync_i32
113 : llvm::Intrinsic::nvvm_match_any_sync_i64;
114 case NVVM::MatchSyncKind::all:
115 // match.all instruction has two variants -- one returns a single value,
116 // another returns a pair {value, predicate}. We currently only implement
117 // the latter as that's the variant exposed by CUDA API.
118 return valType.isInteger(width: 32) ? llvm::Intrinsic::nvvm_match_all_sync_i32p
119 : llvm::Intrinsic::nvvm_match_all_sync_i64p;
120 }
121 llvm_unreachable("unsupported match sync kind");
122}
123
124static llvm::Intrinsic::ID getVoteSyncIntrinsicId(NVVM::VoteSyncKind kind) {
125 switch (kind) {
126 case NVVM::VoteSyncKind::any:
127 return llvm::Intrinsic::nvvm_vote_any_sync;
128 case NVVM::VoteSyncKind::all:
129 return llvm::Intrinsic::nvvm_vote_all_sync;
130 case NVVM::VoteSyncKind::ballot:
131 return llvm::Intrinsic::nvvm_vote_ballot_sync;
132 case NVVM::VoteSyncKind::uni:
133 return llvm::Intrinsic::nvvm_vote_uni_sync;
134 }
135 llvm_unreachable("unsupported vote kind");
136}
137
138static llvm::Intrinsic::ID
139getLdMatrixIntrinsicId(NVVM::MMALayout layout, int32_t num,
140 NVVM::LdStMatrixShapeAttr shape,
141 NVVM::LdStMatrixEltType eltType) {
142 if (shape.getM() == 8 && shape.getN() == 8) {
143 switch (num) {
144 case 1:
145 return (layout == NVVM::MMALayout::row)
146 ? llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_b16
147 : llvm::Intrinsic::
148 nvvm_ldmatrix_sync_aligned_m8n8_x1_trans_b16;
149 case 2:
150 return (layout == NVVM::MMALayout::row)
151 ? llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_b16
152 : llvm::Intrinsic::
153 nvvm_ldmatrix_sync_aligned_m8n8_x2_trans_b16;
154 case 4:
155 return (layout == NVVM::MMALayout::row)
156 ? llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_b16
157 : llvm::Intrinsic::
158 nvvm_ldmatrix_sync_aligned_m8n8_x4_trans_b16;
159 }
160 } else if (shape.getM() == 8 && shape.getN() == 16) {
161 if (eltType == NVVM::LdStMatrixEltType::B8X16_B6X16_P32) {
162 switch (num) {
163 case 1:
164 return llvm::Intrinsic::
165 nvvm_ldmatrix_sync_aligned_m8n16_x1_b8x16_b6x16_p32;
166 case 2:
167 return llvm::Intrinsic::
168 nvvm_ldmatrix_sync_aligned_m8n16_x2_b8x16_b6x16_p32;
169 case 4:
170 return llvm::Intrinsic::
171 nvvm_ldmatrix_sync_aligned_m8n16_x4_b8x16_b6x16_p32;
172 }
173 } else if (eltType == NVVM::LdStMatrixEltType::B8X16_B4X16_P64) {
174 switch (num) {
175 case 1:
176 return llvm::Intrinsic::
177 nvvm_ldmatrix_sync_aligned_m8n16_x1_b8x16_b4x16_p64;
178 case 2:
179 return llvm::Intrinsic::
180 nvvm_ldmatrix_sync_aligned_m8n16_x2_b8x16_b4x16_p64;
181 case 4:
182 return llvm::Intrinsic::
183 nvvm_ldmatrix_sync_aligned_m8n16_x4_b8x16_b4x16_p64;
184 }
185 } else if (eltType == NVVM::LdStMatrixEltType::S8_S4) {
186 switch (num) {
187 case 1:
188 return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x1_s8_s4;
189 case 2:
190 return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x2_s8_s4;
191 case 4:
192 return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x4_s8_s4;
193 }
194 }
195 } else if (shape.getM() == 16 && shape.getN() == 16) {
196 if (eltType == NVVM::LdStMatrixEltType::B8) {
197 switch (num) {
198 case 1:
199 return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8;
200 case 2:
201 return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8;
202 }
203 } else if (eltType == NVVM::LdStMatrixEltType::B8X16_B6X16_P32) {
204 switch (num) {
205 case 1:
206 return llvm::Intrinsic::
207 nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8x16_b6x16_p32;
208 case 2:
209 return llvm::Intrinsic::
210 nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8x16_b6x16_p32;
211 }
212 } else if (eltType == NVVM::LdStMatrixEltType::B8X16_B4X16_P64) {
213 switch (num) {
214 case 1:
215 return llvm::Intrinsic::
216 nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8x16_b4x16_p64;
217 case 2:
218 return llvm::Intrinsic::
219 nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8x16_b4x16_p64;
220 }
221 }
222 }
223 llvm_unreachable("unknown ldmatrix kind");
224}
225
226/// Return the intrinsic ID associated with stmatrix for the given paramters.
227static llvm::Intrinsic::ID
228getStMatrixIntrinsicId(NVVM::MMALayout layout, int32_t num,
229 NVVM::LdStMatrixShapeAttr shape,
230 NVVM::LdStMatrixEltType eltType) {
231 if (shape.getM() == 8 && shape.getN() == 8) {
232 switch (num) {
233 case 1:
234 return (layout == NVVM::MMALayout::row)
235 ? llvm::Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x1_b16
236 : llvm::Intrinsic::
237 nvvm_stmatrix_sync_aligned_m8n8_x1_trans_b16;
238 case 2:
239 return (layout == NVVM::MMALayout::row)
240 ? llvm::Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x2_b16
241 : llvm::Intrinsic::
242 nvvm_stmatrix_sync_aligned_m8n8_x2_trans_b16;
243 case 4:
244 return (layout == NVVM::MMALayout::row)
245 ? llvm::Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x4_b16
246 : llvm::Intrinsic::
247 nvvm_stmatrix_sync_aligned_m8n8_x4_trans_b16;
248 }
249 } else if (shape.getM() == 16 && shape.getN() == 8) {
250 switch (num) {
251 case 1:
252 return llvm::Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x1_trans_b8;
253 case 2:
254 return llvm::Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x2_trans_b8;
255 case 4:
256 return llvm::Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x4_trans_b8;
257 }
258 }
259 llvm_unreachable("unknown stmatrix kind");
260}
261
262static unsigned getUnidirectionalFenceProxyID(NVVM::ProxyKind fromProxy,
263 NVVM::ProxyKind toProxy,
264 NVVM::MemScopeKind scope,
265 bool isRelease) {
266 if (fromProxy == NVVM::ProxyKind::GENERIC &&
267 toProxy == NVVM::ProxyKind::TENSORMAP) {
268 switch (scope) {
269 case NVVM::MemScopeKind::CTA: {
270 if (isRelease)
271 return llvm::Intrinsic::nvvm_fence_proxy_tensormap_generic_release_cta;
272 return llvm::Intrinsic::nvvm_fence_proxy_tensormap_generic_acquire_cta;
273 }
274 case NVVM::MemScopeKind::CLUSTER: {
275 if (isRelease)
276 return llvm::Intrinsic::
277 nvvm_fence_proxy_tensormap_generic_release_cluster;
278 return llvm::Intrinsic::
279 nvvm_fence_proxy_tensormap_generic_acquire_cluster;
280 }
281 case NVVM::MemScopeKind::GPU: {
282 if (isRelease)
283 return llvm::Intrinsic::nvvm_fence_proxy_tensormap_generic_release_gpu;
284 return llvm::Intrinsic::nvvm_fence_proxy_tensormap_generic_acquire_gpu;
285 }
286 case NVVM::MemScopeKind::SYS: {
287 if (isRelease)
288 return llvm::Intrinsic::nvvm_fence_proxy_tensormap_generic_release_sys;
289 return llvm::Intrinsic::nvvm_fence_proxy_tensormap_generic_acquire_sys;
290 }
291 }
292 llvm_unreachable("Unknown scope for uni-directional fence.proxy operation");
293 }
294 llvm_unreachable("Unsupported proxy kinds");
295}
296
297static unsigned getMembarIntrinsicID(NVVM::MemScopeKind scope) {
298 switch (scope) {
299 case NVVM::MemScopeKind::CTA:
300 return llvm::Intrinsic::nvvm_membar_cta;
301 case NVVM::MemScopeKind::CLUSTER:
302 return llvm::Intrinsic::nvvm_fence_sc_cluster;
303 case NVVM::MemScopeKind::GPU:
304 return llvm::Intrinsic::nvvm_membar_gl;
305 case NVVM::MemScopeKind::SYS:
306 return llvm::Intrinsic::nvvm_membar_sys;
307 }
308 llvm_unreachable("Unknown scope for memory barrier");
309}
310
311#define TCGEN05LD(SHAPE, NUM) llvm::Intrinsic::nvvm_tcgen05_ld_##SHAPE##_##NUM
312
313static llvm::Intrinsic::ID
314getTcgen05LdIntrinsicID(mlir::NVVM::Tcgen05LdStShape shape, uint32_t num) {
315 llvm::Intrinsic::ID Shape16x64b[] = {
316 TCGEN05LD(16x64b, x1), TCGEN05LD(16x64b, x2), TCGEN05LD(16x64b, x4),
317 TCGEN05LD(16x64b, x8), TCGEN05LD(16x64b, x16), TCGEN05LD(16x64b, x32),
318 TCGEN05LD(16x64b, x64), TCGEN05LD(16x64b, x128),
319 };
320
321 llvm::Intrinsic::ID Shape16x128b[] = {
322 TCGEN05LD(16x128b, x1), TCGEN05LD(16x128b, x2), TCGEN05LD(16x128b, x4),
323 TCGEN05LD(16x128b, x8), TCGEN05LD(16x128b, x16), TCGEN05LD(16x128b, x32),
324 TCGEN05LD(16x128b, x64),
325 };
326
327 llvm::Intrinsic::ID Shape16x256b[] = {
328 TCGEN05LD(16x256b, x1), TCGEN05LD(16x256b, x2), TCGEN05LD(16x256b, x4),
329 TCGEN05LD(16x256b, x8), TCGEN05LD(16x256b, x16), TCGEN05LD(16x256b, x32),
330 };
331
332 llvm::Intrinsic::ID Shape16x32bx2[] = {
333 TCGEN05LD(16x32bx2, x1), TCGEN05LD(16x32bx2, x2),
334 TCGEN05LD(16x32bx2, x4), TCGEN05LD(16x32bx2, x8),
335 TCGEN05LD(16x32bx2, x16), TCGEN05LD(16x32bx2, x32),
336 TCGEN05LD(16x32bx2, x64), TCGEN05LD(16x32bx2, x128),
337 };
338
339 llvm::Intrinsic::ID Shape32x32b[] = {
340 TCGEN05LD(32x32b, x1), TCGEN05LD(32x32b, x2), TCGEN05LD(32x32b, x4),
341 TCGEN05LD(32x32b, x8), TCGEN05LD(32x32b, x16), TCGEN05LD(32x32b, x32),
342 TCGEN05LD(32x32b, x64), TCGEN05LD(32x32b, x128),
343 };
344
345 // `num` contains the length of vector and log2 of `num` returns the index
346 // into the shape array
347 unsigned Idx = std::log2(x: num);
348
349 switch (shape) {
350 case NVVM::Tcgen05LdStShape::SHAPE_16X64B:
351 return Shape16x64b[Idx];
352 case NVVM::Tcgen05LdStShape::SHAPE_16X128B:
353 return Shape16x128b[Idx - 1];
354 case NVVM::Tcgen05LdStShape::SHAPE_16X256B:
355 return Shape16x256b[Idx - 2];
356 case NVVM::Tcgen05LdStShape::SHAPE_32X32B:
357 return Shape32x32b[Idx];
358 case NVVM::Tcgen05LdStShape::SHAPE_16X32BX2:
359 return Shape16x32bx2[Idx];
360 }
361 llvm_unreachable("unhandled tcgen05.ld lowering");
362}
363
364#define TCGEN05ST(SHAPE, NUM) llvm::Intrinsic::nvvm_tcgen05_st_##SHAPE##_##NUM
365
366static llvm::Intrinsic::ID
367getTcgen05StIntrinsicID(mlir::NVVM::Tcgen05LdStShape shape, uint32_t num) {
368 llvm::Intrinsic::ID Shape16x64b[] = {
369 TCGEN05ST(16x64b, x1), TCGEN05ST(16x64b, x2), TCGEN05ST(16x64b, x4),
370 TCGEN05ST(16x64b, x8), TCGEN05ST(16x64b, x16), TCGEN05ST(16x64b, x32),
371 TCGEN05ST(16x64b, x64), TCGEN05ST(16x64b, x128),
372 };
373
374 llvm::Intrinsic::ID Shape16x128b[] = {
375 TCGEN05ST(16x128b, x1), TCGEN05ST(16x128b, x2), TCGEN05ST(16x128b, x4),
376 TCGEN05ST(16x128b, x8), TCGEN05ST(16x128b, x16), TCGEN05ST(16x128b, x32),
377 TCGEN05ST(16x128b, x64),
378 };
379
380 llvm::Intrinsic::ID Shape16x256b[] = {
381 TCGEN05ST(16x256b, x1), TCGEN05ST(16x256b, x2), TCGEN05ST(16x256b, x4),
382 TCGEN05ST(16x256b, x8), TCGEN05ST(16x256b, x16), TCGEN05ST(16x256b, x32),
383 };
384
385 llvm::Intrinsic::ID Shape16x32bx2[] = {
386 TCGEN05ST(16x32bx2, x1), TCGEN05ST(16x32bx2, x2),
387 TCGEN05ST(16x32bx2, x4), TCGEN05ST(16x32bx2, x8),
388 TCGEN05ST(16x32bx2, x16), TCGEN05ST(16x32bx2, x32),
389 TCGEN05ST(16x32bx2, x64), TCGEN05ST(16x32bx2, x128),
390 };
391
392 llvm::Intrinsic::ID Shape32x32b[] = {
393 TCGEN05ST(32x32b, x1), TCGEN05ST(32x32b, x2), TCGEN05ST(32x32b, x4),
394 TCGEN05ST(32x32b, x8), TCGEN05ST(32x32b, x16), TCGEN05ST(32x32b, x32),
395 TCGEN05ST(32x32b, x64), TCGEN05ST(32x32b, x128),
396 };
397
398 // `num` contains the length of vector and log2 of `num` returns the index
399 // into the shape array
400 unsigned Idx = std::log2(x: num);
401
402 switch (shape) {
403 case NVVM::Tcgen05LdStShape::SHAPE_16X64B:
404 return Shape16x64b[Idx];
405 case NVVM::Tcgen05LdStShape::SHAPE_16X128B:
406 return Shape16x128b[Idx - 1];
407 case NVVM::Tcgen05LdStShape::SHAPE_16X256B:
408 return Shape16x256b[Idx - 2];
409 case NVVM::Tcgen05LdStShape::SHAPE_32X32B:
410 return Shape32x32b[Idx];
411 case NVVM::Tcgen05LdStShape::SHAPE_16X32BX2:
412 return Shape16x32bx2[Idx];
413 }
414 llvm_unreachable("unhandled tcgen05.st lowering");
415}
416
417static llvm::Intrinsic::ID getFenceSyncRestrictID(NVVM::MemOrderKind order) {
418 return order == NVVM::MemOrderKind::ACQUIRE
419 ? llvm::Intrinsic::
420 nvvm_fence_acquire_sync_restrict_space_cluster_scope_cluster
421 : llvm::Intrinsic::
422 nvvm_fence_release_sync_restrict_space_cta_scope_cluster;
423}
424
425static llvm::Intrinsic::ID
426getFenceProxyID(NVVM::ProxyKind kind, std::optional<NVVM::SharedSpace> space) {
427 switch (kind) {
428 case NVVM::ProxyKind::alias:
429 return llvm::Intrinsic::nvvm_fence_proxy_alias;
430 case NVVM::ProxyKind::async:
431 return llvm::Intrinsic::nvvm_fence_proxy_async;
432 case NVVM::ProxyKind::async_global:
433 return llvm::Intrinsic::nvvm_fence_proxy_async_global;
434 case NVVM::ProxyKind::async_shared:
435 return *space == NVVM::SharedSpace::shared_cta
436 ? llvm::Intrinsic::nvvm_fence_proxy_async_shared_cta
437 : llvm::Intrinsic::nvvm_fence_proxy_async_shared_cluster;
438 default:
439 llvm_unreachable("unsupported proxy kind");
440 }
441}
442
443static llvm::Intrinsic::ID
444getFenceProxySyncRestrictID(NVVM::MemOrderKind order) {
445 return order == NVVM::MemOrderKind::ACQUIRE
446 ? llvm::Intrinsic::
447 nvvm_fence_proxy_async_generic_acquire_sync_restrict_space_cluster_scope_cluster
448 : llvm::Intrinsic::
449 nvvm_fence_proxy_async_generic_release_sync_restrict_space_cta_scope_cluster;
450}
451
452static llvm::RoundingMode
453getLLVMRoundingModeForFPArith(NVVM::FPRoundingMode rndMode) {
454 switch (rndMode) {
455 case NVVM::FPRoundingMode::RN:
456 return llvm::RoundingMode::NearestTiesToEven;
457 case NVVM::FPRoundingMode::RM:
458 return llvm::RoundingMode::TowardNegative;
459 case NVVM::FPRoundingMode::RP:
460 return llvm::RoundingMode::TowardPositive;
461 case NVVM::FPRoundingMode::RZ:
462 return llvm::RoundingMode::TowardZero;
463 default:
464 // default rounding mode is RN
465 assert(rndMode == NVVM::FPRoundingMode::NONE &&
466 "unsupported rounding mode for nvvm fp arithmetic");
467 return llvm::RoundingMode::NearestTiesToEven;
468 }
469}
470
471// Calls an LLVM intrinsic on the given operands. For f32/f64 vector types,
472// the intrinsic is called per-element and the results are packed back into a
473// vector. If retType is non-null, it is forwarded as the return-type
474// overload to `createIntrinsicCall`.
475static llvm::Value *
476createScalarizedIntrinsicCall(llvm::IRBuilderBase &builder,
477 llvm::Intrinsic::ID IID, llvm::Type *opTypeLLVM,
478 ArrayRef<llvm::Value *> operands,
479 llvm::Type *retType) {
480 if (opTypeLLVM->isVectorTy() && (opTypeLLVM->getScalarType()->isFloatTy() ||
481 opTypeLLVM->getScalarType()->isDoubleTy())) {
482 llvm::Value *result = llvm::PoisonValue::get(
483 T: llvm::FixedVectorType::get(ElementType: opTypeLLVM->getScalarType(), NumElts: 2));
484 for (int64_t i = 0; i < 2; ++i) {
485 llvm::SmallVector<llvm::Value *> scalarArgs;
486 for (llvm::Value *op : operands)
487 scalarArgs.push_back(
488 Elt: op->getType()->isVectorTy()
489 ? builder.CreateExtractElement(Vec: op, Idx: builder.getInt32(C: i))
490 : op);
491 llvm::Value *res = createIntrinsicCall(builder, intrinsic: IID, retTy: retType, args: scalarArgs);
492 result = builder.CreateInsertElement(Vec: result, NewElt: res, Idx: builder.getInt32(C: i));
493 }
494 return result;
495 }
496
497 return createIntrinsicCall(builder, intrinsic: IID, retTy: retType, args: operands);
498}
499
500// Lowers a binary FP arithmetic op to the intrinsic selected from `ids`, which
501// is indexed by [ftz][sat].
502static void lowerFloatBinaryOpToLLVMIR(const llvm::Intrinsic::ID (&ids)[2][2],
503 llvm::Value *argLHS, llvm::Value *argRHS,
504 Value res, NVVM::FPRoundingMode rndMode,
505 NVVM::SaturationMode satMode, bool isFTZ,
506 LLVM::ModuleTranslation &mt,
507 llvm::IRBuilderBase &builder) {
508 llvm::Type *opTypeLLVM = argLHS->getType();
509 bool isSat = satMode != NVVM::SaturationMode::NONE;
510
511 llvm::Intrinsic::ID id = ids[isFTZ][isSat];
512 llvm::Value *rnd = builder.getInt32(
513 C: static_cast<int>(getLLVMRoundingModeForFPArith(rndMode)));
514
515 // For f64 vectors, and f32 vectors with saturation, we need to scalarize
516 // the intrinsic call.
517 llvm::Type *scalarTypeLLVM = opTypeLLVM->getScalarType();
518 if (opTypeLLVM->isVectorTy() && (scalarTypeLLVM->isDoubleTy() ||
519 (isSat && scalarTypeLLVM->isFloatTy()))) {
520 mt.mapValue(mlir: res, llvm: createScalarizedIntrinsicCall(builder, IID: id, opTypeLLVM,
521 operands: {argLHS, argRHS, rnd},
522 retType: scalarTypeLLVM));
523 return;
524 }
525
526 mt.mapValue(
527 mlir: res, llvm: createIntrinsicCall(builder, intrinsic: id, retTy: opTypeLLVM, args: {argLHS, argRHS, rnd}));
528}
529
530void NVVM::AddFOp::lowerAddFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS,
531 Value res, NVVM::FPRoundingMode rndMode,
532 NVVM::SaturationMode satMode, bool isFTZ,
533 LLVM::ModuleTranslation &mt,
534 llvm::IRBuilderBase &builder) {
535 static constexpr llvm::Intrinsic::ID addIDs[2][2] = {
536 {llvm::Intrinsic::nvvm_fadd, llvm::Intrinsic::nvvm_fadd_sat},
537 {llvm::Intrinsic::nvvm_fadd_ftz, llvm::Intrinsic::nvvm_fadd_ftz_sat}};
538 lowerFloatBinaryOpToLLVMIR(ids: addIDs, argLHS, argRHS, res, rndMode, satMode,
539 isFTZ, mt, builder);
540}
541
542void NVVM::MulFOp::lowerMulFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS,
543 Value res, NVVM::FPRoundingMode rndMode,
544 NVVM::SaturationMode satMode, bool isFTZ,
545 LLVM::ModuleTranslation &mt,
546 llvm::IRBuilderBase &builder) {
547 static constexpr llvm::Intrinsic::ID mulIDs[2][2] = {
548 {llvm::Intrinsic::nvvm_fmul, llvm::Intrinsic::nvvm_fmul_sat},
549 {llvm::Intrinsic::nvvm_fmul_ftz, llvm::Intrinsic::nvvm_fmul_ftz_sat}};
550 lowerFloatBinaryOpToLLVMIR(ids: mulIDs, argLHS, argRHS, res, rndMode, satMode,
551 isFTZ, mt, builder);
552}
553
554void NVVM::FmaOp::lowerFmaToLLVMIR(Operation &op, LLVM::ModuleTranslation &mt,
555 llvm::IRBuilderBase &builder) {
556 auto thisOp = cast<NVVM::FmaOp>(Val&: op);
557 mlir::NVVM::FPRoundingMode rndMode = thisOp.getRnd();
558 bool isFTZ = thisOp.getFtz();
559 bool isRelu = thisOp.getRelu();
560 bool isSat = thisOp.getSat() == NVVM::SaturationMode::SAT;
561 bool isOOB = thisOp.getOob();
562
563 mlir::Type opType = thisOp.getRes().getType();
564 llvm::Type *opTypeLLVM = mt.convertType(type: opType);
565
566 llvm::Value *argA = mt.lookupValue(value: thisOp.getA());
567 llvm::Value *argB = mt.lookupValue(value: thisOp.getB());
568 llvm::Value *argC = mt.lookupValue(value: thisOp.getC());
569
570 // The oob variants only support RN and take no rounding-mode operand.
571 if (isOOB) {
572 llvm::Intrinsic::ID id = isRelu ? llvm::Intrinsic::nvvm_fma_rn_oob_relu
573 : llvm::Intrinsic::nvvm_fma_rn_oob;
574 mt.mapValue(mlir: thisOp.getRes(), llvm: createIntrinsicCall(builder, intrinsic: id, retTy: opTypeLLVM,
575 args: {argA, argB, argC}));
576 return;
577 }
578
579 static constexpr llvm::Intrinsic::ID fmaIDs[2][3] = {
580 {llvm::Intrinsic::nvvm_fma, llvm::Intrinsic::nvvm_fma_sat,
581 llvm::Intrinsic::nvvm_fma_relu},
582 {llvm::Intrinsic::nvvm_fma_ftz, llvm::Intrinsic::nvvm_fma_ftz_sat,
583 llvm::Intrinsic::nvvm_fma_ftz_relu}};
584
585 llvm::Intrinsic::ID id = fmaIDs[isFTZ][isSat ? 1 : (isRelu ? 2 : 0)];
586 llvm::Value *rnd = builder.getInt32(
587 C: static_cast<int>(getLLVMRoundingModeForFPArith(rndMode)));
588
589 // For f64 vector fma, and f32 vector fma with saturation, we need to
590 // scalarize the intrinsic call.
591 llvm::Type *scalarTypeLLVM = opTypeLLVM->getScalarType();
592 if (opTypeLLVM->isVectorTy() && (scalarTypeLLVM->isDoubleTy() ||
593 (isSat && scalarTypeLLVM->isFloatTy()))) {
594 mt.mapValue(mlir: thisOp.getRes(), llvm: createScalarizedIntrinsicCall(
595 builder, IID: id, opTypeLLVM,
596 operands: {argA, argB, argC, rnd}, retType: scalarTypeLLVM));
597 return;
598 }
599
600 mt.mapValue(mlir: thisOp.getRes(), llvm: createIntrinsicCall(builder, intrinsic: id, retTy: opTypeLLVM,
601 args: {argA, argB, argC, rnd}));
602}
603
604// Returns the LLVM type the sp{de}compress intrinsics use for metadata: i32 for
605// a single register (including vector<1xi32>) and a vector of i32 otherwise.
606static llvm::Type *getSPMetadataType(Type type, llvm::IRBuilderBase &builder) {
607 auto vecTy = dyn_cast<VectorType>(Val&: type);
608 if (!vecTy || vecTy.getNumElements() == 1)
609 return builder.getInt32Ty();
610 return llvm::FixedVectorType::get(ElementType: builder.getInt32Ty(),
611 NumElts: vecTy.getNumElements());
612}
613
614void NVVM::SparseCompressOp::lowerToLLVMIR(Operation &op,
615 LLVM::ModuleTranslation &mt,
616 llvm::IRBuilderBase &builder) {
617 auto thisOp = cast<NVVM::SparseCompressOp>(Val&: op);
618 Value metadata = thisOp.getMetadata();
619 Value compressedData = thisOp.getCompressedData();
620 llvm::Type *resultTy =
621 llvm::StructType::get(elt1: getSPMetadataType(type: metadata.getType(), builder),
622 elts: mt.convertType(type: compressedData.getType()));
623
624 llvm::Value *result = createIntrinsicCall(
625 builder, intrinsic: llvm::Intrinsic::nvvm_spcompress, retTy: resultTy,
626 args: {mt.lookupValue(value: thisOp.getData()), mt.lookupValue(value: thisOp.getSpDesc()),
627 builder.getInt32(C: thisOp.getIdxSize()),
628 // spcompress only supports the 2:4 sparsity factor.
629 builder.getInt32(C: 4)});
630
631 mt.mapValue(value: metadata) =
632 builder.CreateBitCast(V: builder.CreateExtractValue(Agg: result, Idxs: 0),
633 DestTy: mt.convertType(type: metadata.getType()));
634 mt.mapValue(value: compressedData) = builder.CreateExtractValue(Agg: result, Idxs: 1);
635}
636
637void NVVM::SparseDecompressOp::lowerToLLVMIR(Operation &op,
638 LLVM::ModuleTranslation &mt,
639 llvm::IRBuilderBase &builder) {
640 auto thisOp = cast<NVVM::SparseDecompressOp>(Val&: op);
641 Value metadata = thisOp.getMetadata();
642 mt.mapValue(value: thisOp.getData()) = createIntrinsicCall(
643 builder, intrinsic: llvm::Intrinsic::nvvm_spdecompress,
644 retTy: mt.convertType(type: thisOp.getData().getType()),
645 args: {builder.CreateBitCast(V: mt.lookupValue(value: metadata),
646 DestTy: getSPMetadataType(type: metadata.getType(), builder)),
647 mt.lookupValue(value: thisOp.getCompressedData()),
648 builder.getInt32(C: thisOp.getIdxSize()),
649 builder.getInt32(C: thisOp.getNumTgt())});
650}
651
652namespace {
653/// Implementation of the dialect interface that converts operations belonging
654/// to the NVVM dialect to LLVM IR.
655class NVVMDialectLLVMIRTranslationInterface
656 : public LLVMTranslationDialectInterface {
657public:
658 using LLVMTranslationDialectInterface::LLVMTranslationDialectInterface;
659
660 /// Translates the given operation to LLVM IR using the provided IR builder
661 /// and saving the state in `moduleTranslation`.
662 LogicalResult
663 convertOperation(Operation *op, llvm::IRBuilderBase &builder,
664 LLVM::ModuleTranslation &moduleTranslation) const final {
665 // All NVVM ops are instruction-level and require an active insertion point.
666 // A null insert block means the op is misplaced (e.g., at module scope),
667 // which would otherwise cause a null dereference in createIntrinsicCall.
668 if (!builder.GetInsertBlock())
669 return op->emitOpError(
670 message: "cannot be translated to LLVM IR without an active insertion "
671 "point; make sure the op is inside a function");
672 Operation &opInst = *op;
673#include "mlir/Dialect/LLVMIR/NVVMConversions.inc"
674
675 return failure();
676 }
677
678 /// Attaches module-level metadata for functions marked as kernels
679 /// and managed annotations for global variables.
680 LogicalResult
681 amendOperation(Operation *op, ArrayRef<llvm::Instruction *> instructions,
682 NamedAttribute attribute,
683 LLVM::ModuleTranslation &moduleTranslation) const final {
684 if (auto globalOp = dyn_cast<LLVM::GlobalOp>(Val: op)) {
685 if (attribute.getName() == NVVM::NVVMDialect::getManagedAttrName()) {
686 auto *gv = cast<llvm::GlobalVariable>(
687 Val: moduleTranslation.lookupGlobal(op: globalOp));
688 llvm::Module *m = gv->getParent();
689 llvm::LLVMContext &ctx = m->getContext();
690 llvm::NamedMDNode *md = m->getOrInsertNamedMetadata(Name: "nvvm.annotations");
691 md->addOperand(M: llvm::MDNode::get(
692 Context&: ctx, MDs: {llvm::ConstantAsMetadata::get(C: gv),
693 llvm::MDString::get(Context&: ctx, Str: "managed"),
694 llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
695 Ty: llvm::Type::getInt32Ty(C&: ctx), V: 1))}));
696 }
697 return success();
698 }
699
700 auto func = dyn_cast<LLVM::LLVMFuncOp>(Val: op);
701 if (!func)
702 return failure();
703 llvm::Function *llvmFunc = moduleTranslation.lookupFunction(name: func.getName());
704
705 if (attribute.getName() == NVVM::NVVMDialect::getMaxntidAttrName()) {
706 if (!isa<DenseI32ArrayAttr>(Val: attribute.getValue()))
707 return failure();
708 auto values = cast<DenseI32ArrayAttr>(Val: attribute.getValue());
709 const std::string attr = llvm::formatv(
710 Fmt: "{0:$[,]}", Vals: llvm::make_range(x: values.asArrayRef().begin(),
711 y: values.asArrayRef().end()));
712 llvmFunc->addFnAttr(Kind: llvm::NVVMAttr::MaxNTID, Val: attr);
713 } else if (attribute.getName() == NVVM::NVVMDialect::getReqntidAttrName()) {
714 if (!isa<DenseI32ArrayAttr>(Val: attribute.getValue()))
715 return failure();
716 auto values = cast<DenseI32ArrayAttr>(Val: attribute.getValue());
717 const std::string attr = llvm::formatv(
718 Fmt: "{0:$[,]}", Vals: llvm::make_range(x: values.asArrayRef().begin(),
719 y: values.asArrayRef().end()));
720 llvmFunc->addFnAttr(Kind: llvm::NVVMAttr::ReqNTID, Val: attr);
721 } else if (attribute.getName() ==
722 NVVM::NVVMDialect::getClusterDimAttrName()) {
723 if (!isa<DenseI32ArrayAttr>(Val: attribute.getValue()))
724 return failure();
725 auto values = cast<DenseI32ArrayAttr>(Val: attribute.getValue());
726 const std::string attr = llvm::formatv(
727 Fmt: "{0:$[,]}", Vals: llvm::make_range(x: values.asArrayRef().begin(),
728 y: values.asArrayRef().end()));
729 llvmFunc->addFnAttr(Kind: llvm::NVVMAttr::ClusterDim, Val: attr);
730 } else if (attribute.getName() ==
731 NVVM::NVVMDialect::getClusterMaxBlocksAttrName()) {
732 auto value = dyn_cast<IntegerAttr>(Val: attribute.getValue());
733 llvmFunc->addFnAttr(Kind: llvm::NVVMAttr::MaxClusterRank,
734 Val: llvm::utostr(X: value.getInt()));
735 } else if (attribute.getName() ==
736 NVVM::NVVMDialect::getMinctasmAttrName()) {
737 auto value = dyn_cast<IntegerAttr>(Val: attribute.getValue());
738 llvmFunc->addFnAttr(Kind: llvm::NVVMAttr::MinCTASm,
739 Val: llvm::utostr(X: value.getInt()));
740 } else if (attribute.getName() == NVVM::NVVMDialect::getMaxnregAttrName()) {
741 auto value = dyn_cast<IntegerAttr>(Val: attribute.getValue());
742 llvmFunc->addFnAttr(Kind: llvm::NVVMAttr::MaxNReg,
743 Val: llvm::utostr(X: value.getInt()));
744 } else if (attribute.getName() ==
745 NVVM::NVVMDialect::getKernelFuncAttrName()) {
746 llvmFunc->setCallingConv(llvm::CallingConv::PTX_Kernel);
747 } else if (attribute.getName() ==
748 NVVM::NVVMDialect::getBlocksAreClustersAttrName()) {
749 llvmFunc->addFnAttr(Kind: llvm::NVVMAttr::BlocksAreClusters);
750 }
751
752 return success();
753 }
754
755 LogicalResult
756 convertParameterAttr(LLVMFuncOp funcOp, int argIdx, NamedAttribute attribute,
757 LLVM::ModuleTranslation &moduleTranslation) const final {
758
759 llvm::LLVMContext &llvmContext = moduleTranslation.getLLVMContext();
760 llvm::Function *llvmFunc =
761 moduleTranslation.lookupFunction(name: funcOp.getName());
762
763 if (attribute.getName() == NVVM::NVVMDialect::getGridConstantAttrName()) {
764 llvmFunc->addParamAttr(
765 ArgNo: argIdx,
766 Attr: llvm::Attribute::get(Context&: llvmContext, Kind: llvm::NVVMAttr::GridConstant));
767 }
768 return success();
769 }
770};
771} // namespace
772
773void mlir::registerNVVMDialectTranslation(DialectRegistry &registry) {
774 registry.insert<NVVM::NVVMDialect>();
775 registry.addExtension(extensionFn: +[](MLIRContext *ctx, NVVM::NVVMDialect *dialect) {
776 dialect->addInterfaces<NVVMDialectLLVMIRTranslationInterface>();
777 });
778}
779
780void mlir::registerNVVMDialectTranslation(MLIRContext &context) {
781 DialectRegistry registry;
782 registerNVVMDialectTranslation(registry);
783 context.appendDialectRegistry(registry);
784}
785