1//===- LowerGpuOpsToNVVMOps.cpp - MLIR GPU to NVVM lowering passes --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a pass to generate NVVMIR operations for higher-level
10// GPU operations.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
15#include "mlir/Conversion/ConvertToLLVM/ToLLVMPass.h"
16#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
17#include "mlir/Conversion/GPUToNVVM/GPUToNVVM.h"
18#include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h"
19#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
20#include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
21#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
22#include "mlir/Conversion/MathToNVVM/MathToNVVM.h"
23#include "mlir/Conversion/NVGPUToNVVM/NVGPUToNVVM.h"
24#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
25#include "mlir/Dialect/Func/IR/FuncOps.h"
26#include "mlir/Dialect/GPU/IR/GPUDialect.h"
27#include "mlir/Dialect/GPU/Transforms/Passes.h"
28#include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h"
29#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
30#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
31#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
32#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
33#include "mlir/Dialect/LLVMIR/LLVMFloatIntrinsicOps.h"
34#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
35#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
36#include "mlir/Dialect/LLVMIR/NVVMDialect.h"
37#include "mlir/Dialect/Math/IR/Math.h"
38#include "mlir/Dialect/MemRef/IR/MemRef.h"
39#include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
40#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
41#include "mlir/IR/SymbolTable.h"
42#include "mlir/Interfaces/FunctionInterfaces.h"
43#include "mlir/Transforms/DialectConversion.h"
44#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
45
46#include "../GPUCommon/GPUOpsLowering.h"
47#include "../GPUCommon/IndexIntrinsicsOpLowering.h"
48#include "../GPUCommon/OpToFuncCallLowering.h"
49#include <optional>
50
51namespace mlir {
52#define GEN_PASS_DEF_CONVERTGPUOPSTONVVMOPS
53#include "mlir/Conversion/Passes.h.inc"
54} // namespace mlir
55
56using namespace mlir;
57
58namespace {
59
60/// Convert gpu dialect shfl mode enum to the equivalent nvvm one.
61static NVVM::ShflKind convertShflKind(gpu::ShuffleMode mode) {
62 switch (mode) {
63 case gpu::ShuffleMode::XOR:
64 return NVVM::ShflKind::bfly;
65 case gpu::ShuffleMode::UP:
66 return NVVM::ShflKind::up;
67 case gpu::ShuffleMode::DOWN:
68 return NVVM::ShflKind::down;
69 case gpu::ShuffleMode::IDX:
70 return NVVM::ShflKind::idx;
71 }
72 llvm_unreachable("unknown shuffle mode");
73}
74
75static std::optional<NVVM::ReductionKind>
76convertToNVVMReductionKind(gpu::AllReduceOperation mode) {
77 switch (mode) {
78 case gpu::AllReduceOperation::ADD:
79 return NVVM::ReductionKind::ADD;
80 case gpu::AllReduceOperation::MUL:
81 return std::nullopt;
82 case gpu::AllReduceOperation::MINSI:
83 return NVVM::ReductionKind::MIN;
84 case gpu::AllReduceOperation::MINUI:
85 return std::nullopt;
86 case gpu::AllReduceOperation::MINNUMF:
87 return NVVM::ReductionKind::MIN;
88 case gpu::AllReduceOperation::MAXSI:
89 return NVVM::ReductionKind::MAX;
90 case gpu::AllReduceOperation::MAXUI:
91 return std::nullopt;
92 case gpu::AllReduceOperation::MAXNUMF:
93 return NVVM::ReductionKind::MAX;
94 case gpu::AllReduceOperation::AND:
95 return NVVM::ReductionKind::AND;
96 case gpu::AllReduceOperation::OR:
97 return NVVM::ReductionKind::OR;
98 case gpu::AllReduceOperation::XOR:
99 return NVVM::ReductionKind::XOR;
100 case gpu::AllReduceOperation::MINIMUMF:
101 case gpu::AllReduceOperation::MAXIMUMF:
102 case gpu::AllReduceOperation::MINIMUMNUMF:
103 case gpu::AllReduceOperation::MAXIMUMNUMF:
104 return std::nullopt;
105 }
106 return std::nullopt;
107}
108
109static constexpr llvm::StringLiteral kNVVMNamedBarrierIdPrefix =
110 "__named_barrier_id";
111static constexpr int32_t kNVVMFirstNamedBarrierId = 1;
112static constexpr int32_t kNVVMLastNamedBarrierId = 15;
113static constexpr int32_t kNVVMWarpSize = 32;
114
115static FailureOr<StringAttr>
116createNVVMNamedBarrierIdGlobal(gpu::InitializeNamedBarrierOp op,
117 ConversionPatternRewriter &rewriter) {
118 auto funcOp = op->getParentOfType<FunctionOpInterface>();
119 if (!funcOp) {
120 op.emitOpError(message: "must be inside a function-like op");
121 return failure();
122 }
123 Operation *symbolTableOp = funcOp->getParentWithTrait<OpTrait::SymbolTable>();
124 if (!symbolTableOp) {
125 op.emitOpError(
126 message: "enclosing function-like op must have a symbol-table parent");
127 return failure();
128 }
129
130 int32_t numNamedBarriers = 0;
131 for (auto globalOp :
132 symbolTableOp->getRegion(index: 0).front().getOps<LLVM::GlobalOp>())
133 if (globalOp.getSymName().starts_with(Prefix: kNVVMNamedBarrierIdPrefix))
134 ++numNamedBarriers;
135
136 int32_t barrierId = kNVVMFirstNamedBarrierId + numNamedBarriers;
137 if (barrierId > kNVVMLastNamedBarrierId) {
138 op.emitOpError(message: "NVVM supports at most 15 named barriers per CTA");
139 return failure();
140 }
141
142 OpBuilder detachedBuilder(rewriter.getContext());
143 Type i32 = rewriter.getI32Type();
144 auto globalOp = LLVM::GlobalOp::create(
145 builder&: detachedBuilder, location: op.getLoc(), type: i32, /*isConstant=*/true,
146 linkage: LLVM::Linkage::Internal, name: kNVVMNamedBarrierIdPrefix,
147 value: rewriter.getI32IntegerAttr(value: barrierId), /*alignment=*/0,
148 /*addrSpace=*/0);
149 return SymbolTable(symbolTableOp).insert(symbol: globalOp);
150}
151
152/// This pass lowers gpu.subgroup_reduce op into to the nvvm.redux op. The op
153/// must be run by the entire subgroup, otherwise it is undefined behaviour.
154struct GPUSubgroupReduceOpLowering
155 : public ConvertOpToLLVMPattern<gpu::SubgroupReduceOp> {
156 using ConvertOpToLLVMPattern<gpu::SubgroupReduceOp>::ConvertOpToLLVMPattern;
157 LogicalResult
158
159 matchAndRewrite(gpu::SubgroupReduceOp op, OpAdaptor adaptor,
160 ConversionPatternRewriter &rewriter) const override {
161 if (op.getClusterSize())
162 return rewriter.notifyMatchFailure(
163 arg&: op, msg: "lowering for clustered reduce not implemented");
164
165 if (!op.getUniform())
166 return rewriter.notifyMatchFailure(
167 arg&: op, msg: "cannot be lowered to redux as the op must be run "
168 "uniformly (entire subgroup).");
169 if (!op.getValue().getType().isInteger(width: 32))
170 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported data type");
171
172 std::optional<NVVM::ReductionKind> mode =
173 convertToNVVMReductionKind(mode: op.getOp());
174 if (!mode.has_value())
175 return rewriter.notifyMatchFailure(
176 arg&: op, msg: "unsupported reduction mode for redux");
177
178 Location loc = op->getLoc();
179 auto int32Type = IntegerType::get(context: rewriter.getContext(), width: 32);
180 Value offset = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: int32Type, value: -1);
181
182 auto reduxOp = NVVM::ReduxOp::create(builder&: rewriter, location: loc, res: int32Type,
183 val: op.getValue(), kind: mode.value(), mask_and_clamp: offset);
184
185 rewriter.replaceOp(op, newValues: reduxOp->getResult(idx: 0));
186 return success();
187 }
188};
189
190struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
191 using ConvertOpToLLVMPattern<gpu::ShuffleOp>::ConvertOpToLLVMPattern;
192
193 /// Lowers a shuffle to the corresponding NVVM op.
194 ///
195 /// Convert the `width` argument into an activeMask (a bitmask which specifies
196 /// which threads participate in the shuffle) and a maskAndClamp (specifying
197 /// the highest lane which participates in the shuffle).
198 ///
199 /// %one = llvm.constant(1 : i32) : i32
200 /// %minus_one = llvm.constant(-1 : i32) : i32
201 /// %thirty_two = llvm.constant(32 : i32) : i32
202 /// %num_lanes = llvm.sub %thirty_two, %width : i32
203 /// %active_mask = llvm.lshr %minus_one, %num_lanes : i32
204 /// %mask_and_clamp = llvm.sub %width, %one : i32
205 /// %shfl = nvvm.shfl.sync.bfly %active_mask, %value, %offset,
206 /// %mask_and_clamp : !llvm<"{ float, i1 }">
207 /// %shfl_value = llvm.extractvalue %shfl[0] :
208 /// !llvm<"{ float, i1 }">
209 /// %shfl_pred = llvm.extractvalue %shfl[1] :
210 /// !llvm<"{ float, i1 }">
211 LogicalResult
212 matchAndRewrite(gpu::ShuffleOp op, OpAdaptor adaptor,
213 ConversionPatternRewriter &rewriter) const override {
214 Location loc = op->getLoc();
215
216 auto valueTy = adaptor.getValue().getType();
217 auto int32Type = IntegerType::get(context: rewriter.getContext(), width: 32);
218 auto predTy = IntegerType::get(context: rewriter.getContext(), width: 1);
219
220 Value one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: int32Type, value: 1);
221 Value minusOne = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: int32Type, value: -1);
222 Value thirtyTwo = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: int32Type, value: 32);
223 Value numLeadInactiveLane = LLVM::SubOp::create(
224 builder&: rewriter, location: loc, res: int32Type, lhs: thirtyTwo, rhs: adaptor.getWidth());
225 // Bit mask of active lanes: `(-1) >> (32 - activeWidth)`.
226 Value activeMask = LLVM::LShrOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: minusOne,
227 rhs: numLeadInactiveLane);
228 Value maskAndClamp;
229 if (op.getMode() == gpu::ShuffleMode::UP) {
230 // Clamp lane: `32 - activeWidth`
231 maskAndClamp = numLeadInactiveLane;
232 } else {
233 // Clamp lane: `activeWidth - 1`
234 maskAndClamp = LLVM::SubOp::create(builder&: rewriter, location: loc, res: int32Type,
235 lhs: adaptor.getWidth(), rhs: one);
236 }
237
238 bool predIsUsed = !op->getResult(idx: 1).use_empty();
239 UnitAttr returnValueAndIsValidAttr = nullptr;
240 Type resultTy = valueTy;
241 if (predIsUsed) {
242 returnValueAndIsValidAttr = rewriter.getUnitAttr();
243 resultTy = LLVM::LLVMStructType::getLiteral(context: rewriter.getContext(),
244 types: {valueTy, predTy});
245 }
246 Value shfl = NVVM::ShflOp::create(
247 builder&: rewriter, location: loc, res: resultTy, thread_mask: activeMask, val: adaptor.getValue(),
248 offset: adaptor.getOffset(), mask_and_clamp: maskAndClamp, kind: convertShflKind(mode: op.getMode()),
249 return_value_and_is_valid: returnValueAndIsValidAttr);
250 if (predIsUsed) {
251 Value shflValue = LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: shfl, position: 0);
252 Value isActiveSrcLane =
253 LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: shfl, position: 1);
254 rewriter.replaceOp(op, newValues: {shflValue, isActiveSrcLane});
255 } else {
256 rewriter.replaceOp(op, newValues: {shfl, nullptr});
257 }
258 return success();
259 }
260};
261
262struct GPULaneIdOpToNVVM : ConvertOpToLLVMPattern<gpu::LaneIdOp> {
263 using ConvertOpToLLVMPattern<gpu::LaneIdOp>::ConvertOpToLLVMPattern;
264
265 LogicalResult
266 matchAndRewrite(gpu::LaneIdOp op, gpu::LaneIdOp::Adaptor adaptor,
267 ConversionPatternRewriter &rewriter) const override {
268 auto loc = op->getLoc();
269 MLIRContext *context = rewriter.getContext();
270 LLVM::ConstantRangeAttr bounds = nullptr;
271 if (std::optional<APInt> upperBound = op.getUpperBound())
272 bounds = rewriter.getAttr<LLVM::ConstantRangeAttr>(
273 /*bitWidth=*/args: 32, /*lower=*/args: 0, args: upperBound->getZExtValue());
274 else
275 bounds = rewriter.getAttr<LLVM::ConstantRangeAttr>(
276 /*bitWidth=*/args: 32, /*lower=*/args: 0, /*upper=*/args: kWarpSize);
277 Value newOp =
278 NVVM::LaneIdOp::create(builder&: rewriter, location: loc, res: rewriter.getI32Type(), range: bounds);
279 // Truncate or extend the result depending on the index bitwidth specified
280 // by the LLVMTypeConverter options.
281 const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
282 if (indexBitwidth > 32) {
283 newOp = LLVM::SExtOp::create(
284 builder&: rewriter, location: loc, res: IntegerType::get(context, width: indexBitwidth), arg: newOp);
285 } else if (indexBitwidth < 32) {
286 newOp = LLVM::TruncOp::create(
287 builder&: rewriter, location: loc, res: IntegerType::get(context, width: indexBitwidth), arg: newOp);
288 }
289 rewriter.replaceOp(op, newValues: {newOp});
290 return success();
291 }
292};
293
294struct GPUBallotOpToNVVM : public ConvertOpToLLVMPattern<gpu::BallotOp> {
295 using ConvertOpToLLVMPattern<gpu::BallotOp>::ConvertOpToLLVMPattern;
296
297 LogicalResult
298 matchAndRewrite(gpu::BallotOp op, gpu::BallotOp::Adaptor adaptor,
299 ConversionPatternRewriter &rewriter) const override {
300 Location loc = op->getLoc();
301 auto int32Type = IntegerType::get(context: rewriter.getContext(), width: 32);
302 auto intType = cast<IntegerType>(Val: op.getType());
303 unsigned width = intType.getWidth();
304
305 // NVVM ballot natively returns i32. For i64 results, zero-extend since
306 // NVIDIA warps have exactly 32 threads, so upper 32 bits are always zero.
307 if (width != 32 && width != 64)
308 return rewriter.notifyMatchFailure(
309 arg&: op, msg: "nvvm.vote.sync ballot only supports i32 and i64 result types");
310
311 // Use full mask (-1) so all 32 lanes participate in the ballot.
312 Value mask = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: int32Type,
313 value: rewriter.getI32IntegerAttr(value: -1));
314
315 auto voteKind = NVVM::VoteSyncKindAttr::get(context: rewriter.getContext(),
316 value: NVVM::VoteSyncKind::ballot);
317 Value result = NVVM::VoteSyncOp::create(builder&: rewriter, location: loc, res: int32Type, mask,
318 pred: adaptor.getPredicate(), kind: voteKind);
319
320 if (width == 64)
321 result = LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: op.getType(), arg: result);
322
323 rewriter.replaceOp(op, newValues: result);
324 return success();
325 }
326};
327
328/// Lowering of cf.assert into a conditional __assertfail.
329struct AssertOpToAssertfailLowering
330 : public ConvertOpToLLVMPattern<cf::AssertOp> {
331 using ConvertOpToLLVMPattern<cf::AssertOp>::ConvertOpToLLVMPattern;
332
333 LogicalResult
334 matchAndRewrite(cf::AssertOp assertOp, cf::AssertOpAdaptor adaptor,
335 ConversionPatternRewriter &rewriter) const override {
336 MLIRContext *ctx = rewriter.getContext();
337 Location loc = assertOp.getLoc();
338 Type i8Type = typeConverter->convertType(t: rewriter.getIntegerType(width: 8));
339 Type i32Type = typeConverter->convertType(t: rewriter.getIntegerType(width: 32));
340 Type i64Type = typeConverter->convertType(t: rewriter.getIntegerType(width: 64));
341 Type ptrType = LLVM::LLVMPointerType::get(context: ctx);
342 Type voidType = LLVM::LLVMVoidType::get(ctx);
343
344 // Find or create __assertfail function declaration.
345 auto moduleOp = assertOp->getParentOfType<gpu::GPUModuleOp>();
346 auto assertfailType = LLVM::LLVMFunctionType::get(
347 result: voidType, arguments: {ptrType, ptrType, i32Type, ptrType, i64Type});
348 LLVM::LLVMFuncOp assertfailDecl = getOrDefineFunction(
349 moduleOp, loc, b&: rewriter, name: "__assertfail", type: assertfailType);
350 assertfailDecl.setPassthroughAttr(
351 ArrayAttr::get(context: ctx, value: StringAttr::get(context: ctx, bytes: "noreturn")));
352
353 // Split blocks and insert conditional branch.
354 // ^before:
355 // ...
356 // cf.cond_br %condition, ^after, ^assert
357 // ^assert:
358 // cf.assert
359 // cf.br ^after
360 // ^after:
361 // ...
362 Block *beforeBlock = assertOp->getBlock();
363 Block *assertBlock =
364 rewriter.splitBlock(block: beforeBlock, before: assertOp->getIterator());
365 Block *afterBlock =
366 rewriter.splitBlock(block: assertBlock, before: ++assertOp->getIterator());
367 rewriter.setInsertionPointToEnd(beforeBlock);
368 cf::CondBranchOp::create(builder&: rewriter, location: loc, condition: adaptor.getArg(), trueDest: afterBlock,
369 falseDest: assertBlock);
370 rewriter.setInsertionPointToEnd(assertBlock);
371 cf::BranchOp::create(builder&: rewriter, location: loc, dest: afterBlock);
372
373 // Continue cf.assert lowering.
374 rewriter.setInsertionPoint(assertOp);
375
376 // Populate file name, file number and function name from the location of
377 // the AssertOp.
378 StringRef fileName = "(unknown)";
379 StringRef funcName = "(unknown)";
380 int32_t fileLine = 0;
381 while (auto callSiteLoc = dyn_cast<CallSiteLoc>(Val&: loc))
382 loc = callSiteLoc.getCallee();
383 if (auto fileLineColLoc = dyn_cast<FileLineColRange>(Val&: loc)) {
384 fileName = fileLineColLoc.getFilename().strref();
385 fileLine = fileLineColLoc.getStartLine();
386 } else if (auto nameLoc = dyn_cast<NameLoc>(Val&: loc)) {
387 funcName = nameLoc.getName().strref();
388 if (auto fileLineColLoc =
389 dyn_cast<FileLineColRange>(Val: nameLoc.getChildLoc())) {
390 fileName = fileLineColLoc.getFilename().strref();
391 fileLine = fileLineColLoc.getStartLine();
392 }
393 }
394
395 // Create constants.
396 auto getGlobal = [&](LLVM::GlobalOp global) {
397 // Get a pointer to the format string's first element.
398 Value globalPtr = LLVM::AddressOfOp::create(
399 builder&: rewriter, location: loc, res: LLVM::LLVMPointerType::get(context: ctx, addressSpace: global.getAddrSpace()),
400 global_name: global.getSymNameAttr());
401 Value start =
402 LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: global.getGlobalType(),
403 basePtr: globalPtr, indices: ArrayRef<LLVM::GEPArg>{0, 0});
404 return start;
405 };
406 Value assertMessage = getGlobal(getOrCreateStringConstant(
407 b&: rewriter, loc, moduleOp, llvmI8: i8Type, namePrefix: "assert_message_", str: assertOp.getMsg()));
408 Value assertFile = getGlobal(getOrCreateStringConstant(
409 b&: rewriter, loc, moduleOp, llvmI8: i8Type, namePrefix: "assert_file_", str: fileName));
410 Value assertFunc = getGlobal(getOrCreateStringConstant(
411 b&: rewriter, loc, moduleOp, llvmI8: i8Type, namePrefix: "assert_func_", str: funcName));
412 Value assertLine =
413 LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Type, value: fileLine);
414 Value c1 = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i64Type, value: 1);
415
416 // Insert function call to __assertfail.
417 SmallVector<Value> arguments{assertMessage, assertFile, assertLine,
418 assertFunc, c1};
419 rewriter.replaceOpWithNewOp<LLVM::CallOp>(op: assertOp, args&: assertfailDecl,
420 args&: arguments);
421 return success();
422 }
423};
424
425struct GPUBarrierOpToNVVMLowering final
426 : public ConvertOpToLLVMPattern<gpu::BarrierOp> {
427 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
428
429 LogicalResult
430 matchAndRewrite(gpu::BarrierOp op, gpu::BarrierOp::Adaptor adaptor,
431 ConversionPatternRewriter &rewriter) const override {
432 if (Value namedBarrier = adaptor.getNamedBarrier()) {
433 Location loc = op.getLoc();
434 Value barrierId =
435 LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: namedBarrier, position: 0);
436 Value numberOfThreads =
437 LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: namedBarrier, position: 1);
438 NVVM::BarrierOp::create(builder&: rewriter, location: loc, barrierId, numberOfThreads);
439 rewriter.eraseOp(op);
440 return success();
441 }
442
443 gpu::BarrierScope scope = op.getScope();
444 switch (scope) {
445 case gpu::BarrierScope::Workgroup:
446 rewriter.replaceOpWithNewOp<NVVM::BarrierOp>(op);
447 return success();
448 case gpu::BarrierScope::Subgroup: {
449 // Emit __syncwarp(0xFFFFFFFF) for full-warp sync.
450 Value mask =
451 LLVM::ConstantOp::create(builder&: rewriter, location: op.getLoc(), res: rewriter.getI32Type(),
452 value: rewriter.getI32IntegerAttr(value: 0xFFFFFFFF));
453 rewriter.replaceOpWithNewOp<NVVM::SyncWarpOp>(op, args&: mask);
454 return success();
455 }
456 case gpu::BarrierScope::Cluster: {
457 UnitAttr aligned = rewriter.getUnitAttr();
458 if (op.getAddressSpaces() && op.getAddressSpaces()->empty())
459 NVVM::ClusterArriveRelaxedOp::create(builder&: rewriter, location: op.getLoc(), aligned);
460 else
461 NVVM::ClusterArriveOp::create(builder&: rewriter, location: op.getLoc(), aligned);
462 rewriter.replaceOpWithNewOp<NVVM::ClusterWaitOp>(op, args&: aligned);
463 return success();
464 }
465 }
466 llvm_unreachable("unknown barrier scope");
467 }
468};
469
470struct GPUInitializeNamedBarrierOpToNVVMLowering final
471 : public ConvertOpToLLVMPattern<gpu::InitializeNamedBarrierOp> {
472 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
473
474 LogicalResult
475 matchAndRewrite(gpu::InitializeNamedBarrierOp op,
476 gpu::InitializeNamedBarrierOp::Adaptor adaptor,
477 ConversionPatternRewriter &rewriter) const override {
478 Location loc = op.getLoc();
479 MLIRContext *ctx = rewriter.getContext();
480 Type i32 = rewriter.getI32Type();
481 Type namedBarrierType =
482 getTypeConverter()->convertType(t: op.getResult().getType());
483 if (!namedBarrierType)
484 return rewriter.notifyMatchFailure(arg&: op, msg: "failed to convert result type");
485
486 FailureOr<StringAttr> maybeGlobalName =
487 createNVVMNamedBarrierIdGlobal(op, rewriter);
488 if (failed(Result: maybeGlobalName))
489 return failure();
490
491 auto addressOf = LLVM::AddressOfOp::create(
492 builder&: rewriter, location: loc, res: LLVM::LLVMPointerType::get(context: ctx), global_name: *maybeGlobalName);
493 Value barrierId =
494 LLVM::LoadOp::create(builder&: rewriter, location: loc, type: i32, addr: addressOf.getResult());
495
496 Value warpSize = LLVM::ConstantOp::create(
497 builder&: rewriter, location: loc, res: i32, value: rewriter.getI32IntegerAttr(value: kNVVMWarpSize));
498 Value numberOfThreads =
499 LLVM::MulOp::create(builder&: rewriter, location: loc, lhs: adaptor.getMemberCount(), rhs: warpSize);
500
501 Value namedBarrier =
502 LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: namedBarrierType);
503 DenseI64ArrayAttr barrierIdPos = rewriter.getDenseI64ArrayAttr(values: {0});
504 DenseI64ArrayAttr numberOfThreadsPos = rewriter.getDenseI64ArrayAttr(values: {1});
505 namedBarrier = LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: namedBarrier,
506 value: barrierId, position: barrierIdPos);
507 namedBarrier = LLVM::InsertValueOp::create(
508 builder&: rewriter, location: loc, container: namedBarrier, value: numberOfThreads, position: numberOfThreadsPos);
509 rewriter.replaceOp(op, newValues: namedBarrier);
510 return success();
511 }
512};
513
514/// A pass that replaces all occurrences of GPU device operations with their
515/// corresponding NVVM equivalent.
516///
517/// This pass only handles device code and is not meant to be run on GPU host
518/// code.
519struct LowerGpuOpsToNVVMOpsPass final
520 : public impl::ConvertGpuOpsToNVVMOpsBase<LowerGpuOpsToNVVMOpsPass> {
521 using Base::Base;
522
523 void getDependentDialects(DialectRegistry &registry) const override {
524 Base::getDependentDialects(registry);
525 registerConvertToLLVMDependentDialectLoading(registry);
526 }
527
528 void runOnOperation() override {
529 gpu::GPUModuleOp m = getOperation();
530
531 // Request C wrapper emission for externally visible functions only. A
532 // private function cannot be called from outside its module, so its C
533 // interface wrapper is unreachable and would only anchor the wrapped
534 // function against dead-code elimination.
535 for (auto func : m.getOps<func::FuncOp>()) {
536 if (func.isPrivate())
537 continue;
538 func->setDiscardableAttr(name: LLVM::LLVMDialect::getEmitCWrapperAttrName(),
539 value: UnitAttr::get(context: &getContext()));
540 }
541
542 // Customize the bitwidth used for the device side index computations.
543 LowerToLLVMOptions options(
544 m.getContext(),
545 DataLayout(cast<DataLayoutOpInterface>(Val: m.getOperation())));
546 if (indexBitwidth != kDeriveIndexBitwidthFromDataLayout)
547 options.overrideIndexBitwidth(bitwidth: indexBitwidth);
548 options.useBarePtrCallConv = useBarePtrCallConv;
549
550 // Apply in-dialect lowering. In-dialect lowering will replace
551 // ops which need to be lowered further, which is not supported by a
552 // single conversion pass.
553 {
554 RewritePatternSet patterns(m.getContext());
555 populateGpuRewritePatterns(patterns);
556 // Transform N-D vector.from_elements to 1-D vector.from_elements before
557 // conversion.
558 vector::populateVectorFromElementsUnrollPatterns(patterns);
559 if (failed(Result: applyPatternsGreedily(op: m, patterns: std::move(patterns))))
560 return signalPassFailure();
561 }
562
563 LLVMTypeConverter converter(m.getContext(), options);
564 configureGpuToNVVMTypeConverter(converter);
565 RewritePatternSet llvmPatterns(m.getContext());
566 LLVMConversionTarget target(getContext());
567
568 // Set higher benefit, so patterns will run before generic LLVM lowering.
569 populateGpuToNVVMConversionPatterns(converter, patterns&: llvmPatterns,
570 /*benefit=*/10);
571
572 llvm::SmallDenseSet<StringRef> allowedDialectsSet(allowedDialects.begin(),
573 allowedDialects.end());
574 for (Dialect *dialect : getContext().getLoadedDialects()) {
575 // Skip math patterns as nvvm needs custom math lowering.
576 if (isa<math::MathDialect>(Val: dialect))
577 continue;
578
579 bool allowed = allowedDialectsSet.contains(V: dialect->getNamespace());
580 // Empty `allowedDialectsSet` means all dialects are allowed.
581 if (!allowedDialectsSet.empty() && !allowed)
582 continue;
583
584 auto *iface = dyn_cast<ConvertToLLVMPatternInterface>(Val: dialect);
585 if (!iface) {
586 // Error out if dialect was explicily specified but doesn't implement
587 // conversion interface.
588 if (allowed) {
589 m.emitError()
590 << "dialect does not implement ConvertToLLVMPatternInterface: "
591 << dialect->getNamespace();
592 return signalPassFailure();
593 }
594 continue;
595 }
596
597 iface->populateConvertToLLVMConversionPatterns(target, typeConverter&: converter,
598 patterns&: llvmPatterns);
599 }
600
601 populateGpuWMMAToNVVMConversionPatterns(converter, patterns&: llvmPatterns);
602 if (this->hasRedux)
603 populateGpuSubgroupReduceOpLoweringPattern(converter, patterns&: llvmPatterns);
604 configureGpuToNVVMConversionLegality(target);
605 ConversionConfig config;
606 config.allowPatternRollback = allowPatternRollback;
607 if (failed(
608 Result: applyPartialConversion(op: m, target, patterns: std::move(llvmPatterns), config)))
609 signalPassFailure();
610 }
611};
612
613} // namespace
614
615void mlir::configureGpuToNVVMConversionLegality(ConversionTarget &target) {
616 target.addIllegalOp<func::FuncOp>();
617 target.addIllegalOp<cf::AssertOp>();
618 target.addLegalDialect<::mlir::LLVM::LLVMDialect>();
619 target.addLegalDialect<::mlir::NVVM::NVVMDialect>();
620 target.addIllegalDialect<gpu::GPUDialect>();
621 target.addIllegalOp<LLVM::CopySignOp, LLVM::CosOp, LLVM::ExpOp, LLVM::Exp2Op,
622 LLVM::FAbsOp, LLVM::FCeilOp, LLVM::FFloorOp, LLVM::FRemOp,
623 LLVM::LogOp, LLVM::Log10Op, LLVM::Log2Op, LLVM::PowOp,
624 LLVM::RoundEvenOp, LLVM::RoundOp, LLVM::SinOp,
625 LLVM::SincosOp, LLVM::SqrtOp>();
626
627 // TODO: Remove once we support replacing non-root ops.
628 target.addLegalOp<gpu::YieldOp, gpu::GPUModuleOp>();
629}
630
631void mlir::configureGpuToNVVMTypeConverter(LLVMTypeConverter &converter) {
632 nvgpu::populateCommonGPUTypeAndAttributeConversions(typeConverter&: converter);
633
634 converter.addConversion(callback: [&](gpu::NamedBarrierType type) -> Type {
635 Type i32 = IntegerType::get(context: type.getContext(), width: 32);
636 return LLVM::LLVMStructType::getLiteral(context: type.getContext(), types: {i32, i32});
637 });
638
639 // Lowering for MMAMatrixType.
640 converter.addConversion(callback: [&](gpu::MMAMatrixType type) -> Type {
641 return convertMMAToLLVMType(type);
642 });
643}
644
645void mlir::populateGpuSubgroupReduceOpLoweringPattern(
646 const LLVMTypeConverter &converter, RewritePatternSet &patterns,
647 PatternBenefit benefit) {
648 patterns.add<GPUSubgroupReduceOpLowering>(arg: converter, args&: benefit);
649}
650
651void mlir::populateGpuToNVVMConversionPatterns(
652 const LLVMTypeConverter &converter, RewritePatternSet &patterns,
653 PatternBenefit benefit) {
654 using gpu::index_lowering::IndexKind;
655 using gpu::index_lowering::IntrType;
656
657 patterns.add<GPUBarrierOpToNVVMLowering,
658 GPUInitializeNamedBarrierOpToNVVMLowering,
659 GPUPrintfOpToVPrintfLowering, AssertOpToAssertfailLowering>(
660 arg: converter, args&: benefit);
661 patterns.add<
662 gpu::index_lowering::OpLowering<gpu::ThreadIdOp, NVVM::ThreadIdXOp,
663 NVVM::ThreadIdYOp, NVVM::ThreadIdZOp>>(
664 arg: converter, args: IndexKind::Block, args: IntrType::Id, args&: benefit);
665 patterns.add<
666 gpu::index_lowering::OpLowering<gpu::BlockDimOp, NVVM::BlockDimXOp,
667 NVVM::BlockDimYOp, NVVM::BlockDimZOp>>(
668 arg: converter, args: IndexKind::Block, args: IntrType::Dim, args&: benefit);
669 patterns.add<
670 gpu::index_lowering::OpLowering<gpu::ClusterIdOp, NVVM::ClusterIdXOp,
671 NVVM::ClusterIdYOp, NVVM::ClusterIdZOp>>(
672 arg: converter, args: IndexKind::Other, args: IntrType::Id, args&: benefit);
673 patterns.add<gpu::index_lowering::OpLowering<
674 gpu::ClusterDimOp, NVVM::ClusterDimXOp, NVVM::ClusterDimYOp,
675 NVVM::ClusterDimZOp>>(arg: converter, args: IndexKind::Other, args: IntrType::Dim,
676 args&: benefit);
677 patterns.add<gpu::index_lowering::OpLowering<
678 gpu::ClusterBlockIdOp, NVVM::BlockInClusterIdXOp,
679 NVVM::BlockInClusterIdYOp, NVVM::BlockInClusterIdZOp>>(
680 arg: converter, args: IndexKind::Cluster, args: IntrType::Id, args&: benefit);
681 patterns.add<gpu::index_lowering::OpLowering<
682 gpu::ClusterDimBlocksOp, NVVM::ClusterDimBlocksXOp,
683 NVVM::ClusterDimBlocksYOp, NVVM::ClusterDimBlocksZOp>>(
684 arg: converter, args: IndexKind::Cluster, args: IntrType::Dim, args&: benefit);
685 patterns.add<gpu::index_lowering::OpLowering<
686 gpu::BlockIdOp, NVVM::BlockIdXOp, NVVM::BlockIdYOp, NVVM::BlockIdZOp>>(
687 arg: converter, args: IndexKind::Grid, args: IntrType::Id, args&: benefit);
688 patterns.add<gpu::index_lowering::OpLowering<
689 gpu::GridDimOp, NVVM::GridDimXOp, NVVM::GridDimYOp, NVVM::GridDimZOp>>(
690 arg: converter, args: IndexKind::Grid, args: IntrType::Dim, args&: benefit);
691 patterns.add<GPULaneIdOpToNVVM, GPUBallotOpToNVVM, GPUShuffleOpLowering,
692 GPUReturnOpLowering>(arg: converter, args&: benefit);
693
694 patterns.add<GPUDynamicSharedMemoryOpLowering>(
695 arg: converter, args: NVVM::kSharedMemoryAlignmentBit, args&: benefit);
696
697 // Explicitly drop memory space when lowering private memory
698 // attributions since NVVM models it as `alloca`s in the default
699 // memory space and does not support `alloca`s with addrspace(5).
700 patterns.add<GPUFuncOpLowering>(
701 arg: converter,
702 args: GPUFuncOpLoweringOptions{
703 /*allocaAddrSpace=*/0,
704 /*workgroupAddrSpace=*/
705 static_cast<unsigned>(NVVM::NVVMMemorySpace::Shared),
706 .kernelAttributeName: StringAttr::get(context: &converter.getContext(),
707 bytes: NVVM::NVVMDialect::getKernelFuncAttrName()),
708 .kernelBlockSizeAttributeName: StringAttr::get(context: &converter.getContext(),
709 bytes: NVVM::NVVMDialect::getMaxntidAttrName()),
710 .kernelClusterSizeAttributeName: StringAttr::get(context: &converter.getContext(),
711 bytes: NVVM::NVVMDialect::getClusterDimAttrName())},
712 args&: benefit);
713
714 populateLibDeviceConversionPatterns(converter, patterns, benefit);
715}
716
717//===----------------------------------------------------------------------===//
718// NVVMTargetAttr convert to LLVM attr interface
719//===----------------------------------------------------------------------===//
720
721namespace {
722struct NVVMTargetConvertToLLVMAttrInterface
723 : public ConvertToLLVMAttrInterface::ExternalModel<
724 NVVMTargetConvertToLLVMAttrInterface, NVVM::NVVMTargetAttr> {
725 /// Configure GPU to NVVM.
726 void populateConvertToLLVMConversionPatterns(
727 Attribute attr, ConversionTarget &target,
728 LLVMTypeConverter &typeConverter, RewritePatternSet &patterns) const;
729};
730} // namespace
731
732void NVVMTargetConvertToLLVMAttrInterface::
733 populateConvertToLLVMConversionPatterns(Attribute attr,
734 ConversionTarget &target,
735 LLVMTypeConverter &typeConverter,
736 RewritePatternSet &patterns) const {
737 configureGpuToNVVMConversionLegality(target);
738 configureGpuToNVVMTypeConverter(converter&: typeConverter);
739 populateGpuToNVVMConversionPatterns(converter: typeConverter, patterns);
740}
741
742void mlir::NVVM::registerConvertGpuToNVVMInterface(DialectRegistry &registry) {
743 registry.addExtension(extensionFn: +[](MLIRContext *ctx, NVVMDialect *dialect) {
744 NVVMTargetAttr::attachInterface<NVVMTargetConvertToLLVMAttrInterface>(context&: *ctx);
745 });
746}
747