1//===- LowerGpuOpsToROCDLOps.cpp - MLIR GPU to ROCDL lowering passes ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a pass to generate ROCDLIR operations for higher-level
10// GPU operations.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h"
15#include "mlir/Dialect/Arith/Transforms/Passes.h"
16#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
17#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
18#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
19#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
20#include "mlir/Dialect/LLVMIR/LLVMFloatIntrinsicOps.h"
21#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
22#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
23#include "mlir/Dialect/LLVMIR/LLVMTerminatorOps.h"
24#include "mlir/Pass/Pass.h"
25#include "mlir/Pass/PassManager.h"
26
27#include "mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h"
28#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
29#include "mlir/Conversion/ConvertToLLVM/ToLLVMPass.h"
30#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
31#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
32#include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
33#include "mlir/Conversion/LLVMCommon/Pattern.h"
34#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
35#include "mlir/Conversion/MathToLLVM/MathToLLVM.h"
36#include "mlir/Conversion/MathToROCDL/MathToROCDL.h"
37#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
38#include "mlir/Dialect/ControlFlow/IR/ControlFlow.h"
39#include "mlir/Dialect/Func/IR/FuncOps.h"
40#include "mlir/Dialect/GPU/IR/GPUDialect.h"
41#include "mlir/Dialect/GPU/Transforms/Passes.h"
42#include "mlir/Dialect/GPU/Transforms/ROCDL/Passes.h"
43#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
44#include "mlir/Dialect/MemRef/IR/MemRef.h"
45#include "mlir/Dialect/Vector/IR/VectorOps.h"
46#include "mlir/IR/BuiltinAttributes.h"
47#include "mlir/IR/Matchers.h"
48#include "mlir/Transforms/DialectConversion.h"
49#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
50
51#include "../GPUCommon/GPUOpsLowering.h"
52#include "../GPUCommon/IndexIntrinsicsOpLowering.h"
53
54namespace mlir {
55#define GEN_PASS_DEF_CONVERTGPUOPSTOROCDLOPS
56#include "mlir/Conversion/Passes.h.inc"
57} // namespace mlir
58
59using namespace mlir;
60
61// Truncate or extend the result depending on the index bitwidth specified
62// by the LLVMTypeConverter options.
63static Value truncOrExtToLLVMType(ConversionPatternRewriter &rewriter,
64 Location loc, Value value,
65 const LLVMTypeConverter &converter) {
66 int64_t intWidth = cast<IntegerType>(Val: value.getType()).getWidth();
67 int64_t indexBitwidth = converter.getIndexTypeBitwidth();
68 auto indexBitwidthType =
69 IntegerType::get(context: rewriter.getContext(), width: converter.getIndexTypeBitwidth());
70 // TODO: use <=> in C++20.
71 if (indexBitwidth > intWidth) {
72 return LLVM::SExtOp::create(builder&: rewriter, location: loc, res: indexBitwidthType, arg: value);
73 }
74 if (indexBitwidth < intWidth) {
75 return LLVM::TruncOp::create(builder&: rewriter, location: loc, res: indexBitwidthType, arg: value);
76 }
77 return value;
78}
79
80/// Returns true if the given `gpu.func` can be safely called using the bare
81/// pointer calling convention.
82static bool canBeCalledWithBarePointers(gpu::GPUFuncOp func) {
83 bool canBeBare = true;
84 for (Type type : func.getArgumentTypes())
85 if (auto memrefTy = dyn_cast<BaseMemRefType>(Val&: type))
86 canBeBare &= LLVMTypeConverter::canConvertToBarePtr(type: memrefTy);
87 return canBeBare;
88}
89
90static Value getLaneId(RewriterBase &rewriter, Location loc) {
91 auto int32Type = IntegerType::get(context: rewriter.getContext(), width: 32);
92 Value zero = arith::ConstantIntOp::create(builder&: rewriter, location: loc, value: 0, width: 32);
93 Value minus1 = arith::ConstantIntOp::create(builder&: rewriter, location: loc, value: -1, width: 32);
94 NamedAttribute noundef = rewriter.getNamedAttr(
95 name: LLVM::LLVMDialect::getNoUndefAttrName(), val: rewriter.getUnitAttr());
96 NamedAttribute lowRange = rewriter.getNamedAttr(
97 name: LLVM::LLVMDialect::getRangeAttrName(),
98 val: LLVM::ConstantRangeAttr::get(context: rewriter.getContext(), lower: APInt::getZero(numBits: 32),
99 upper: APInt(32, 32)));
100 NamedAttribute highRange = rewriter.getNamedAttr(
101 name: LLVM::LLVMDialect::getRangeAttrName(),
102 val: LLVM::ConstantRangeAttr::get(context: rewriter.getContext(), lower: APInt::getZero(numBits: 32),
103 upper: APInt(32, 64)));
104 Value mbcntLo = ROCDL::MbcntLoOp::create(
105 builder&: rewriter, location: loc, res: int32Type, in0: minus1, in1: zero, /*arg_attrs=*/{},
106 /*res_attrs=*/
107 rewriter.getArrayAttr(value: rewriter.getDictionaryAttr(value: {noundef, lowRange})));
108 Value laneId = ROCDL::MbcntHiOp::create(
109 builder&: rewriter, location: loc, res: int32Type, in0: minus1, in1: mbcntLo, /*arg_attrs=*/{},
110 res_attrs: rewriter.getArrayAttr(value: rewriter.getDictionaryAttr(value: {noundef, highRange})));
111 return laneId;
112}
113
114/// Maximum number of threads per block dimension on AMD GPUs.
115static constexpr int64_t kMaxThreadsPerBlockDim = 1024;
116
117/// Emits a call to an OCKL block/grid size function corresponding to
118/// `indexKind` with argument `dim`, except that if the context around
119/// `contextOp` gives an exact size for that dimension, return that as
120/// an `i64` constant instead.
121static Value getKnownOrOcklDim(RewriterBase &rewriter,
122 gpu::index_lowering::IndexKind indexKind,
123 gpu::Dimension dim, Operation *contextOp,
124 std::optional<uint32_t> opUpperBound) {
125 Location loc = contextOp->getLoc();
126 MLIRContext *context = contextOp->getContext();
127
128 auto i32Ty = IntegerType::get(context, width: 32);
129 auto i64Ty = IntegerType::get(context, width: 64);
130
131 if (std::optional<uint32_t> knownDim =
132 gpu::getKnownDimensionSizeAround(op: contextOp, kind: indexKind, dim))
133 return LLVM::ConstantOp::create(builder&: rewriter, location: loc,
134 value: rewriter.getI64IntegerAttr(value: *knownDim));
135
136 int32_t dimParam = static_cast<int32_t>(dim);
137
138 StringRef functionName;
139 switch (indexKind) {
140 case gpu::index_lowering::IndexKind::Block:
141 functionName = "__ockl_get_local_size";
142 break;
143 case gpu::index_lowering::IndexKind::Grid:
144 functionName = "__ockl_get_num_groups";
145 break;
146 case gpu::index_lowering::IndexKind::Cluster:
147 case gpu::index_lowering::IndexKind::Other:
148 llvm_unreachable("Not valid index kinds for ockl lookup");
149 }
150
151 // Declare the ockl function: i64 @functionName(i32).
152 auto fnType = LLVM::LLVMFunctionType::get(result: i64Ty, arguments: {i32Ty});
153 Operation *moduleOp = contextOp->getParentWithTrait<OpTrait::SymbolTable>();
154 LLVM::LLVMFuncOp funcOp =
155 getOrDefineFunction(moduleOp, loc, b&: rewriter, name: functionName, type: fnType);
156
157 // Create the call.
158 Value dimConst = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: dimParam);
159 auto callOp =
160 LLVM::CallOp::create(builder&: rewriter, location: loc, func: funcOp, args: ValueRange{dimConst});
161
162 LLVM::ConstantRangeAttr range;
163 if (opUpperBound) {
164 range = LLVM::ConstantRangeAttr::get(
165 context, lower: APInt(64, 1),
166 upper: APInt(64, static_cast<uint64_t>(*opUpperBound) + 1));
167 } else if (indexKind == gpu::index_lowering::IndexKind::Block) {
168 // Set the hardware limit for block ranges as the bounds on block dim calls.
169 range = LLVM::ConstantRangeAttr::get(context, lower: APInt(64, 1),
170 upper: APInt(64, kMaxThreadsPerBlockDim + 1));
171 }
172 if (range) {
173 callOp.setResAttrsAttr(rewriter.getArrayAttr(value: rewriter.getDictionaryAttr(
174 value: rewriter.getNamedAttr(name: LLVM::LLVMDialect::getRangeAttrName(), val: range))));
175 }
176 return callOp.getResult();
177}
178
179static constexpr StringLiteral amdgcnDataLayout =
180 "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32"
181 "-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:"
182 "32-v32:"
183 "32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:"
184 "64-S32-A5-G1-ni:7:8:9";
185
186namespace {
187
188/// Lowers gpu.block_dim / gpu.grid_dim to direct __ockl_get_local_size /
189/// __ockl_get_num_groups function calls.
190template <typename OpTy>
191struct GPUDimOpToOcklCall final : ConvertOpToLLVMPattern<OpTy> {
192 GPUDimOpToOcklCall(const LLVMTypeConverter &converter,
193 gpu::index_lowering::IndexKind indexKind)
194 : ConvertOpToLLVMPattern<OpTy>(converter), indexKind(indexKind) {}
195
196 LogicalResult
197 matchAndRewrite(OpTy op, typename OpTy::Adaptor adaptor,
198 ConversionPatternRewriter &rewriter) const override {
199 Location loc = op.getLoc();
200
201 std::optional<uint32_t> opUpperBound;
202 if (auto bound = op.getUpperBound())
203 opUpperBound = static_cast<uint32_t>(bound->getZExtValue());
204
205 Value ocklCall = getKnownOrOcklDim(rewriter, indexKind, op.getDimension(),
206 op, opUpperBound);
207 Value result = truncOrExtToLLVMType(rewriter, loc, ocklCall,
208 *this->getTypeConverter());
209 rewriter.replaceOp(op, result);
210 return success();
211 }
212
213private:
214 const gpu::index_lowering::IndexKind indexKind;
215};
216
217struct GPULaneIdOpToROCDL : ConvertOpToLLVMPattern<gpu::LaneIdOp> {
218 using ConvertOpToLLVMPattern<gpu::LaneIdOp>::ConvertOpToLLVMPattern;
219
220 LogicalResult
221 matchAndRewrite(gpu::LaneIdOp op, gpu::LaneIdOp::Adaptor adaptor,
222 ConversionPatternRewriter &rewriter) const override {
223 Location loc = op.getLoc();
224 MLIRContext *context = rewriter.getContext();
225 // convert to:
226 // %mlo = call noundef range(i32 0, 32)
227 // @llvm.amdgcn.mbcnt.lo(-1, 0)
228 // followed by:
229 // %lid = call noundef range(i32 0, 64)
230 // @llvm.amdgcn.mbcnt.hi(-1, %mlo)
231
232 Value laneId = getLaneId(rewriter, loc);
233 // Truncate or extend the result depending on the index bitwidth specified
234 // by the LLVMTypeConverter options.
235 const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
236 if (indexBitwidth > 32) {
237 laneId = LLVM::SExtOp::create(
238 builder&: rewriter, location: loc, res: IntegerType::get(context, width: indexBitwidth), arg: laneId);
239 } else if (indexBitwidth < 32) {
240 laneId = LLVM::TruncOp::create(
241 builder&: rewriter, location: loc, res: IntegerType::get(context, width: indexBitwidth), arg: laneId);
242 }
243 rewriter.replaceOp(op, newValues: {laneId});
244 return success();
245 }
246};
247
248struct GPUSubgroupSizeOpToROCDL : ConvertOpToLLVMPattern<gpu::SubgroupSizeOp> {
249 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
250
251 GPUSubgroupSizeOpToROCDL(const LLVMTypeConverter &converter,
252 const ROCDL::TargetInfo &target)
253 : ConvertOpToLLVMPattern<gpu::SubgroupSizeOp>(converter), target(target) {
254 }
255
256 LogicalResult
257 matchAndRewrite(gpu::SubgroupSizeOp op, gpu::SubgroupSizeOp::Adaptor adaptor,
258 ConversionPatternRewriter &rewriter) const override {
259 LLVM::ConstantRangeAttr bounds = nullptr;
260 bool isWave64 = target.getWavefrontSize() == 64;
261 if (auto upperBoundAttr = op.getUpperBoundAttr()) {
262 bounds = rewriter.getAttr<LLVM::ConstantRangeAttr>(
263 /*bitWidth=*/args: 32, /*lower=*/args: isWave64 ? 64 : 32,
264 /*upper=*/args: op.getUpperBoundAttr().getInt() + 1);
265 }
266 Value wavefrontOp = ROCDL::WavefrontSizeOp::create(
267 builder&: rewriter, location: op.getLoc(), res: rewriter.getI32Type(), range: bounds);
268 wavefrontOp = truncOrExtToLLVMType(rewriter, loc: op.getLoc(), value: wavefrontOp,
269 converter: *getTypeConverter());
270 rewriter.replaceOp(op, newValues: {wavefrontOp});
271 return success();
272 }
273
274 const ROCDL::TargetInfo target;
275};
276
277struct GPUSubgroupIdOpToROCDL : ConvertOpToLLVMPattern<gpu::SubgroupIdOp> {
278 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
279
280 GPUSubgroupIdOpToROCDL(const LLVMTypeConverter &converter,
281 const ROCDL::TargetInfo &target)
282 : ConvertOpToLLVMPattern<gpu::SubgroupIdOp>(converter), target(target) {}
283
284 LogicalResult
285 matchAndRewrite(gpu::SubgroupIdOp op, gpu::SubgroupIdOp::Adaptor adaptor,
286 ConversionPatternRewriter &rewriter) const override {
287 Location loc = op.getLoc();
288 auto int32Type = rewriter.getI32Type();
289
290 Value subgroupId;
291 if (target.has(feature: llvm::AMDGPU::FEAT_GFX12_INSTS)) {
292 // For gfx12+, use the hardware wave.id register directly.
293 LLVM::ConstantRangeAttr bounds;
294 if (auto upperBoundAttr = op.getUpperBoundAttr())
295 bounds = rewriter.getAttr<LLVM::ConstantRangeAttr>(
296 /*bitWidth=*/args: 32, /*lower=*/args: 0,
297 /*upper=*/args: upperBoundAttr.getInt());
298 subgroupId = ROCDL::WaveId::create(builder&: rewriter, location: loc, res: int32Type, range: bounds);
299 } else {
300 // For older architectures, compute:
301 // subgroup_id = linearized_thread_id / subgroup_size
302 // where linearized_thread_id = tid.x + dim.x * (tid.y + dim.y * tid.z)
303 auto tidX = ROCDL::ThreadIdXOp::create(builder&: rewriter, location: loc, resultType: int32Type);
304 auto tidY = ROCDL::ThreadIdYOp::create(builder&: rewriter, location: loc, resultType: int32Type);
305 auto tidZ = ROCDL::ThreadIdZOp::create(builder&: rewriter, location: loc, resultType: int32Type);
306 auto setBoundFromContext = [&](Operation *tidOp, gpu::Dimension dim) {
307 if (LLVM::ConstantRangeAttr range =
308 gpu::index_lowering::getIndexOpRange(
309 op, dim, opUpperBound: std::nullopt,
310 indexKind: gpu::index_lowering::IndexKind::Block,
311 intrType: gpu::index_lowering::IntrType::Id, bitWidth: 32))
312 tidOp->setInherentAttr(name: rewriter.getStringAttr(bytes: "range"), value: range);
313 };
314 setBoundFromContext(tidX, gpu::Dimension::x);
315 setBoundFromContext(tidY, gpu::Dimension::y);
316 setBoundFromContext(tidZ, gpu::Dimension::z);
317
318 auto flags =
319 LLVM::IntegerOverflowFlags::nsw | LLVM::IntegerOverflowFlags::nuw;
320
321 auto getBlockDim = [&](gpu::Dimension dim) {
322 Value dim64 =
323 getKnownOrOcklDim(rewriter, indexKind: gpu::index_lowering::IndexKind::Block,
324 dim, contextOp: op, opUpperBound: std::nullopt);
325 Value dimTrunc =
326 LLVM::TruncOp::create(builder&: rewriter, location: loc, res: int32Type, arg: dim64, overflowFlags: flags);
327 return dimTrunc;
328 };
329 Value dimX = getBlockDim(gpu::Dimension::x);
330 Value dimY = getBlockDim(gpu::Dimension::y);
331
332 // linearized = tid.x + dim.x * (tid.y + dim.y * tid.z)
333 // Thread IDs and dimensions are non-negative and small, so use nuw+nsw.
334 Value dimYxTidZ =
335 LLVM::MulOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: dimY, rhs: tidZ, overflowFlags: flags);
336 Value tidYPlusDimYxTidZ =
337 LLVM::AddOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: tidY, rhs: dimYxTidZ, overflowFlags: flags);
338 Value dimXxInner = LLVM::MulOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: dimX,
339 rhs: tidYPlusDimYxTidZ, overflowFlags: flags);
340 Value linearized = LLVM::AddOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: tidX,
341 rhs: dimXxInner, overflowFlags: flags);
342
343 Value subgroupSize =
344 ROCDL::WavefrontSizeOp::create(builder&: rewriter, location: loc, resultType: int32Type);
345 subgroupId = LLVM::UDivOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: linearized,
346 rhs: subgroupSize);
347 }
348
349 subgroupId =
350 truncOrExtToLLVMType(rewriter, loc, value: subgroupId, converter: *getTypeConverter());
351 rewriter.replaceOp(op, newValues: subgroupId);
352 return success();
353 }
354
355 const ROCDL::TargetInfo target;
356};
357
358static bool isSupportedReadLaneType(Type type) {
359 // https://llvm.org/docs/AMDGPUUsage.html#llvm-ir-intrinsics
360 if (isa<Float16Type, BFloat16Type, Float32Type, Float64Type,
361 LLVM::LLVMPointerType>(Val: type))
362 return true;
363
364 if (auto intType = dyn_cast<IntegerType>(Val&: type))
365 return llvm::is_contained(Set: {16, 32, 64},
366 Element: static_cast<int>(intType.getWidth()));
367
368 if (auto vecType = dyn_cast<VectorType>(Val&: type)) {
369 Type elementType = vecType.getElementType();
370 if (elementType.isInteger(width: 32))
371 return true;
372
373 if (vecType.getNumElements() == 2 &&
374 (isa<Float16Type, BFloat16Type>(Val: elementType) ||
375 elementType.isInteger(width: 16)))
376 return true;
377 }
378
379 return false;
380}
381
382struct GPUSubgroupBroadcastOpToROCDL
383 : public ConvertOpToLLVMPattern<gpu::SubgroupBroadcastOp> {
384 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
385
386 LogicalResult
387 matchAndRewrite(gpu::SubgroupBroadcastOp op, OpAdaptor adaptor,
388 ConversionPatternRewriter &rewriter) const override {
389 Value src = adaptor.getSrc();
390 if (isSupportedReadLaneType(type: src.getType())) {
391 Value result = createReadlaneOp(op, adaptor, rewriter, src);
392 rewriter.replaceOp(op, newValues: result);
393 return success();
394 }
395
396 Type i32 = rewriter.getI32Type();
397 Location loc = op.getLoc();
398 SmallVector<Value> decomposed;
399 if (failed(Result: LLVM::decomposeValue(builder&: rewriter, loc, src, dstType: i32, result&: decomposed,
400 /*permitVariablySizedScalars=*/true)))
401 return rewriter.notifyMatchFailure(arg&: op,
402 msg: "Unexpected decomposition failure");
403
404 SmallVector<Value> results;
405 results.reserve(N: decomposed.size());
406 for (Value v : decomposed)
407 results.emplace_back(Args: createReadlaneOp(op, adaptor, rewriter, src: v));
408
409 Value result = LLVM::composeValue(builder&: rewriter, loc, src: results, dstType: src.getType());
410 rewriter.replaceOp(op, newValues: result);
411 return success();
412 }
413
414private:
415 static Value createReadlaneOp(gpu::SubgroupBroadcastOp op, OpAdaptor adaptor,
416 ConversionPatternRewriter &rewriter,
417 Value src) {
418 if (adaptor.getBroadcastType() == gpu::BroadcastType::specific_lane) {
419 return ROCDL::ReadlaneOp::create(builder&: rewriter, location: op.getLoc(), res: src.getType(),
420 src0: src, src1: adaptor.getLane());
421 } else { // first_active_lane
422 return ROCDL::ReadfirstlaneOp::create(builder&: rewriter, location: op.getLoc(),
423 res: src.getType(), src);
424 }
425 }
426};
427
428struct GPUBallotOpToROCDL : public ConvertOpToLLVMPattern<gpu::BallotOp> {
429 using ConvertOpToLLVMPattern<gpu::BallotOp>::ConvertOpToLLVMPattern;
430
431 LogicalResult
432 matchAndRewrite(gpu::BallotOp op, gpu::BallotOp::Adaptor adaptor,
433 ConversionPatternRewriter &rewriter) const override {
434 auto intType = cast<IntegerType>(Val: op.getType());
435 unsigned width = intType.getWidth();
436
437 // ROCDL ballot natively supports i32 and i64 for wavefront sizes of
438 // 32 and 64 lanes.
439 if (width != 32 && width != 64)
440 return rewriter.notifyMatchFailure(
441 arg&: op, msg: "rocdl.ballot only supports i32 and i64 result types");
442
443 rewriter.replaceOpWithNewOp<ROCDL::BallotOp>(op, args: op.getType(),
444 args: adaptor.getPredicate());
445 return success();
446 }
447};
448
449struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
450 using ConvertOpToLLVMPattern<gpu::ShuffleOp>::ConvertOpToLLVMPattern;
451
452 /// Lowers a shuffle to the corresponding ROCDL ops.
453 ///
454 /// Use the `width` argument to see if src lane is participating.
455 /// If not the dstLane would be itself.
456 ///
457 /// Shuffle with DS Bpermute:
458 /// let shflMode = [xor, up, down, idx]
459 /// let width = 32(usually warpsize), step = [1, 2, 4, 8, 16, ... , width].
460 /// 1. curLaneId = using mbcnt.lo + mbcnt.hi
461 /// 2. widthOrZeroIfOutside = (curLaneId + width) & -width
462 /// 3. dstLane = shflMode(curLaneId, step)
463 /// 4. isActiveSrcLane = dstLane < isActiveSrcLane
464 /// 5. dstLane = isActiveSrcLane ? dstLane : curLaneId
465 /// 6. dwordAlignedDstLane = dstLane * 4 or dstLane << 2.
466 /// 7. bpermute(dwordAlignedDstLane, shfl_value).
467 ///
468 LogicalResult
469 matchAndRewrite(gpu::ShuffleOp op, OpAdaptor adaptor,
470 ConversionPatternRewriter &rewriter) const override {
471 Location loc = op->getLoc();
472 Value initShflValue = adaptor.getValue();
473
474 Value srcLaneId = getLaneId(rewriter, loc);
475
476 auto int32Type = IntegerType::get(context: rewriter.getContext(), width: 32);
477 Value width = adaptor.getWidth();
478 Value zero = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: int32Type, value: 0);
479 Value negwidth = LLVM::SubOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: zero, rhs: width);
480 Value add = LLVM::AddOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: srcLaneId, rhs: width);
481 Value widthOrZeroIfOutside =
482 LLVM::AndOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: add, rhs: negwidth);
483 Value dstLane;
484
485 switch (op.getMode()) {
486 case gpu::ShuffleMode::UP:
487 dstLane = LLVM::SubOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: srcLaneId,
488 rhs: adaptor.getOffset());
489 break;
490 case gpu::ShuffleMode::DOWN:
491 dstLane = LLVM::AddOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: srcLaneId,
492 rhs: adaptor.getOffset());
493 break;
494 case gpu::ShuffleMode::XOR:
495 dstLane = LLVM::XOrOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: srcLaneId,
496 rhs: adaptor.getOffset());
497 break;
498 case gpu::ShuffleMode::IDX:
499 dstLane = adaptor.getOffset();
500 break;
501 }
502 Value isActiveSrcLane = LLVM::ICmpOp::create(
503 builder&: rewriter, location: loc, predicate: LLVM::ICmpPredicate::slt, lhs: dstLane, rhs: widthOrZeroIfOutside);
504 Value selectDstLane = LLVM::SelectOp::create(builder&: rewriter, location: loc, condition: isActiveSrcLane,
505 trueValue: dstLane, falseValue: srcLaneId);
506 Value two = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: int32Type, value: 2);
507 Value dwordAlignedDstLane =
508 LLVM::ShlOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: selectDstLane, rhs: two);
509
510 SmallVector<Value> decomposed;
511 if (failed(Result: LLVM::decomposeValue(builder&: rewriter, loc, src: initShflValue, dstType: int32Type,
512 result&: decomposed)))
513 return rewriter.notifyMatchFailure(arg&: op,
514 msg: "failed to decompose value to i32");
515 SmallVector<Value> swizzled;
516 for (Value v : decomposed) {
517 Value res = ROCDL::DsBpermuteOp::create(builder&: rewriter, location: loc, res: int32Type,
518 index: dwordAlignedDstLane, src: v);
519 swizzled.emplace_back(Args&: res);
520 }
521 Value shflValue =
522 LLVM::composeValue(builder&: rewriter, loc, src: swizzled, dstType: initShflValue.getType());
523 rewriter.replaceOp(op, newValues: {shflValue, isActiveSrcLane});
524 return success();
525 }
526};
527
528/// Emit an LLVM fence with MMRA metadata based on the given address spaces.
529/// If `addrSpaces` is nullopt, all memory is fenced (global + LDS).
530static void emitFences(std::optional<ArrayAttr> addrSpaces,
531 ConversionPatternRewriter &rewriter, Location loc,
532 StringRef scope, bool before) {
533 bool fenceGlobal = false;
534 bool fenceLDS = false;
535
536 if (addrSpaces) {
537 for (auto spaceAttr : addrSpaces->getAsRange<gpu::AddressSpaceAttr>()) {
538 switch (spaceAttr.getValue()) {
539 case gpu::AddressSpace::Global:
540 fenceGlobal = true;
541 break;
542 case gpu::AddressSpace::Workgroup:
543 fenceLDS = true;
544 break;
545 case gpu::AddressSpace::Private:
546 case gpu::AddressSpace::Constant:
547 break;
548 }
549 }
550 } else {
551 fenceGlobal = true;
552 fenceLDS = true;
553 }
554
555 if (!fenceGlobal && !fenceLDS)
556 return;
557
558 Attribute mmra;
559 if (fenceLDS && !fenceGlobal)
560 mmra =
561 rewriter.getAttr<LLVM::MMRATagAttr>(args: "amdgpu-synchronize-as", args: "local");
562 else if (fenceGlobal && !fenceLDS)
563 mmra =
564 rewriter.getAttr<LLVM::MMRATagAttr>(args: "amdgpu-synchronize-as", args: "global");
565
566 auto ordering =
567 before ? LLVM::AtomicOrdering::release : LLVM::AtomicOrdering::acquire;
568 auto fence = LLVM::FenceOp::create(builder&: rewriter, location: loc, ordering, syncscope: scope);
569 if (mmra)
570 fence->setDiscardableAttr(name: LLVM::LLVMDialect::getMmraAttrName(), value: mmra);
571}
572
573static constexpr int32_t kWholeClusterBarrierId = -3;
574static constexpr int32_t kWholeWorkgroupBarrierId = -1;
575struct GPUBarrierOpLowering final : ConvertOpToLLVMPattern<gpu::BarrierOp> {
576 GPUBarrierOpLowering(const LLVMTypeConverter &converter,
577 const ROCDL::TargetInfo &target)
578 : ConvertOpToLLVMPattern<gpu::BarrierOp>(converter), target(target) {}
579
580 ROCDL::TargetInfo target;
581
582 LogicalResult
583 matchAndRewrite(gpu::BarrierOp op, gpu::BarrierOp::Adaptor adaptor,
584 ConversionPatternRewriter &rewriter) const override {
585 Location loc = op.getLoc();
586 gpu::BarrierScope scope = op.getScope();
587
588 // Subgroup (wave) scope.
589 if (scope == gpu::BarrierScope::Subgroup) {
590 emitFences(addrSpaces: op.getAddressSpaces(), rewriter, loc, scope: "wavefront",
591 /*before=*/true);
592 ROCDL::WaveBarrierOp::create(builder&: rewriter, location: loc);
593 emitFences(addrSpaces: op.getAddressSpaces(), rewriter, loc, scope: "wavefront",
594 /*before=*/false);
595 rewriter.eraseOp(op);
596 return success();
597 }
598
599 // Cluster scope: gfx1250+ only, signal/wait with constant -3.
600 if (scope == gpu::BarrierScope::Cluster) {
601 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS))
602 return op.emitOpError(message: "cluster scope barriers require gfx1250+");
603 emitFences(addrSpaces: op.getAddressSpaces(), rewriter, loc, scope: "cluster",
604 /*before=*/true);
605 ROCDL::BarrierSignalOp::create(builder&: rewriter, location: loc, id: kWholeClusterBarrierId);
606 ROCDL::BarrierWaitOp::create(
607 builder&: rewriter, location: loc, id: static_cast<int16_t>(kWholeClusterBarrierId));
608 emitFences(addrSpaces: op.getAddressSpaces(), rewriter, loc, scope: "cluster",
609 /*before=*/false);
610 rewriter.eraseOp(op);
611 return success();
612 }
613
614 // Workgroup scope (default).
615 assert(scope == gpu::BarrierScope::Workgroup && "unsupported scope");
616
617 // Named barrier path.
618 if (Value namedBarrier = adaptor.getNamedBarrier()) {
619 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX12_INSTS))
620 return op.emitOpError(message: "named barriers require gfx12+");
621
622 emitFences(addrSpaces: op.getAddressSpaces(), rewriter, loc, scope: "workgroup",
623 /*before=*/true);
624 // A wave must join the named barrier before it may signal it.
625 ROCDL::BarrierJoinOp::create(builder&: rewriter, location: loc, ptr: namedBarrier);
626 // Signal with memberCnt=0 retains the count from s.barrier.init.
627 ROCDL::BarrierSignalVarOp::create(builder&: rewriter, location: loc, ptr: namedBarrier,
628 /*memberCnt=*/0);
629 // id=1 selects the named-barrier wait class; the actual barrier waited
630 // on is the last one this wave joined.
631 ROCDL::BarrierWaitOp::create(builder&: rewriter, location: loc, id: static_cast<int16_t>(1));
632 emitFences(addrSpaces: op.getAddressSpaces(), rewriter, loc, scope: "workgroup",
633 /*before=*/false);
634 rewriter.eraseOp(op);
635 return success();
636 }
637
638 // Regular workgroup barrier.
639 emitFences(addrSpaces: op.getAddressSpaces(), rewriter, loc, scope: "workgroup",
640 /*before=*/true);
641 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX12_INSTS)) {
642 ROCDL::SBarrierOp::create(builder&: rewriter, location: loc);
643 } else {
644 ROCDL::BarrierSignalOp::create(builder&: rewriter, location: loc, id: kWholeWorkgroupBarrierId);
645 ROCDL::BarrierWaitOp::create(
646 builder&: rewriter, location: loc, id: static_cast<int16_t>(kWholeWorkgroupBarrierId));
647 }
648 emitFences(addrSpaces: op.getAddressSpaces(), rewriter, loc, scope: "workgroup",
649 /*before=*/false);
650 rewriter.eraseOp(op);
651 return success();
652 }
653};
654
655struct GPUInitializeNamedBarrierOpLowering final
656 : ConvertOpToLLVMPattern<gpu::InitializeNamedBarrierOp> {
657 GPUInitializeNamedBarrierOpLowering(const LLVMTypeConverter &converter,
658 const ROCDL::TargetInfo &target)
659 : ConvertOpToLLVMPattern<gpu::InitializeNamedBarrierOp>(converter),
660 target(target) {}
661
662 ROCDL::TargetInfo target;
663
664 LogicalResult
665 matchAndRewrite(gpu::InitializeNamedBarrierOp op,
666 gpu::InitializeNamedBarrierOp::Adaptor adaptor,
667 ConversionPatternRewriter &rewriter) const override {
668 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX12_INSTS))
669 return op.emitOpError(message: "named barriers require gfx12+");
670
671 Location loc = op.getLoc();
672
673 // The count must be a constant for rocdl.s.barrier.init.
674 IntegerAttr countAttr;
675 if (!matchPattern(value: op.getMemberCount(), pattern: m_Constant(bind_value: &countAttr)))
676 return op.emitOpError(
677 message: "named barrier member count must be a constant for ROCDL lowering");
678 int32_t count = countAttr.getInt();
679
680 // Place the global in the symbol-table scope enclosing the function-like
681 // op that contains this barrier (typically a module).
682 auto funcOp = op->getParentOfType<FunctionOpInterface>();
683 if (!funcOp)
684 return op.emitOpError(message: "must be inside a function-like op");
685 Operation *symbolTableOp =
686 funcOp->getParentWithTrait<OpTrait::SymbolTable>();
687 if (!symbolTableOp)
688 return op.emitOpError(
689 message: "enclosing function-like op must have a symbol-table parent");
690
691 auto targetTy = LLVM::LLVMTargetExtType::get(
692 context: rewriter.getContext(), extTypeName: "amdgcn.named.barrier", typeParams: {}, intParams: {0});
693 auto ptrTy = LLVM::LLVMPointerType::get(
694 context: rewriter.getContext(), addressSpace: ROCDL::ROCDLDialect::kBarrierAddressSpace);
695
696 // Build the global detached so SymbolTable::insert can both place it and
697 // rename it as needed without creating a transient name conflict in IR.
698 OpBuilder detachedBuilder(rewriter.getContext());
699 auto globalOp = LLVM::GlobalOp::create(
700 builder&: detachedBuilder, location: loc, type: targetTy, /*isConstant=*/false,
701 linkage: LLVM::Linkage::Internal, name: "__named_barrier", /*value=*/Attribute(),
702 /*alignment=*/0,
703 /*addrSpace=*/ROCDL::ROCDLDialect::kBarrierAddressSpace);
704 // Initialize with poison.
705 {
706 Region &region = globalOp.getInitializerRegion();
707 Block *block = detachedBuilder.createBlock(parent: &region);
708 detachedBuilder.setInsertionPointToStart(block);
709 auto poison = LLVM::PoisonOp::create(builder&: detachedBuilder, location: loc, res: targetTy);
710 LLVM::ReturnOp::create(builder&: detachedBuilder, location: loc, arg: poison);
711 }
712 // SymbolTable::insert places the op in the symbol-table body and renames
713 // the symbol to avoid collisions with any existing entries.
714 StringAttr globalName = SymbolTable(symbolTableOp).insert(symbol: globalOp);
715
716 // Get address of the global.
717 rewriter.setInsertionPoint(op);
718 auto addrOf = LLVM::AddressOfOp::create(builder&: rewriter, location: loc, res: ptrTy, global_name: globalName);
719
720 // Initialize the barrier.
721 ROCDL::BarrierInitOp::create(builder&: rewriter, location: loc, ptr: addrOf, memberCnt: count);
722
723 rewriter.replaceOp(op, newValues: addrOf.getResult());
724 return success();
725 }
726};
727
728/// Import the GPU Ops to ROCDL Patterns.
729#include "GPUToROCDL.cpp.inc"
730
731// A pass that replaces all occurrences of GPU device operations with their
732// corresponding ROCDL equivalent.
733//
734// This pass only handles device code and is not meant to be run on GPU host
735// code.
736struct LowerGpuOpsToROCDLOpsPass final
737 : public impl::ConvertGpuOpsToROCDLOpsBase<LowerGpuOpsToROCDLOpsPass> {
738 using Base::Base;
739
740 void getDependentDialects(DialectRegistry &registry) const override {
741 Base::getDependentDialects(registry);
742 registerConvertToLLVMDependentDialectLoading(registry);
743 }
744
745 void runOnOperation() override {
746 gpu::GPUModuleOp m = getOperation();
747 MLIRContext *ctx = m.getContext();
748
749 auto llvmDataLayout = m->getDiscardableAttrOfType<StringAttr>(
750 name: LLVM::LLVMDialect::getDataLayoutAttrName());
751 if (!llvmDataLayout) {
752 llvmDataLayout = StringAttr::get(context: ctx, bytes: amdgcnDataLayout);
753 m->setDiscardableAttr(LLVM::LLVMDialect::getDataLayoutAttrName(),
754 llvmDataLayout);
755 }
756 // Request C wrapper emission for externally visible functions only. A
757 // private function cannot be called from outside its module, so its C
758 // interface wrapper is unreachable and would only anchor the wrapped
759 // function against dead-code elimination.
760 for (auto func : m.getOps<func::FuncOp>()) {
761 if (func.isPrivate())
762 continue;
763 func->setDiscardableAttr(LLVM::LLVMDialect::getEmitCWrapperAttrName(),
764 UnitAttr::get(ctx));
765 }
766
767 FailureOr<ROCDL::TargetInfo> targetInfo = ROCDL::TargetInfo::get(
768 ROCDL::resolveArchOption(arch, chipset), waveSize,
769 [&] { return emitError(UnknownLoc::get(ctx)); });
770 if (failed(Result: targetInfo))
771 return signalPassFailure();
772
773 /// Customize the bitwidth used for the device side index computations.
774 LowerToLLVMOptions options(
775 ctx, DataLayout(cast<DataLayoutOpInterface>(Val: m.getOperation())));
776 options.dataLayout = llvm::DataLayout(llvmDataLayout.getValue());
777 if (indexBitwidth != kDeriveIndexBitwidthFromDataLayout)
778 options.overrideIndexBitwidth(indexBitwidth);
779
780 if (useBarePtrCallConv) {
781 options.useBarePtrCallConv = true;
782 WalkResult canUseBarePointers =
783 m.walk(callback: [](gpu::GPUFuncOp func) -> WalkResult {
784 if (canBeCalledWithBarePointers(func))
785 return WalkResult::advance();
786 return WalkResult::interrupt();
787 });
788 if (canUseBarePointers.wasInterrupted()) {
789 emitError(loc: UnknownLoc::get(context: ctx),
790 message: "bare pointer calling convention requires all memrefs to "
791 "have static shape and use the identity map");
792 return signalPassFailure();
793 }
794 }
795
796 // Apply in-dialect lowering. In-dialect lowering will replace
797 // ops which need to be lowered further, which is not supported by a
798 // single conversion pass.
799 {
800 RewritePatternSet patterns(ctx);
801 populateGpuRewritePatterns(patterns);
802 populateGpuPromoteShuffleToAMDGPUPatterns(patterns, target: *targetInfo);
803 (void)applyPatternsGreedily(op: m, patterns: std::move(patterns));
804 }
805
806 LLVMTypeConverter converter(ctx, options);
807 amdgpu::populateCommonGPUTypeAndAttributeConversions(typeConverter&: converter);
808
809 RewritePatternSet llvmPatterns(ctx);
810 LLVMConversionTarget target(getContext());
811
812 llvm::SmallDenseSet<StringRef> allowedDialectsSet(allowedDialects.begin(),
813 allowedDialects.end());
814 for (Dialect *dialect : ctx->getLoadedDialects()) {
815 bool allowed = allowedDialectsSet.contains(dialect->getNamespace());
816 // Empty `allowedDialectsSet` means all dialects are allowed.
817 if (!allowedDialectsSet.empty() && !allowed)
818 continue;
819
820 auto *iface = dyn_cast<ConvertToLLVMPatternInterface>(dialect);
821 if (!iface) {
822 // Error out if dialect was explicily specified but doesn't implement
823 // conversion interface.
824 if (allowed) {
825 m.emitError()
826 << "dialect does not implement ConvertToLLVMPatternInterface: "
827 << dialect->getNamespace();
828 return signalPassFailure();
829 }
830 continue;
831 }
832
833 iface->populateConvertToLLVMConversionPatterns(target, converter,
834 llvmPatterns);
835 }
836
837 populateAMDGPUToROCDLConversionPatterns(converter, patterns&: llvmPatterns,
838 target: *targetInfo);
839 populateGpuToROCDLConversionPatterns(converter, llvmPatterns, runtime,
840 *targetInfo);
841 configureGpuToROCDLConversionLegality(target);
842 if (failed(Result: applyPartialConversion(op: m, target, patterns: std::move(llvmPatterns))))
843 signalPassFailure();
844 auto *rocdlDialect = getContext().getLoadedDialect<ROCDL::ROCDLDialect>();
845 auto reqdWorkGroupSizeAttrHelper =
846 rocdlDialect->getReqdWorkGroupSizeAttrHelper();
847 auto flatWorkGroupSizeAttrHelper =
848 rocdlDialect->getFlatWorkGroupSizeAttrHelper();
849 // Manually rewrite known block size attributes so the LLVMIR translation
850 // infrastructure can pick them up.
851 m.walk(callback: [&](LLVM::LLVMFuncOp op) {
852 if (reqdWorkGroupSizeAttrHelper.isAttrPresent(op)) {
853 auto blockSizes = reqdWorkGroupSizeAttrHelper.getAttr(op);
854 // Also set up the rocdl.flat_work_group_size attribute to prevent
855 // conflicting metadata.
856 uint32_t flatSize = 1;
857 for (uint32_t size : blockSizes.asArrayRef()) {
858 flatSize *= size;
859 }
860 StringAttr flatSizeAttr =
861 StringAttr::get(context: ctx, bytes: Twine(flatSize) + "," + Twine(flatSize));
862 flatWorkGroupSizeAttrHelper.setAttr(op, flatSizeAttr);
863 }
864 });
865 }
866};
867
868} // namespace
869
870void mlir::configureGpuToROCDLConversionLegality(ConversionTarget &target) {
871 target.addIllegalOp<func::FuncOp>();
872 target.addLegalDialect<::mlir::LLVM::LLVMDialect>();
873 target.addLegalDialect<ROCDL::ROCDLDialect>();
874 target.addIllegalDialect<gpu::GPUDialect>();
875 target.addIllegalOp<LLVM::CosOp, LLVM::ExpOp, LLVM::Exp2Op, LLVM::FCeilOp,
876 LLVM::FFloorOp, LLVM::FRemOp, LLVM::LogOp, LLVM::Log10Op,
877 LLVM::Log2Op, LLVM::PowOp, LLVM::SinOp>();
878 // These ops are legal for f32 type.
879 target.addDynamicallyLegalOp<LLVM::ExpOp, LLVM::LogOp>(callback: [](Operation *op) {
880 return any_of(Range: op->getOperandTypes(), P: llvm::IsaPred<Float32Type>);
881 });
882 // TODO: Remove once we support replacing non-root ops.
883 target.addLegalOp<gpu::YieldOp, gpu::GPUModuleOp>();
884}
885
886void mlir::populateGpuToROCDLConversionPatterns(
887 const LLVMTypeConverter &converter, RewritePatternSet &patterns,
888 mlir::gpu::amd::Runtime runtime, const ROCDL::TargetInfo &target) {
889 using gpu::index_lowering::IndexKind;
890 using gpu::index_lowering::IntrType;
891 using mlir::gpu::amd::Runtime;
892 auto *rocdlDialect =
893 converter.getContext().getLoadedDialect<ROCDL::ROCDLDialect>();
894 populateWithGenerated(patterns);
895 patterns.add<
896 gpu::index_lowering::OpLowering<gpu::ThreadIdOp, ROCDL::ThreadIdXOp,
897 ROCDL::ThreadIdYOp, ROCDL::ThreadIdZOp>>(
898 arg: converter, args: IndexKind::Block, args: IntrType::Id);
899 patterns.add<gpu::index_lowering::OpLowering<
900 gpu::BlockIdOp, ROCDL::BlockIdXOp, ROCDL::BlockIdYOp, ROCDL::BlockIdZOp>>(
901 arg: converter, args: IndexKind::Grid, args: IntrType::Id);
902 patterns.add<GPUDimOpToOcklCall<gpu::BlockDimOp>>(arg: converter,
903 args: IndexKind::Block);
904 patterns.add<GPUDimOpToOcklCall<gpu::GridDimOp>>(arg: converter, args: IndexKind::Grid);
905 patterns.add<GPUReturnOpLowering>(arg: converter);
906 patterns.add<GPUFuncOpLowering>(
907 arg: converter,
908 args: GPUFuncOpLoweringOptions{
909 /*allocaAddrSpace=*/ROCDL::ROCDLDialect::kPrivateMemoryAddressSpace,
910 /*workgroupAddrSpace=*/ROCDL::ROCDLDialect::kSharedMemoryAddressSpace,
911 .kernelAttributeName: rocdlDialect->getKernelAttrHelper().getName(),
912 .kernelBlockSizeAttributeName: rocdlDialect->getReqdWorkGroupSizeAttrHelper().getName(),
913 /*kernelClusterSizeAttributeName=*/{}});
914 if (Runtime::HIP == runtime) {
915 patterns.add<GPUPrintfOpToHIPLowering>(arg: converter);
916 } else if (Runtime::OpenCL == runtime) {
917 // Use address space = 4 to match the OpenCL definition of printf()
918 patterns.add<GPUPrintfOpToLLVMCallLowering>(arg: converter, /*addressSpace=*/args: 4);
919 }
920 // TODO: Add alignment for workgroup memory
921 patterns.add<GPUDynamicSharedMemoryOpLowering>(arg: converter);
922
923 patterns.add<GPUShuffleOpLowering, GPULaneIdOpToROCDL,
924 GPUSubgroupBroadcastOpToROCDL, GPUBallotOpToROCDL>(arg: converter);
925 patterns.add<GPUSubgroupIdOpToROCDL, GPUSubgroupSizeOpToROCDL,
926 GPUBarrierOpLowering, GPUInitializeNamedBarrierOpLowering>(
927 arg: converter, args: target);
928
929 populateMathToROCDLConversionPatterns(converter, patterns, target);
930}
931