| 1 | //===- LowerGpuOpsToROCDLOps.cpp - MLIR GPU to ROCDL lowering passes ------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements a pass to generate ROCDLIR operations for higher-level |
| 10 | // GPU operations. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h" |
| 15 | #include "mlir/Dialect/Arith/Transforms/Passes.h" |
| 16 | #include "mlir/Dialect/LLVMIR/LLVMArithOps.h" |
| 17 | #include "mlir/Dialect/LLVMIR/LLVMCallOps.h" |
| 18 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 19 | #include "mlir/Dialect/LLVMIR/LLVMConversionOps.h" |
| 20 | #include "mlir/Dialect/LLVMIR/LLVMFloatIntrinsicOps.h" |
| 21 | #include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h" |
| 22 | #include "mlir/Dialect/LLVMIR/LLVMModuleOps.h" |
| 23 | #include "mlir/Dialect/LLVMIR/LLVMTerminatorOps.h" |
| 24 | #include "mlir/Pass/Pass.h" |
| 25 | #include "mlir/Pass/PassManager.h" |
| 26 | |
| 27 | #include "mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h" |
| 28 | #include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h" |
| 29 | #include "mlir/Conversion/ConvertToLLVM/ToLLVMPass.h" |
| 30 | #include "mlir/Conversion/GPUCommon/GPUCommonPass.h" |
| 31 | #include "mlir/Conversion/LLVMCommon/ConversionTarget.h" |
| 32 | #include "mlir/Conversion/LLVMCommon/LoweringOptions.h" |
| 33 | #include "mlir/Conversion/LLVMCommon/Pattern.h" |
| 34 | #include "mlir/Conversion/LLVMCommon/TypeConverter.h" |
| 35 | #include "mlir/Conversion/MathToLLVM/MathToLLVM.h" |
| 36 | #include "mlir/Conversion/MathToROCDL/MathToROCDL.h" |
| 37 | #include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h" |
| 38 | #include "mlir/Dialect/ControlFlow/IR/ControlFlow.h" |
| 39 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 40 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 41 | #include "mlir/Dialect/GPU/Transforms/Passes.h" |
| 42 | #include "mlir/Dialect/GPU/Transforms/ROCDL/Passes.h" |
| 43 | #include "mlir/Dialect/LLVMIR/ROCDLDialect.h" |
| 44 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 45 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 46 | #include "mlir/IR/BuiltinAttributes.h" |
| 47 | #include "mlir/IR/Matchers.h" |
| 48 | #include "mlir/Transforms/DialectConversion.h" |
| 49 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 50 | |
| 51 | #include "../GPUCommon/GPUOpsLowering.h" |
| 52 | #include "../GPUCommon/IndexIntrinsicsOpLowering.h" |
| 53 | |
| 54 | namespace mlir { |
| 55 | #define GEN_PASS_DEF_CONVERTGPUOPSTOROCDLOPS |
| 56 | #include "mlir/Conversion/Passes.h.inc" |
| 57 | } // namespace mlir |
| 58 | |
| 59 | using namespace mlir; |
| 60 | |
| 61 | // Truncate or extend the result depending on the index bitwidth specified |
| 62 | // by the LLVMTypeConverter options. |
| 63 | static Value truncOrExtToLLVMType(ConversionPatternRewriter &rewriter, |
| 64 | Location loc, Value value, |
| 65 | const LLVMTypeConverter &converter) { |
| 66 | int64_t intWidth = cast<IntegerType>(Val: value.getType()).getWidth(); |
| 67 | int64_t indexBitwidth = converter.getIndexTypeBitwidth(); |
| 68 | auto indexBitwidthType = |
| 69 | IntegerType::get(context: rewriter.getContext(), width: converter.getIndexTypeBitwidth()); |
| 70 | // TODO: use <=> in C++20. |
| 71 | if (indexBitwidth > intWidth) { |
| 72 | return LLVM::SExtOp::create(builder&: rewriter, location: loc, res: indexBitwidthType, arg: value); |
| 73 | } |
| 74 | if (indexBitwidth < intWidth) { |
| 75 | return LLVM::TruncOp::create(builder&: rewriter, location: loc, res: indexBitwidthType, arg: value); |
| 76 | } |
| 77 | return value; |
| 78 | } |
| 79 | |
| 80 | /// Returns true if the given `gpu.func` can be safely called using the bare |
| 81 | /// pointer calling convention. |
| 82 | static bool canBeCalledWithBarePointers(gpu::GPUFuncOp func) { |
| 83 | bool canBeBare = true; |
| 84 | for (Type type : func.getArgumentTypes()) |
| 85 | if (auto memrefTy = dyn_cast<BaseMemRefType>(Val&: type)) |
| 86 | canBeBare &= LLVMTypeConverter::canConvertToBarePtr(type: memrefTy); |
| 87 | return canBeBare; |
| 88 | } |
| 89 | |
| 90 | static Value getLaneId(RewriterBase &rewriter, Location loc) { |
| 91 | auto int32Type = IntegerType::get(context: rewriter.getContext(), width: 32); |
| 92 | Value zero = arith::ConstantIntOp::create(builder&: rewriter, location: loc, value: 0, width: 32); |
| 93 | Value minus1 = arith::ConstantIntOp::create(builder&: rewriter, location: loc, value: -1, width: 32); |
| 94 | NamedAttribute noundef = rewriter.getNamedAttr( |
| 95 | name: LLVM::LLVMDialect::getNoUndefAttrName(), val: rewriter.getUnitAttr()); |
| 96 | NamedAttribute lowRange = rewriter.getNamedAttr( |
| 97 | name: LLVM::LLVMDialect::getRangeAttrName(), |
| 98 | val: LLVM::ConstantRangeAttr::get(context: rewriter.getContext(), lower: APInt::getZero(numBits: 32), |
| 99 | upper: APInt(32, 32))); |
| 100 | NamedAttribute highRange = rewriter.getNamedAttr( |
| 101 | name: LLVM::LLVMDialect::getRangeAttrName(), |
| 102 | val: LLVM::ConstantRangeAttr::get(context: rewriter.getContext(), lower: APInt::getZero(numBits: 32), |
| 103 | upper: APInt(32, 64))); |
| 104 | Value mbcntLo = ROCDL::MbcntLoOp::create( |
| 105 | builder&: rewriter, location: loc, res: int32Type, in0: minus1, in1: zero, /*arg_attrs=*/{}, |
| 106 | /*res_attrs=*/ |
| 107 | rewriter.getArrayAttr(value: rewriter.getDictionaryAttr(value: {noundef, lowRange}))); |
| 108 | Value laneId = ROCDL::MbcntHiOp::create( |
| 109 | builder&: rewriter, location: loc, res: int32Type, in0: minus1, in1: mbcntLo, /*arg_attrs=*/{}, |
| 110 | res_attrs: rewriter.getArrayAttr(value: rewriter.getDictionaryAttr(value: {noundef, highRange}))); |
| 111 | return laneId; |
| 112 | } |
| 113 | |
| 114 | /// Maximum number of threads per block dimension on AMD GPUs. |
| 115 | static constexpr int64_t kMaxThreadsPerBlockDim = 1024; |
| 116 | |
| 117 | /// Emits a call to an OCKL block/grid size function corresponding to |
| 118 | /// `indexKind` with argument `dim`, except that if the context around |
| 119 | /// `contextOp` gives an exact size for that dimension, return that as |
| 120 | /// an `i64` constant instead. |
| 121 | static Value getKnownOrOcklDim(RewriterBase &rewriter, |
| 122 | gpu::index_lowering::IndexKind indexKind, |
| 123 | gpu::Dimension dim, Operation *contextOp, |
| 124 | std::optional<uint32_t> opUpperBound) { |
| 125 | Location loc = contextOp->getLoc(); |
| 126 | MLIRContext *context = contextOp->getContext(); |
| 127 | |
| 128 | auto i32Ty = IntegerType::get(context, width: 32); |
| 129 | auto i64Ty = IntegerType::get(context, width: 64); |
| 130 | |
| 131 | if (std::optional<uint32_t> knownDim = |
| 132 | gpu::getKnownDimensionSizeAround(op: contextOp, kind: indexKind, dim)) |
| 133 | return LLVM::ConstantOp::create(builder&: rewriter, location: loc, |
| 134 | value: rewriter.getI64IntegerAttr(value: *knownDim)); |
| 135 | |
| 136 | int32_t dimParam = static_cast<int32_t>(dim); |
| 137 | |
| 138 | StringRef functionName; |
| 139 | switch (indexKind) { |
| 140 | case gpu::index_lowering::IndexKind::Block: |
| 141 | functionName = "__ockl_get_local_size" ; |
| 142 | break; |
| 143 | case gpu::index_lowering::IndexKind::Grid: |
| 144 | functionName = "__ockl_get_num_groups" ; |
| 145 | break; |
| 146 | case gpu::index_lowering::IndexKind::Cluster: |
| 147 | case gpu::index_lowering::IndexKind::Other: |
| 148 | llvm_unreachable("Not valid index kinds for ockl lookup" ); |
| 149 | } |
| 150 | |
| 151 | // Declare the ockl function: i64 @functionName(i32). |
| 152 | auto fnType = LLVM::LLVMFunctionType::get(result: i64Ty, arguments: {i32Ty}); |
| 153 | Operation *moduleOp = contextOp->getParentWithTrait<OpTrait::SymbolTable>(); |
| 154 | LLVM::LLVMFuncOp funcOp = |
| 155 | getOrDefineFunction(moduleOp, loc, b&: rewriter, name: functionName, type: fnType); |
| 156 | |
| 157 | // Create the call. |
| 158 | Value dimConst = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: dimParam); |
| 159 | auto callOp = |
| 160 | LLVM::CallOp::create(builder&: rewriter, location: loc, func: funcOp, args: ValueRange{dimConst}); |
| 161 | |
| 162 | LLVM::ConstantRangeAttr range; |
| 163 | if (opUpperBound) { |
| 164 | range = LLVM::ConstantRangeAttr::get( |
| 165 | context, lower: APInt(64, 1), |
| 166 | upper: APInt(64, static_cast<uint64_t>(*opUpperBound) + 1)); |
| 167 | } else if (indexKind == gpu::index_lowering::IndexKind::Block) { |
| 168 | // Set the hardware limit for block ranges as the bounds on block dim calls. |
| 169 | range = LLVM::ConstantRangeAttr::get(context, lower: APInt(64, 1), |
| 170 | upper: APInt(64, kMaxThreadsPerBlockDim + 1)); |
| 171 | } |
| 172 | if (range) { |
| 173 | callOp.setResAttrsAttr(rewriter.getArrayAttr(value: rewriter.getDictionaryAttr( |
| 174 | value: rewriter.getNamedAttr(name: LLVM::LLVMDialect::getRangeAttrName(), val: range)))); |
| 175 | } |
| 176 | return callOp.getResult(); |
| 177 | } |
| 178 | |
| 179 | static constexpr StringLiteral amdgcnDataLayout = |
| 180 | "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32" |
| 181 | "-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:" |
| 182 | "32-v32:" |
| 183 | "32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:" |
| 184 | "64-S32-A5-G1-ni:7:8:9" ; |
| 185 | |
| 186 | namespace { |
| 187 | |
| 188 | /// Lowers gpu.block_dim / gpu.grid_dim to direct __ockl_get_local_size / |
| 189 | /// __ockl_get_num_groups function calls. |
| 190 | template <typename OpTy> |
| 191 | struct GPUDimOpToOcklCall final : ConvertOpToLLVMPattern<OpTy> { |
| 192 | GPUDimOpToOcklCall(const LLVMTypeConverter &converter, |
| 193 | gpu::index_lowering::IndexKind indexKind) |
| 194 | : ConvertOpToLLVMPattern<OpTy>(converter), indexKind(indexKind) {} |
| 195 | |
| 196 | LogicalResult |
| 197 | matchAndRewrite(OpTy op, typename OpTy::Adaptor adaptor, |
| 198 | ConversionPatternRewriter &rewriter) const override { |
| 199 | Location loc = op.getLoc(); |
| 200 | |
| 201 | std::optional<uint32_t> opUpperBound; |
| 202 | if (auto bound = op.getUpperBound()) |
| 203 | opUpperBound = static_cast<uint32_t>(bound->getZExtValue()); |
| 204 | |
| 205 | Value ocklCall = getKnownOrOcklDim(rewriter, indexKind, op.getDimension(), |
| 206 | op, opUpperBound); |
| 207 | Value result = truncOrExtToLLVMType(rewriter, loc, ocklCall, |
| 208 | *this->getTypeConverter()); |
| 209 | rewriter.replaceOp(op, result); |
| 210 | return success(); |
| 211 | } |
| 212 | |
| 213 | private: |
| 214 | const gpu::index_lowering::IndexKind indexKind; |
| 215 | }; |
| 216 | |
| 217 | struct GPULaneIdOpToROCDL : ConvertOpToLLVMPattern<gpu::LaneIdOp> { |
| 218 | using ConvertOpToLLVMPattern<gpu::LaneIdOp>::ConvertOpToLLVMPattern; |
| 219 | |
| 220 | LogicalResult |
| 221 | matchAndRewrite(gpu::LaneIdOp op, gpu::LaneIdOp::Adaptor adaptor, |
| 222 | ConversionPatternRewriter &rewriter) const override { |
| 223 | Location loc = op.getLoc(); |
| 224 | MLIRContext *context = rewriter.getContext(); |
| 225 | // convert to: |
| 226 | // %mlo = call noundef range(i32 0, 32) |
| 227 | // @llvm.amdgcn.mbcnt.lo(-1, 0) |
| 228 | // followed by: |
| 229 | // %lid = call noundef range(i32 0, 64) |
| 230 | // @llvm.amdgcn.mbcnt.hi(-1, %mlo) |
| 231 | |
| 232 | Value laneId = getLaneId(rewriter, loc); |
| 233 | // Truncate or extend the result depending on the index bitwidth specified |
| 234 | // by the LLVMTypeConverter options. |
| 235 | const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth(); |
| 236 | if (indexBitwidth > 32) { |
| 237 | laneId = LLVM::SExtOp::create( |
| 238 | builder&: rewriter, location: loc, res: IntegerType::get(context, width: indexBitwidth), arg: laneId); |
| 239 | } else if (indexBitwidth < 32) { |
| 240 | laneId = LLVM::TruncOp::create( |
| 241 | builder&: rewriter, location: loc, res: IntegerType::get(context, width: indexBitwidth), arg: laneId); |
| 242 | } |
| 243 | rewriter.replaceOp(op, newValues: {laneId}); |
| 244 | return success(); |
| 245 | } |
| 246 | }; |
| 247 | |
| 248 | struct GPUSubgroupSizeOpToROCDL : ConvertOpToLLVMPattern<gpu::SubgroupSizeOp> { |
| 249 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 250 | |
| 251 | GPUSubgroupSizeOpToROCDL(const LLVMTypeConverter &converter, |
| 252 | const ROCDL::TargetInfo &target) |
| 253 | : ConvertOpToLLVMPattern<gpu::SubgroupSizeOp>(converter), target(target) { |
| 254 | } |
| 255 | |
| 256 | LogicalResult |
| 257 | matchAndRewrite(gpu::SubgroupSizeOp op, gpu::SubgroupSizeOp::Adaptor adaptor, |
| 258 | ConversionPatternRewriter &rewriter) const override { |
| 259 | LLVM::ConstantRangeAttr bounds = nullptr; |
| 260 | bool isWave64 = target.getWavefrontSize() == 64; |
| 261 | if (auto upperBoundAttr = op.getUpperBoundAttr()) { |
| 262 | bounds = rewriter.getAttr<LLVM::ConstantRangeAttr>( |
| 263 | /*bitWidth=*/args: 32, /*lower=*/args: isWave64 ? 64 : 32, |
| 264 | /*upper=*/args: op.getUpperBoundAttr().getInt() + 1); |
| 265 | } |
| 266 | Value wavefrontOp = ROCDL::WavefrontSizeOp::create( |
| 267 | builder&: rewriter, location: op.getLoc(), res: rewriter.getI32Type(), range: bounds); |
| 268 | wavefrontOp = truncOrExtToLLVMType(rewriter, loc: op.getLoc(), value: wavefrontOp, |
| 269 | converter: *getTypeConverter()); |
| 270 | rewriter.replaceOp(op, newValues: {wavefrontOp}); |
| 271 | return success(); |
| 272 | } |
| 273 | |
| 274 | const ROCDL::TargetInfo target; |
| 275 | }; |
| 276 | |
| 277 | struct GPUSubgroupIdOpToROCDL : ConvertOpToLLVMPattern<gpu::SubgroupIdOp> { |
| 278 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 279 | |
| 280 | GPUSubgroupIdOpToROCDL(const LLVMTypeConverter &converter, |
| 281 | const ROCDL::TargetInfo &target) |
| 282 | : ConvertOpToLLVMPattern<gpu::SubgroupIdOp>(converter), target(target) {} |
| 283 | |
| 284 | LogicalResult |
| 285 | matchAndRewrite(gpu::SubgroupIdOp op, gpu::SubgroupIdOp::Adaptor adaptor, |
| 286 | ConversionPatternRewriter &rewriter) const override { |
| 287 | Location loc = op.getLoc(); |
| 288 | auto int32Type = rewriter.getI32Type(); |
| 289 | |
| 290 | Value subgroupId; |
| 291 | if (target.has(feature: llvm::AMDGPU::FEAT_GFX12_INSTS)) { |
| 292 | // For gfx12+, use the hardware wave.id register directly. |
| 293 | LLVM::ConstantRangeAttr bounds; |
| 294 | if (auto upperBoundAttr = op.getUpperBoundAttr()) |
| 295 | bounds = rewriter.getAttr<LLVM::ConstantRangeAttr>( |
| 296 | /*bitWidth=*/args: 32, /*lower=*/args: 0, |
| 297 | /*upper=*/args: upperBoundAttr.getInt()); |
| 298 | subgroupId = ROCDL::WaveId::create(builder&: rewriter, location: loc, res: int32Type, range: bounds); |
| 299 | } else { |
| 300 | // For older architectures, compute: |
| 301 | // subgroup_id = linearized_thread_id / subgroup_size |
| 302 | // where linearized_thread_id = tid.x + dim.x * (tid.y + dim.y * tid.z) |
| 303 | auto tidX = ROCDL::ThreadIdXOp::create(builder&: rewriter, location: loc, resultType: int32Type); |
| 304 | auto tidY = ROCDL::ThreadIdYOp::create(builder&: rewriter, location: loc, resultType: int32Type); |
| 305 | auto tidZ = ROCDL::ThreadIdZOp::create(builder&: rewriter, location: loc, resultType: int32Type); |
| 306 | auto setBoundFromContext = [&](Operation *tidOp, gpu::Dimension dim) { |
| 307 | if (LLVM::ConstantRangeAttr range = |
| 308 | gpu::index_lowering::getIndexOpRange( |
| 309 | op, dim, opUpperBound: std::nullopt, |
| 310 | indexKind: gpu::index_lowering::IndexKind::Block, |
| 311 | intrType: gpu::index_lowering::IntrType::Id, bitWidth: 32)) |
| 312 | tidOp->setInherentAttr(name: rewriter.getStringAttr(bytes: "range" ), value: range); |
| 313 | }; |
| 314 | setBoundFromContext(tidX, gpu::Dimension::x); |
| 315 | setBoundFromContext(tidY, gpu::Dimension::y); |
| 316 | setBoundFromContext(tidZ, gpu::Dimension::z); |
| 317 | |
| 318 | auto flags = |
| 319 | LLVM::IntegerOverflowFlags::nsw | LLVM::IntegerOverflowFlags::nuw; |
| 320 | |
| 321 | auto getBlockDim = [&](gpu::Dimension dim) { |
| 322 | Value dim64 = |
| 323 | getKnownOrOcklDim(rewriter, indexKind: gpu::index_lowering::IndexKind::Block, |
| 324 | dim, contextOp: op, opUpperBound: std::nullopt); |
| 325 | Value dimTrunc = |
| 326 | LLVM::TruncOp::create(builder&: rewriter, location: loc, res: int32Type, arg: dim64, overflowFlags: flags); |
| 327 | return dimTrunc; |
| 328 | }; |
| 329 | Value dimX = getBlockDim(gpu::Dimension::x); |
| 330 | Value dimY = getBlockDim(gpu::Dimension::y); |
| 331 | |
| 332 | // linearized = tid.x + dim.x * (tid.y + dim.y * tid.z) |
| 333 | // Thread IDs and dimensions are non-negative and small, so use nuw+nsw. |
| 334 | Value dimYxTidZ = |
| 335 | LLVM::MulOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: dimY, rhs: tidZ, overflowFlags: flags); |
| 336 | Value tidYPlusDimYxTidZ = |
| 337 | LLVM::AddOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: tidY, rhs: dimYxTidZ, overflowFlags: flags); |
| 338 | Value dimXxInner = LLVM::MulOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: dimX, |
| 339 | rhs: tidYPlusDimYxTidZ, overflowFlags: flags); |
| 340 | Value linearized = LLVM::AddOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: tidX, |
| 341 | rhs: dimXxInner, overflowFlags: flags); |
| 342 | |
| 343 | Value subgroupSize = |
| 344 | ROCDL::WavefrontSizeOp::create(builder&: rewriter, location: loc, resultType: int32Type); |
| 345 | subgroupId = LLVM::UDivOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: linearized, |
| 346 | rhs: subgroupSize); |
| 347 | } |
| 348 | |
| 349 | subgroupId = |
| 350 | truncOrExtToLLVMType(rewriter, loc, value: subgroupId, converter: *getTypeConverter()); |
| 351 | rewriter.replaceOp(op, newValues: subgroupId); |
| 352 | return success(); |
| 353 | } |
| 354 | |
| 355 | const ROCDL::TargetInfo target; |
| 356 | }; |
| 357 | |
| 358 | static bool isSupportedReadLaneType(Type type) { |
| 359 | // https://llvm.org/docs/AMDGPUUsage.html#llvm-ir-intrinsics |
| 360 | if (isa<Float16Type, BFloat16Type, Float32Type, Float64Type, |
| 361 | LLVM::LLVMPointerType>(Val: type)) |
| 362 | return true; |
| 363 | |
| 364 | if (auto intType = dyn_cast<IntegerType>(Val&: type)) |
| 365 | return llvm::is_contained(Set: {16, 32, 64}, |
| 366 | Element: static_cast<int>(intType.getWidth())); |
| 367 | |
| 368 | if (auto vecType = dyn_cast<VectorType>(Val&: type)) { |
| 369 | Type elementType = vecType.getElementType(); |
| 370 | if (elementType.isInteger(width: 32)) |
| 371 | return true; |
| 372 | |
| 373 | if (vecType.getNumElements() == 2 && |
| 374 | (isa<Float16Type, BFloat16Type>(Val: elementType) || |
| 375 | elementType.isInteger(width: 16))) |
| 376 | return true; |
| 377 | } |
| 378 | |
| 379 | return false; |
| 380 | } |
| 381 | |
| 382 | struct GPUSubgroupBroadcastOpToROCDL |
| 383 | : public ConvertOpToLLVMPattern<gpu::SubgroupBroadcastOp> { |
| 384 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 385 | |
| 386 | LogicalResult |
| 387 | matchAndRewrite(gpu::SubgroupBroadcastOp op, OpAdaptor adaptor, |
| 388 | ConversionPatternRewriter &rewriter) const override { |
| 389 | Value src = adaptor.getSrc(); |
| 390 | if (isSupportedReadLaneType(type: src.getType())) { |
| 391 | Value result = createReadlaneOp(op, adaptor, rewriter, src); |
| 392 | rewriter.replaceOp(op, newValues: result); |
| 393 | return success(); |
| 394 | } |
| 395 | |
| 396 | Type i32 = rewriter.getI32Type(); |
| 397 | Location loc = op.getLoc(); |
| 398 | SmallVector<Value> decomposed; |
| 399 | if (failed(Result: LLVM::decomposeValue(builder&: rewriter, loc, src, dstType: i32, result&: decomposed, |
| 400 | /*permitVariablySizedScalars=*/true))) |
| 401 | return rewriter.notifyMatchFailure(arg&: op, |
| 402 | msg: "Unexpected decomposition failure" ); |
| 403 | |
| 404 | SmallVector<Value> results; |
| 405 | results.reserve(N: decomposed.size()); |
| 406 | for (Value v : decomposed) |
| 407 | results.emplace_back(Args: createReadlaneOp(op, adaptor, rewriter, src: v)); |
| 408 | |
| 409 | Value result = LLVM::composeValue(builder&: rewriter, loc, src: results, dstType: src.getType()); |
| 410 | rewriter.replaceOp(op, newValues: result); |
| 411 | return success(); |
| 412 | } |
| 413 | |
| 414 | private: |
| 415 | static Value createReadlaneOp(gpu::SubgroupBroadcastOp op, OpAdaptor adaptor, |
| 416 | ConversionPatternRewriter &rewriter, |
| 417 | Value src) { |
| 418 | if (adaptor.getBroadcastType() == gpu::BroadcastType::specific_lane) { |
| 419 | return ROCDL::ReadlaneOp::create(builder&: rewriter, location: op.getLoc(), res: src.getType(), |
| 420 | src0: src, src1: adaptor.getLane()); |
| 421 | } else { // first_active_lane |
| 422 | return ROCDL::ReadfirstlaneOp::create(builder&: rewriter, location: op.getLoc(), |
| 423 | res: src.getType(), src); |
| 424 | } |
| 425 | } |
| 426 | }; |
| 427 | |
| 428 | struct GPUBallotOpToROCDL : public ConvertOpToLLVMPattern<gpu::BallotOp> { |
| 429 | using ConvertOpToLLVMPattern<gpu::BallotOp>::ConvertOpToLLVMPattern; |
| 430 | |
| 431 | LogicalResult |
| 432 | matchAndRewrite(gpu::BallotOp op, gpu::BallotOp::Adaptor adaptor, |
| 433 | ConversionPatternRewriter &rewriter) const override { |
| 434 | auto intType = cast<IntegerType>(Val: op.getType()); |
| 435 | unsigned width = intType.getWidth(); |
| 436 | |
| 437 | // ROCDL ballot natively supports i32 and i64 for wavefront sizes of |
| 438 | // 32 and 64 lanes. |
| 439 | if (width != 32 && width != 64) |
| 440 | return rewriter.notifyMatchFailure( |
| 441 | arg&: op, msg: "rocdl.ballot only supports i32 and i64 result types" ); |
| 442 | |
| 443 | rewriter.replaceOpWithNewOp<ROCDL::BallotOp>(op, args: op.getType(), |
| 444 | args: adaptor.getPredicate()); |
| 445 | return success(); |
| 446 | } |
| 447 | }; |
| 448 | |
| 449 | struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> { |
| 450 | using ConvertOpToLLVMPattern<gpu::ShuffleOp>::ConvertOpToLLVMPattern; |
| 451 | |
| 452 | /// Lowers a shuffle to the corresponding ROCDL ops. |
| 453 | /// |
| 454 | /// Use the `width` argument to see if src lane is participating. |
| 455 | /// If not the dstLane would be itself. |
| 456 | /// |
| 457 | /// Shuffle with DS Bpermute: |
| 458 | /// let shflMode = [xor, up, down, idx] |
| 459 | /// let width = 32(usually warpsize), step = [1, 2, 4, 8, 16, ... , width]. |
| 460 | /// 1. curLaneId = using mbcnt.lo + mbcnt.hi |
| 461 | /// 2. widthOrZeroIfOutside = (curLaneId + width) & -width |
| 462 | /// 3. dstLane = shflMode(curLaneId, step) |
| 463 | /// 4. isActiveSrcLane = dstLane < isActiveSrcLane |
| 464 | /// 5. dstLane = isActiveSrcLane ? dstLane : curLaneId |
| 465 | /// 6. dwordAlignedDstLane = dstLane * 4 or dstLane << 2. |
| 466 | /// 7. bpermute(dwordAlignedDstLane, shfl_value). |
| 467 | /// |
| 468 | LogicalResult |
| 469 | matchAndRewrite(gpu::ShuffleOp op, OpAdaptor adaptor, |
| 470 | ConversionPatternRewriter &rewriter) const override { |
| 471 | Location loc = op->getLoc(); |
| 472 | Value initShflValue = adaptor.getValue(); |
| 473 | |
| 474 | Value srcLaneId = getLaneId(rewriter, loc); |
| 475 | |
| 476 | auto int32Type = IntegerType::get(context: rewriter.getContext(), width: 32); |
| 477 | Value width = adaptor.getWidth(); |
| 478 | Value zero = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: int32Type, value: 0); |
| 479 | Value negwidth = LLVM::SubOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: zero, rhs: width); |
| 480 | Value add = LLVM::AddOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: srcLaneId, rhs: width); |
| 481 | Value widthOrZeroIfOutside = |
| 482 | LLVM::AndOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: add, rhs: negwidth); |
| 483 | Value dstLane; |
| 484 | |
| 485 | switch (op.getMode()) { |
| 486 | case gpu::ShuffleMode::UP: |
| 487 | dstLane = LLVM::SubOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: srcLaneId, |
| 488 | rhs: adaptor.getOffset()); |
| 489 | break; |
| 490 | case gpu::ShuffleMode::DOWN: |
| 491 | dstLane = LLVM::AddOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: srcLaneId, |
| 492 | rhs: adaptor.getOffset()); |
| 493 | break; |
| 494 | case gpu::ShuffleMode::XOR: |
| 495 | dstLane = LLVM::XOrOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: srcLaneId, |
| 496 | rhs: adaptor.getOffset()); |
| 497 | break; |
| 498 | case gpu::ShuffleMode::IDX: |
| 499 | dstLane = adaptor.getOffset(); |
| 500 | break; |
| 501 | } |
| 502 | Value isActiveSrcLane = LLVM::ICmpOp::create( |
| 503 | builder&: rewriter, location: loc, predicate: LLVM::ICmpPredicate::slt, lhs: dstLane, rhs: widthOrZeroIfOutside); |
| 504 | Value selectDstLane = LLVM::SelectOp::create(builder&: rewriter, location: loc, condition: isActiveSrcLane, |
| 505 | trueValue: dstLane, falseValue: srcLaneId); |
| 506 | Value two = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: int32Type, value: 2); |
| 507 | Value dwordAlignedDstLane = |
| 508 | LLVM::ShlOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: selectDstLane, rhs: two); |
| 509 | |
| 510 | SmallVector<Value> decomposed; |
| 511 | if (failed(Result: LLVM::decomposeValue(builder&: rewriter, loc, src: initShflValue, dstType: int32Type, |
| 512 | result&: decomposed))) |
| 513 | return rewriter.notifyMatchFailure(arg&: op, |
| 514 | msg: "failed to decompose value to i32" ); |
| 515 | SmallVector<Value> swizzled; |
| 516 | for (Value v : decomposed) { |
| 517 | Value res = ROCDL::DsBpermuteOp::create(builder&: rewriter, location: loc, res: int32Type, |
| 518 | index: dwordAlignedDstLane, src: v); |
| 519 | swizzled.emplace_back(Args&: res); |
| 520 | } |
| 521 | Value shflValue = |
| 522 | LLVM::composeValue(builder&: rewriter, loc, src: swizzled, dstType: initShflValue.getType()); |
| 523 | rewriter.replaceOp(op, newValues: {shflValue, isActiveSrcLane}); |
| 524 | return success(); |
| 525 | } |
| 526 | }; |
| 527 | |
| 528 | /// Emit an LLVM fence with MMRA metadata based on the given address spaces. |
| 529 | /// If `addrSpaces` is nullopt, all memory is fenced (global + LDS). |
| 530 | static void emitFences(std::optional<ArrayAttr> addrSpaces, |
| 531 | ConversionPatternRewriter &rewriter, Location loc, |
| 532 | StringRef scope, bool before) { |
| 533 | bool fenceGlobal = false; |
| 534 | bool fenceLDS = false; |
| 535 | |
| 536 | if (addrSpaces) { |
| 537 | for (auto spaceAttr : addrSpaces->getAsRange<gpu::AddressSpaceAttr>()) { |
| 538 | switch (spaceAttr.getValue()) { |
| 539 | case gpu::AddressSpace::Global: |
| 540 | fenceGlobal = true; |
| 541 | break; |
| 542 | case gpu::AddressSpace::Workgroup: |
| 543 | fenceLDS = true; |
| 544 | break; |
| 545 | case gpu::AddressSpace::Private: |
| 546 | case gpu::AddressSpace::Constant: |
| 547 | break; |
| 548 | } |
| 549 | } |
| 550 | } else { |
| 551 | fenceGlobal = true; |
| 552 | fenceLDS = true; |
| 553 | } |
| 554 | |
| 555 | if (!fenceGlobal && !fenceLDS) |
| 556 | return; |
| 557 | |
| 558 | Attribute mmra; |
| 559 | if (fenceLDS && !fenceGlobal) |
| 560 | mmra = |
| 561 | rewriter.getAttr<LLVM::MMRATagAttr>(args: "amdgpu-synchronize-as" , args: "local" ); |
| 562 | else if (fenceGlobal && !fenceLDS) |
| 563 | mmra = |
| 564 | rewriter.getAttr<LLVM::MMRATagAttr>(args: "amdgpu-synchronize-as" , args: "global" ); |
| 565 | |
| 566 | auto ordering = |
| 567 | before ? LLVM::AtomicOrdering::release : LLVM::AtomicOrdering::acquire; |
| 568 | auto fence = LLVM::FenceOp::create(builder&: rewriter, location: loc, ordering, syncscope: scope); |
| 569 | if (mmra) |
| 570 | fence->setDiscardableAttr(name: LLVM::LLVMDialect::getMmraAttrName(), value: mmra); |
| 571 | } |
| 572 | |
| 573 | static constexpr int32_t kWholeClusterBarrierId = -3; |
| 574 | static constexpr int32_t kWholeWorkgroupBarrierId = -1; |
| 575 | struct GPUBarrierOpLowering final : ConvertOpToLLVMPattern<gpu::BarrierOp> { |
| 576 | GPUBarrierOpLowering(const LLVMTypeConverter &converter, |
| 577 | const ROCDL::TargetInfo &target) |
| 578 | : ConvertOpToLLVMPattern<gpu::BarrierOp>(converter), target(target) {} |
| 579 | |
| 580 | ROCDL::TargetInfo target; |
| 581 | |
| 582 | LogicalResult |
| 583 | matchAndRewrite(gpu::BarrierOp op, gpu::BarrierOp::Adaptor adaptor, |
| 584 | ConversionPatternRewriter &rewriter) const override { |
| 585 | Location loc = op.getLoc(); |
| 586 | gpu::BarrierScope scope = op.getScope(); |
| 587 | |
| 588 | // Subgroup (wave) scope. |
| 589 | if (scope == gpu::BarrierScope::Subgroup) { |
| 590 | emitFences(addrSpaces: op.getAddressSpaces(), rewriter, loc, scope: "wavefront" , |
| 591 | /*before=*/true); |
| 592 | ROCDL::WaveBarrierOp::create(builder&: rewriter, location: loc); |
| 593 | emitFences(addrSpaces: op.getAddressSpaces(), rewriter, loc, scope: "wavefront" , |
| 594 | /*before=*/false); |
| 595 | rewriter.eraseOp(op); |
| 596 | return success(); |
| 597 | } |
| 598 | |
| 599 | // Cluster scope: gfx1250+ only, signal/wait with constant -3. |
| 600 | if (scope == gpu::BarrierScope::Cluster) { |
| 601 | if (!target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS)) |
| 602 | return op.emitOpError(message: "cluster scope barriers require gfx1250+" ); |
| 603 | emitFences(addrSpaces: op.getAddressSpaces(), rewriter, loc, scope: "cluster" , |
| 604 | /*before=*/true); |
| 605 | ROCDL::BarrierSignalOp::create(builder&: rewriter, location: loc, id: kWholeClusterBarrierId); |
| 606 | ROCDL::BarrierWaitOp::create( |
| 607 | builder&: rewriter, location: loc, id: static_cast<int16_t>(kWholeClusterBarrierId)); |
| 608 | emitFences(addrSpaces: op.getAddressSpaces(), rewriter, loc, scope: "cluster" , |
| 609 | /*before=*/false); |
| 610 | rewriter.eraseOp(op); |
| 611 | return success(); |
| 612 | } |
| 613 | |
| 614 | // Workgroup scope (default). |
| 615 | assert(scope == gpu::BarrierScope::Workgroup && "unsupported scope" ); |
| 616 | |
| 617 | // Named barrier path. |
| 618 | if (Value namedBarrier = adaptor.getNamedBarrier()) { |
| 619 | if (!target.has(feature: llvm::AMDGPU::FEAT_GFX12_INSTS)) |
| 620 | return op.emitOpError(message: "named barriers require gfx12+" ); |
| 621 | |
| 622 | emitFences(addrSpaces: op.getAddressSpaces(), rewriter, loc, scope: "workgroup" , |
| 623 | /*before=*/true); |
| 624 | // A wave must join the named barrier before it may signal it. |
| 625 | ROCDL::BarrierJoinOp::create(builder&: rewriter, location: loc, ptr: namedBarrier); |
| 626 | // Signal with memberCnt=0 retains the count from s.barrier.init. |
| 627 | ROCDL::BarrierSignalVarOp::create(builder&: rewriter, location: loc, ptr: namedBarrier, |
| 628 | /*memberCnt=*/0); |
| 629 | // id=1 selects the named-barrier wait class; the actual barrier waited |
| 630 | // on is the last one this wave joined. |
| 631 | ROCDL::BarrierWaitOp::create(builder&: rewriter, location: loc, id: static_cast<int16_t>(1)); |
| 632 | emitFences(addrSpaces: op.getAddressSpaces(), rewriter, loc, scope: "workgroup" , |
| 633 | /*before=*/false); |
| 634 | rewriter.eraseOp(op); |
| 635 | return success(); |
| 636 | } |
| 637 | |
| 638 | // Regular workgroup barrier. |
| 639 | emitFences(addrSpaces: op.getAddressSpaces(), rewriter, loc, scope: "workgroup" , |
| 640 | /*before=*/true); |
| 641 | if (!target.has(feature: llvm::AMDGPU::FEAT_GFX12_INSTS)) { |
| 642 | ROCDL::SBarrierOp::create(builder&: rewriter, location: loc); |
| 643 | } else { |
| 644 | ROCDL::BarrierSignalOp::create(builder&: rewriter, location: loc, id: kWholeWorkgroupBarrierId); |
| 645 | ROCDL::BarrierWaitOp::create( |
| 646 | builder&: rewriter, location: loc, id: static_cast<int16_t>(kWholeWorkgroupBarrierId)); |
| 647 | } |
| 648 | emitFences(addrSpaces: op.getAddressSpaces(), rewriter, loc, scope: "workgroup" , |
| 649 | /*before=*/false); |
| 650 | rewriter.eraseOp(op); |
| 651 | return success(); |
| 652 | } |
| 653 | }; |
| 654 | |
| 655 | struct GPUInitializeNamedBarrierOpLowering final |
| 656 | : ConvertOpToLLVMPattern<gpu::InitializeNamedBarrierOp> { |
| 657 | GPUInitializeNamedBarrierOpLowering(const LLVMTypeConverter &converter, |
| 658 | const ROCDL::TargetInfo &target) |
| 659 | : ConvertOpToLLVMPattern<gpu::InitializeNamedBarrierOp>(converter), |
| 660 | target(target) {} |
| 661 | |
| 662 | ROCDL::TargetInfo target; |
| 663 | |
| 664 | LogicalResult |
| 665 | matchAndRewrite(gpu::InitializeNamedBarrierOp op, |
| 666 | gpu::InitializeNamedBarrierOp::Adaptor adaptor, |
| 667 | ConversionPatternRewriter &rewriter) const override { |
| 668 | if (!target.has(feature: llvm::AMDGPU::FEAT_GFX12_INSTS)) |
| 669 | return op.emitOpError(message: "named barriers require gfx12+" ); |
| 670 | |
| 671 | Location loc = op.getLoc(); |
| 672 | |
| 673 | // The count must be a constant for rocdl.s.barrier.init. |
| 674 | IntegerAttr countAttr; |
| 675 | if (!matchPattern(value: op.getMemberCount(), pattern: m_Constant(bind_value: &countAttr))) |
| 676 | return op.emitOpError( |
| 677 | message: "named barrier member count must be a constant for ROCDL lowering" ); |
| 678 | int32_t count = countAttr.getInt(); |
| 679 | |
| 680 | // Place the global in the symbol-table scope enclosing the function-like |
| 681 | // op that contains this barrier (typically a module). |
| 682 | auto funcOp = op->getParentOfType<FunctionOpInterface>(); |
| 683 | if (!funcOp) |
| 684 | return op.emitOpError(message: "must be inside a function-like op" ); |
| 685 | Operation *symbolTableOp = |
| 686 | funcOp->getParentWithTrait<OpTrait::SymbolTable>(); |
| 687 | if (!symbolTableOp) |
| 688 | return op.emitOpError( |
| 689 | message: "enclosing function-like op must have a symbol-table parent" ); |
| 690 | |
| 691 | auto targetTy = LLVM::LLVMTargetExtType::get( |
| 692 | context: rewriter.getContext(), extTypeName: "amdgcn.named.barrier" , typeParams: {}, intParams: {0}); |
| 693 | auto ptrTy = LLVM::LLVMPointerType::get( |
| 694 | context: rewriter.getContext(), addressSpace: ROCDL::ROCDLDialect::kBarrierAddressSpace); |
| 695 | |
| 696 | // Build the global detached so SymbolTable::insert can both place it and |
| 697 | // rename it as needed without creating a transient name conflict in IR. |
| 698 | OpBuilder detachedBuilder(rewriter.getContext()); |
| 699 | auto globalOp = LLVM::GlobalOp::create( |
| 700 | builder&: detachedBuilder, location: loc, type: targetTy, /*isConstant=*/false, |
| 701 | linkage: LLVM::Linkage::Internal, name: "__named_barrier" , /*value=*/Attribute(), |
| 702 | /*alignment=*/0, |
| 703 | /*addrSpace=*/ROCDL::ROCDLDialect::kBarrierAddressSpace); |
| 704 | // Initialize with poison. |
| 705 | { |
| 706 | Region ®ion = globalOp.getInitializerRegion(); |
| 707 | Block *block = detachedBuilder.createBlock(parent: ®ion); |
| 708 | detachedBuilder.setInsertionPointToStart(block); |
| 709 | auto poison = LLVM::PoisonOp::create(builder&: detachedBuilder, location: loc, res: targetTy); |
| 710 | LLVM::ReturnOp::create(builder&: detachedBuilder, location: loc, arg: poison); |
| 711 | } |
| 712 | // SymbolTable::insert places the op in the symbol-table body and renames |
| 713 | // the symbol to avoid collisions with any existing entries. |
| 714 | StringAttr globalName = SymbolTable(symbolTableOp).insert(symbol: globalOp); |
| 715 | |
| 716 | // Get address of the global. |
| 717 | rewriter.setInsertionPoint(op); |
| 718 | auto addrOf = LLVM::AddressOfOp::create(builder&: rewriter, location: loc, res: ptrTy, global_name: globalName); |
| 719 | |
| 720 | // Initialize the barrier. |
| 721 | ROCDL::BarrierInitOp::create(builder&: rewriter, location: loc, ptr: addrOf, memberCnt: count); |
| 722 | |
| 723 | rewriter.replaceOp(op, newValues: addrOf.getResult()); |
| 724 | return success(); |
| 725 | } |
| 726 | }; |
| 727 | |
| 728 | /// Import the GPU Ops to ROCDL Patterns. |
| 729 | #include "GPUToROCDL.cpp.inc" |
| 730 | |
| 731 | // A pass that replaces all occurrences of GPU device operations with their |
| 732 | // corresponding ROCDL equivalent. |
| 733 | // |
| 734 | // This pass only handles device code and is not meant to be run on GPU host |
| 735 | // code. |
| 736 | struct LowerGpuOpsToROCDLOpsPass final |
| 737 | : public impl::ConvertGpuOpsToROCDLOpsBase<LowerGpuOpsToROCDLOpsPass> { |
| 738 | using Base::Base; |
| 739 | |
| 740 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 741 | Base::getDependentDialects(registry); |
| 742 | registerConvertToLLVMDependentDialectLoading(registry); |
| 743 | } |
| 744 | |
| 745 | void runOnOperation() override { |
| 746 | gpu::GPUModuleOp m = getOperation(); |
| 747 | MLIRContext *ctx = m.getContext(); |
| 748 | |
| 749 | auto llvmDataLayout = m->getDiscardableAttrOfType<StringAttr>( |
| 750 | name: LLVM::LLVMDialect::getDataLayoutAttrName()); |
| 751 | if (!llvmDataLayout) { |
| 752 | llvmDataLayout = StringAttr::get(context: ctx, bytes: amdgcnDataLayout); |
| 753 | m->setDiscardableAttr(LLVM::LLVMDialect::getDataLayoutAttrName(), |
| 754 | llvmDataLayout); |
| 755 | } |
| 756 | // Request C wrapper emission for externally visible functions only. A |
| 757 | // private function cannot be called from outside its module, so its C |
| 758 | // interface wrapper is unreachable and would only anchor the wrapped |
| 759 | // function against dead-code elimination. |
| 760 | for (auto func : m.getOps<func::FuncOp>()) { |
| 761 | if (func.isPrivate()) |
| 762 | continue; |
| 763 | func->setDiscardableAttr(LLVM::LLVMDialect::getEmitCWrapperAttrName(), |
| 764 | UnitAttr::get(ctx)); |
| 765 | } |
| 766 | |
| 767 | FailureOr<ROCDL::TargetInfo> targetInfo = ROCDL::TargetInfo::get( |
| 768 | ROCDL::resolveArchOption(arch, chipset), waveSize, |
| 769 | [&] { return emitError(UnknownLoc::get(ctx)); }); |
| 770 | if (failed(Result: targetInfo)) |
| 771 | return signalPassFailure(); |
| 772 | |
| 773 | /// Customize the bitwidth used for the device side index computations. |
| 774 | LowerToLLVMOptions options( |
| 775 | ctx, DataLayout(cast<DataLayoutOpInterface>(Val: m.getOperation()))); |
| 776 | options.dataLayout = llvm::DataLayout(llvmDataLayout.getValue()); |
| 777 | if (indexBitwidth != kDeriveIndexBitwidthFromDataLayout) |
| 778 | options.overrideIndexBitwidth(indexBitwidth); |
| 779 | |
| 780 | if (useBarePtrCallConv) { |
| 781 | options.useBarePtrCallConv = true; |
| 782 | WalkResult canUseBarePointers = |
| 783 | m.walk(callback: [](gpu::GPUFuncOp func) -> WalkResult { |
| 784 | if (canBeCalledWithBarePointers(func)) |
| 785 | return WalkResult::advance(); |
| 786 | return WalkResult::interrupt(); |
| 787 | }); |
| 788 | if (canUseBarePointers.wasInterrupted()) { |
| 789 | emitError(loc: UnknownLoc::get(context: ctx), |
| 790 | message: "bare pointer calling convention requires all memrefs to " |
| 791 | "have static shape and use the identity map" ); |
| 792 | return signalPassFailure(); |
| 793 | } |
| 794 | } |
| 795 | |
| 796 | // Apply in-dialect lowering. In-dialect lowering will replace |
| 797 | // ops which need to be lowered further, which is not supported by a |
| 798 | // single conversion pass. |
| 799 | { |
| 800 | RewritePatternSet patterns(ctx); |
| 801 | populateGpuRewritePatterns(patterns); |
| 802 | populateGpuPromoteShuffleToAMDGPUPatterns(patterns, target: *targetInfo); |
| 803 | (void)applyPatternsGreedily(op: m, patterns: std::move(patterns)); |
| 804 | } |
| 805 | |
| 806 | LLVMTypeConverter converter(ctx, options); |
| 807 | amdgpu::populateCommonGPUTypeAndAttributeConversions(typeConverter&: converter); |
| 808 | |
| 809 | RewritePatternSet llvmPatterns(ctx); |
| 810 | LLVMConversionTarget target(getContext()); |
| 811 | |
| 812 | llvm::SmallDenseSet<StringRef> allowedDialectsSet(allowedDialects.begin(), |
| 813 | allowedDialects.end()); |
| 814 | for (Dialect *dialect : ctx->getLoadedDialects()) { |
| 815 | bool allowed = allowedDialectsSet.contains(dialect->getNamespace()); |
| 816 | // Empty `allowedDialectsSet` means all dialects are allowed. |
| 817 | if (!allowedDialectsSet.empty() && !allowed) |
| 818 | continue; |
| 819 | |
| 820 | auto *iface = dyn_cast<ConvertToLLVMPatternInterface>(dialect); |
| 821 | if (!iface) { |
| 822 | // Error out if dialect was explicily specified but doesn't implement |
| 823 | // conversion interface. |
| 824 | if (allowed) { |
| 825 | m.emitError() |
| 826 | << "dialect does not implement ConvertToLLVMPatternInterface: " |
| 827 | << dialect->getNamespace(); |
| 828 | return signalPassFailure(); |
| 829 | } |
| 830 | continue; |
| 831 | } |
| 832 | |
| 833 | iface->populateConvertToLLVMConversionPatterns(target, converter, |
| 834 | llvmPatterns); |
| 835 | } |
| 836 | |
| 837 | populateAMDGPUToROCDLConversionPatterns(converter, patterns&: llvmPatterns, |
| 838 | target: *targetInfo); |
| 839 | populateGpuToROCDLConversionPatterns(converter, llvmPatterns, runtime, |
| 840 | *targetInfo); |
| 841 | configureGpuToROCDLConversionLegality(target); |
| 842 | if (failed(Result: applyPartialConversion(op: m, target, patterns: std::move(llvmPatterns)))) |
| 843 | signalPassFailure(); |
| 844 | auto *rocdlDialect = getContext().getLoadedDialect<ROCDL::ROCDLDialect>(); |
| 845 | auto reqdWorkGroupSizeAttrHelper = |
| 846 | rocdlDialect->getReqdWorkGroupSizeAttrHelper(); |
| 847 | auto flatWorkGroupSizeAttrHelper = |
| 848 | rocdlDialect->getFlatWorkGroupSizeAttrHelper(); |
| 849 | // Manually rewrite known block size attributes so the LLVMIR translation |
| 850 | // infrastructure can pick them up. |
| 851 | m.walk(callback: [&](LLVM::LLVMFuncOp op) { |
| 852 | if (reqdWorkGroupSizeAttrHelper.isAttrPresent(op)) { |
| 853 | auto blockSizes = reqdWorkGroupSizeAttrHelper.getAttr(op); |
| 854 | // Also set up the rocdl.flat_work_group_size attribute to prevent |
| 855 | // conflicting metadata. |
| 856 | uint32_t flatSize = 1; |
| 857 | for (uint32_t size : blockSizes.asArrayRef()) { |
| 858 | flatSize *= size; |
| 859 | } |
| 860 | StringAttr flatSizeAttr = |
| 861 | StringAttr::get(context: ctx, bytes: Twine(flatSize) + "," + Twine(flatSize)); |
| 862 | flatWorkGroupSizeAttrHelper.setAttr(op, flatSizeAttr); |
| 863 | } |
| 864 | }); |
| 865 | } |
| 866 | }; |
| 867 | |
| 868 | } // namespace |
| 869 | |
| 870 | void mlir::configureGpuToROCDLConversionLegality(ConversionTarget &target) { |
| 871 | target.addIllegalOp<func::FuncOp>(); |
| 872 | target.addLegalDialect<::mlir::LLVM::LLVMDialect>(); |
| 873 | target.addLegalDialect<ROCDL::ROCDLDialect>(); |
| 874 | target.addIllegalDialect<gpu::GPUDialect>(); |
| 875 | target.addIllegalOp<LLVM::CosOp, LLVM::ExpOp, LLVM::Exp2Op, LLVM::FCeilOp, |
| 876 | LLVM::FFloorOp, LLVM::FRemOp, LLVM::LogOp, LLVM::Log10Op, |
| 877 | LLVM::Log2Op, LLVM::PowOp, LLVM::SinOp>(); |
| 878 | // These ops are legal for f32 type. |
| 879 | target.addDynamicallyLegalOp<LLVM::ExpOp, LLVM::LogOp>(callback: [](Operation *op) { |
| 880 | return any_of(Range: op->getOperandTypes(), P: llvm::IsaPred<Float32Type>); |
| 881 | }); |
| 882 | // TODO: Remove once we support replacing non-root ops. |
| 883 | target.addLegalOp<gpu::YieldOp, gpu::GPUModuleOp>(); |
| 884 | } |
| 885 | |
| 886 | void mlir::populateGpuToROCDLConversionPatterns( |
| 887 | const LLVMTypeConverter &converter, RewritePatternSet &patterns, |
| 888 | mlir::gpu::amd::Runtime runtime, const ROCDL::TargetInfo &target) { |
| 889 | using gpu::index_lowering::IndexKind; |
| 890 | using gpu::index_lowering::IntrType; |
| 891 | using mlir::gpu::amd::Runtime; |
| 892 | auto *rocdlDialect = |
| 893 | converter.getContext().getLoadedDialect<ROCDL::ROCDLDialect>(); |
| 894 | populateWithGenerated(patterns); |
| 895 | patterns.add< |
| 896 | gpu::index_lowering::OpLowering<gpu::ThreadIdOp, ROCDL::ThreadIdXOp, |
| 897 | ROCDL::ThreadIdYOp, ROCDL::ThreadIdZOp>>( |
| 898 | arg: converter, args: IndexKind::Block, args: IntrType::Id); |
| 899 | patterns.add<gpu::index_lowering::OpLowering< |
| 900 | gpu::BlockIdOp, ROCDL::BlockIdXOp, ROCDL::BlockIdYOp, ROCDL::BlockIdZOp>>( |
| 901 | arg: converter, args: IndexKind::Grid, args: IntrType::Id); |
| 902 | patterns.add<GPUDimOpToOcklCall<gpu::BlockDimOp>>(arg: converter, |
| 903 | args: IndexKind::Block); |
| 904 | patterns.add<GPUDimOpToOcklCall<gpu::GridDimOp>>(arg: converter, args: IndexKind::Grid); |
| 905 | patterns.add<GPUReturnOpLowering>(arg: converter); |
| 906 | patterns.add<GPUFuncOpLowering>( |
| 907 | arg: converter, |
| 908 | args: GPUFuncOpLoweringOptions{ |
| 909 | /*allocaAddrSpace=*/ROCDL::ROCDLDialect::kPrivateMemoryAddressSpace, |
| 910 | /*workgroupAddrSpace=*/ROCDL::ROCDLDialect::kSharedMemoryAddressSpace, |
| 911 | .kernelAttributeName: rocdlDialect->getKernelAttrHelper().getName(), |
| 912 | .kernelBlockSizeAttributeName: rocdlDialect->getReqdWorkGroupSizeAttrHelper().getName(), |
| 913 | /*kernelClusterSizeAttributeName=*/{}}); |
| 914 | if (Runtime::HIP == runtime) { |
| 915 | patterns.add<GPUPrintfOpToHIPLowering>(arg: converter); |
| 916 | } else if (Runtime::OpenCL == runtime) { |
| 917 | // Use address space = 4 to match the OpenCL definition of printf() |
| 918 | patterns.add<GPUPrintfOpToLLVMCallLowering>(arg: converter, /*addressSpace=*/args: 4); |
| 919 | } |
| 920 | // TODO: Add alignment for workgroup memory |
| 921 | patterns.add<GPUDynamicSharedMemoryOpLowering>(arg: converter); |
| 922 | |
| 923 | patterns.add<GPUShuffleOpLowering, GPULaneIdOpToROCDL, |
| 924 | GPUSubgroupBroadcastOpToROCDL, GPUBallotOpToROCDL>(arg: converter); |
| 925 | patterns.add<GPUSubgroupIdOpToROCDL, GPUSubgroupSizeOpToROCDL, |
| 926 | GPUBarrierOpLowering, GPUInitializeNamedBarrierOpLowering>( |
| 927 | arg: converter, args: target); |
| 928 | |
| 929 | populateMathToROCDLConversionPatterns(converter, patterns, target); |
| 930 | } |
| 931 | |