| 1 | //===- LowerGpuOpsToNVVMOps.cpp - MLIR GPU to NVVM lowering passes --------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements a pass to generate NVVMIR operations for higher-level |
| 10 | // GPU operations. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h" |
| 15 | #include "mlir/Conversion/ConvertToLLVM/ToLLVMPass.h" |
| 16 | #include "mlir/Conversion/GPUCommon/GPUCommonPass.h" |
| 17 | #include "mlir/Conversion/GPUToNVVM/GPUToNVVM.h" |
| 18 | #include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h" |
| 19 | #include "mlir/Conversion/LLVMCommon/ConversionTarget.h" |
| 20 | #include "mlir/Conversion/LLVMCommon/LoweringOptions.h" |
| 21 | #include "mlir/Conversion/LLVMCommon/TypeConverter.h" |
| 22 | #include "mlir/Conversion/MathToNVVM/MathToNVVM.h" |
| 23 | #include "mlir/Conversion/NVGPUToNVVM/NVGPUToNVVM.h" |
| 24 | #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" |
| 25 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 26 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 27 | #include "mlir/Dialect/GPU/Transforms/Passes.h" |
| 28 | #include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h" |
| 29 | #include "mlir/Dialect/LLVMIR/LLVMArithOps.h" |
| 30 | #include "mlir/Dialect/LLVMIR/LLVMCallOps.h" |
| 31 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 32 | #include "mlir/Dialect/LLVMIR/LLVMConversionOps.h" |
| 33 | #include "mlir/Dialect/LLVMIR/LLVMFloatIntrinsicOps.h" |
| 34 | #include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h" |
| 35 | #include "mlir/Dialect/LLVMIR/LLVMModuleOps.h" |
| 36 | #include "mlir/Dialect/LLVMIR/NVVMDialect.h" |
| 37 | #include "mlir/Dialect/Math/IR/Math.h" |
| 38 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 39 | #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h" |
| 40 | #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h" |
| 41 | #include "mlir/IR/SymbolTable.h" |
| 42 | #include "mlir/Interfaces/FunctionInterfaces.h" |
| 43 | #include "mlir/Transforms/DialectConversion.h" |
| 44 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 45 | |
| 46 | #include "../GPUCommon/GPUOpsLowering.h" |
| 47 | #include "../GPUCommon/IndexIntrinsicsOpLowering.h" |
| 48 | #include "../GPUCommon/OpToFuncCallLowering.h" |
| 49 | #include <optional> |
| 50 | |
| 51 | namespace mlir { |
| 52 | #define GEN_PASS_DEF_CONVERTGPUOPSTONVVMOPS |
| 53 | #include "mlir/Conversion/Passes.h.inc" |
| 54 | } // namespace mlir |
| 55 | |
| 56 | using namespace mlir; |
| 57 | |
| 58 | namespace { |
| 59 | |
| 60 | /// Convert gpu dialect shfl mode enum to the equivalent nvvm one. |
| 61 | static NVVM::ShflKind convertShflKind(gpu::ShuffleMode mode) { |
| 62 | switch (mode) { |
| 63 | case gpu::ShuffleMode::XOR: |
| 64 | return NVVM::ShflKind::bfly; |
| 65 | case gpu::ShuffleMode::UP: |
| 66 | return NVVM::ShflKind::up; |
| 67 | case gpu::ShuffleMode::DOWN: |
| 68 | return NVVM::ShflKind::down; |
| 69 | case gpu::ShuffleMode::IDX: |
| 70 | return NVVM::ShflKind::idx; |
| 71 | } |
| 72 | llvm_unreachable("unknown shuffle mode" ); |
| 73 | } |
| 74 | |
| 75 | static std::optional<NVVM::ReductionKind> |
| 76 | convertToNVVMReductionKind(gpu::AllReduceOperation mode) { |
| 77 | switch (mode) { |
| 78 | case gpu::AllReduceOperation::ADD: |
| 79 | return NVVM::ReductionKind::ADD; |
| 80 | case gpu::AllReduceOperation::MUL: |
| 81 | return std::nullopt; |
| 82 | case gpu::AllReduceOperation::MINSI: |
| 83 | return NVVM::ReductionKind::MIN; |
| 84 | case gpu::AllReduceOperation::MINUI: |
| 85 | return std::nullopt; |
| 86 | case gpu::AllReduceOperation::MINNUMF: |
| 87 | return NVVM::ReductionKind::MIN; |
| 88 | case gpu::AllReduceOperation::MAXSI: |
| 89 | return NVVM::ReductionKind::MAX; |
| 90 | case gpu::AllReduceOperation::MAXUI: |
| 91 | return std::nullopt; |
| 92 | case gpu::AllReduceOperation::MAXNUMF: |
| 93 | return NVVM::ReductionKind::MAX; |
| 94 | case gpu::AllReduceOperation::AND: |
| 95 | return NVVM::ReductionKind::AND; |
| 96 | case gpu::AllReduceOperation::OR: |
| 97 | return NVVM::ReductionKind::OR; |
| 98 | case gpu::AllReduceOperation::XOR: |
| 99 | return NVVM::ReductionKind::XOR; |
| 100 | case gpu::AllReduceOperation::MINIMUMF: |
| 101 | case gpu::AllReduceOperation::MAXIMUMF: |
| 102 | case gpu::AllReduceOperation::MINIMUMNUMF: |
| 103 | case gpu::AllReduceOperation::MAXIMUMNUMF: |
| 104 | return std::nullopt; |
| 105 | } |
| 106 | return std::nullopt; |
| 107 | } |
| 108 | |
| 109 | static constexpr llvm::StringLiteral kNVVMNamedBarrierIdPrefix = |
| 110 | "__named_barrier_id" ; |
| 111 | static constexpr int32_t kNVVMFirstNamedBarrierId = 1; |
| 112 | static constexpr int32_t kNVVMLastNamedBarrierId = 15; |
| 113 | static constexpr int32_t kNVVMWarpSize = 32; |
| 114 | |
| 115 | static FailureOr<StringAttr> |
| 116 | createNVVMNamedBarrierIdGlobal(gpu::InitializeNamedBarrierOp op, |
| 117 | ConversionPatternRewriter &rewriter) { |
| 118 | auto funcOp = op->getParentOfType<FunctionOpInterface>(); |
| 119 | if (!funcOp) { |
| 120 | op.emitOpError(message: "must be inside a function-like op" ); |
| 121 | return failure(); |
| 122 | } |
| 123 | Operation *symbolTableOp = funcOp->getParentWithTrait<OpTrait::SymbolTable>(); |
| 124 | if (!symbolTableOp) { |
| 125 | op.emitOpError( |
| 126 | message: "enclosing function-like op must have a symbol-table parent" ); |
| 127 | return failure(); |
| 128 | } |
| 129 | |
| 130 | int32_t numNamedBarriers = 0; |
| 131 | for (auto globalOp : |
| 132 | symbolTableOp->getRegion(index: 0).front().getOps<LLVM::GlobalOp>()) |
| 133 | if (globalOp.getSymName().starts_with(Prefix: kNVVMNamedBarrierIdPrefix)) |
| 134 | ++numNamedBarriers; |
| 135 | |
| 136 | int32_t barrierId = kNVVMFirstNamedBarrierId + numNamedBarriers; |
| 137 | if (barrierId > kNVVMLastNamedBarrierId) { |
| 138 | op.emitOpError(message: "NVVM supports at most 15 named barriers per CTA" ); |
| 139 | return failure(); |
| 140 | } |
| 141 | |
| 142 | OpBuilder detachedBuilder(rewriter.getContext()); |
| 143 | Type i32 = rewriter.getI32Type(); |
| 144 | auto globalOp = LLVM::GlobalOp::create( |
| 145 | builder&: detachedBuilder, location: op.getLoc(), type: i32, /*isConstant=*/true, |
| 146 | linkage: LLVM::Linkage::Internal, name: kNVVMNamedBarrierIdPrefix, |
| 147 | value: rewriter.getI32IntegerAttr(value: barrierId), /*alignment=*/0, |
| 148 | /*addrSpace=*/0); |
| 149 | return SymbolTable(symbolTableOp).insert(symbol: globalOp); |
| 150 | } |
| 151 | |
| 152 | /// This pass lowers gpu.subgroup_reduce op into to the nvvm.redux op. The op |
| 153 | /// must be run by the entire subgroup, otherwise it is undefined behaviour. |
| 154 | struct GPUSubgroupReduceOpLowering |
| 155 | : public ConvertOpToLLVMPattern<gpu::SubgroupReduceOp> { |
| 156 | using ConvertOpToLLVMPattern<gpu::SubgroupReduceOp>::ConvertOpToLLVMPattern; |
| 157 | LogicalResult |
| 158 | |
| 159 | matchAndRewrite(gpu::SubgroupReduceOp op, OpAdaptor adaptor, |
| 160 | ConversionPatternRewriter &rewriter) const override { |
| 161 | if (op.getClusterSize()) |
| 162 | return rewriter.notifyMatchFailure( |
| 163 | arg&: op, msg: "lowering for clustered reduce not implemented" ); |
| 164 | |
| 165 | if (!op.getUniform()) |
| 166 | return rewriter.notifyMatchFailure( |
| 167 | arg&: op, msg: "cannot be lowered to redux as the op must be run " |
| 168 | "uniformly (entire subgroup)." ); |
| 169 | if (!op.getValue().getType().isInteger(width: 32)) |
| 170 | return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported data type" ); |
| 171 | |
| 172 | std::optional<NVVM::ReductionKind> mode = |
| 173 | convertToNVVMReductionKind(mode: op.getOp()); |
| 174 | if (!mode.has_value()) |
| 175 | return rewriter.notifyMatchFailure( |
| 176 | arg&: op, msg: "unsupported reduction mode for redux" ); |
| 177 | |
| 178 | Location loc = op->getLoc(); |
| 179 | auto int32Type = IntegerType::get(context: rewriter.getContext(), width: 32); |
| 180 | Value offset = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: int32Type, value: -1); |
| 181 | |
| 182 | auto reduxOp = NVVM::ReduxOp::create(builder&: rewriter, location: loc, res: int32Type, |
| 183 | val: op.getValue(), kind: mode.value(), mask_and_clamp: offset); |
| 184 | |
| 185 | rewriter.replaceOp(op, newValues: reduxOp->getResult(idx: 0)); |
| 186 | return success(); |
| 187 | } |
| 188 | }; |
| 189 | |
| 190 | struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> { |
| 191 | using ConvertOpToLLVMPattern<gpu::ShuffleOp>::ConvertOpToLLVMPattern; |
| 192 | |
| 193 | /// Lowers a shuffle to the corresponding NVVM op. |
| 194 | /// |
| 195 | /// Convert the `width` argument into an activeMask (a bitmask which specifies |
| 196 | /// which threads participate in the shuffle) and a maskAndClamp (specifying |
| 197 | /// the highest lane which participates in the shuffle). |
| 198 | /// |
| 199 | /// %one = llvm.constant(1 : i32) : i32 |
| 200 | /// %minus_one = llvm.constant(-1 : i32) : i32 |
| 201 | /// %thirty_two = llvm.constant(32 : i32) : i32 |
| 202 | /// %num_lanes = llvm.sub %thirty_two, %width : i32 |
| 203 | /// %active_mask = llvm.lshr %minus_one, %num_lanes : i32 |
| 204 | /// %mask_and_clamp = llvm.sub %width, %one : i32 |
| 205 | /// %shfl = nvvm.shfl.sync.bfly %active_mask, %value, %offset, |
| 206 | /// %mask_and_clamp : !llvm<"{ float, i1 }"> |
| 207 | /// %shfl_value = llvm.extractvalue %shfl[0] : |
| 208 | /// !llvm<"{ float, i1 }"> |
| 209 | /// %shfl_pred = llvm.extractvalue %shfl[1] : |
| 210 | /// !llvm<"{ float, i1 }"> |
| 211 | LogicalResult |
| 212 | matchAndRewrite(gpu::ShuffleOp op, OpAdaptor adaptor, |
| 213 | ConversionPatternRewriter &rewriter) const override { |
| 214 | Location loc = op->getLoc(); |
| 215 | |
| 216 | auto valueTy = adaptor.getValue().getType(); |
| 217 | auto int32Type = IntegerType::get(context: rewriter.getContext(), width: 32); |
| 218 | auto predTy = IntegerType::get(context: rewriter.getContext(), width: 1); |
| 219 | |
| 220 | Value one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: int32Type, value: 1); |
| 221 | Value minusOne = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: int32Type, value: -1); |
| 222 | Value thirtyTwo = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: int32Type, value: 32); |
| 223 | Value numLeadInactiveLane = LLVM::SubOp::create( |
| 224 | builder&: rewriter, location: loc, res: int32Type, lhs: thirtyTwo, rhs: adaptor.getWidth()); |
| 225 | // Bit mask of active lanes: `(-1) >> (32 - activeWidth)`. |
| 226 | Value activeMask = LLVM::LShrOp::create(builder&: rewriter, location: loc, res: int32Type, lhs: minusOne, |
| 227 | rhs: numLeadInactiveLane); |
| 228 | Value maskAndClamp; |
| 229 | if (op.getMode() == gpu::ShuffleMode::UP) { |
| 230 | // Clamp lane: `32 - activeWidth` |
| 231 | maskAndClamp = numLeadInactiveLane; |
| 232 | } else { |
| 233 | // Clamp lane: `activeWidth - 1` |
| 234 | maskAndClamp = LLVM::SubOp::create(builder&: rewriter, location: loc, res: int32Type, |
| 235 | lhs: adaptor.getWidth(), rhs: one); |
| 236 | } |
| 237 | |
| 238 | bool predIsUsed = !op->getResult(idx: 1).use_empty(); |
| 239 | UnitAttr returnValueAndIsValidAttr = nullptr; |
| 240 | Type resultTy = valueTy; |
| 241 | if (predIsUsed) { |
| 242 | returnValueAndIsValidAttr = rewriter.getUnitAttr(); |
| 243 | resultTy = LLVM::LLVMStructType::getLiteral(context: rewriter.getContext(), |
| 244 | types: {valueTy, predTy}); |
| 245 | } |
| 246 | Value shfl = NVVM::ShflOp::create( |
| 247 | builder&: rewriter, location: loc, res: resultTy, thread_mask: activeMask, val: adaptor.getValue(), |
| 248 | offset: adaptor.getOffset(), mask_and_clamp: maskAndClamp, kind: convertShflKind(mode: op.getMode()), |
| 249 | return_value_and_is_valid: returnValueAndIsValidAttr); |
| 250 | if (predIsUsed) { |
| 251 | Value shflValue = LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: shfl, position: 0); |
| 252 | Value isActiveSrcLane = |
| 253 | LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: shfl, position: 1); |
| 254 | rewriter.replaceOp(op, newValues: {shflValue, isActiveSrcLane}); |
| 255 | } else { |
| 256 | rewriter.replaceOp(op, newValues: {shfl, nullptr}); |
| 257 | } |
| 258 | return success(); |
| 259 | } |
| 260 | }; |
| 261 | |
| 262 | struct GPULaneIdOpToNVVM : ConvertOpToLLVMPattern<gpu::LaneIdOp> { |
| 263 | using ConvertOpToLLVMPattern<gpu::LaneIdOp>::ConvertOpToLLVMPattern; |
| 264 | |
| 265 | LogicalResult |
| 266 | matchAndRewrite(gpu::LaneIdOp op, gpu::LaneIdOp::Adaptor adaptor, |
| 267 | ConversionPatternRewriter &rewriter) const override { |
| 268 | auto loc = op->getLoc(); |
| 269 | MLIRContext *context = rewriter.getContext(); |
| 270 | LLVM::ConstantRangeAttr bounds = nullptr; |
| 271 | if (std::optional<APInt> upperBound = op.getUpperBound()) |
| 272 | bounds = rewriter.getAttr<LLVM::ConstantRangeAttr>( |
| 273 | /*bitWidth=*/args: 32, /*lower=*/args: 0, args: upperBound->getZExtValue()); |
| 274 | else |
| 275 | bounds = rewriter.getAttr<LLVM::ConstantRangeAttr>( |
| 276 | /*bitWidth=*/args: 32, /*lower=*/args: 0, /*upper=*/args: kWarpSize); |
| 277 | Value newOp = |
| 278 | NVVM::LaneIdOp::create(builder&: rewriter, location: loc, res: rewriter.getI32Type(), range: bounds); |
| 279 | // Truncate or extend the result depending on the index bitwidth specified |
| 280 | // by the LLVMTypeConverter options. |
| 281 | const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth(); |
| 282 | if (indexBitwidth > 32) { |
| 283 | newOp = LLVM::SExtOp::create( |
| 284 | builder&: rewriter, location: loc, res: IntegerType::get(context, width: indexBitwidth), arg: newOp); |
| 285 | } else if (indexBitwidth < 32) { |
| 286 | newOp = LLVM::TruncOp::create( |
| 287 | builder&: rewriter, location: loc, res: IntegerType::get(context, width: indexBitwidth), arg: newOp); |
| 288 | } |
| 289 | rewriter.replaceOp(op, newValues: {newOp}); |
| 290 | return success(); |
| 291 | } |
| 292 | }; |
| 293 | |
| 294 | struct GPUBallotOpToNVVM : public ConvertOpToLLVMPattern<gpu::BallotOp> { |
| 295 | using ConvertOpToLLVMPattern<gpu::BallotOp>::ConvertOpToLLVMPattern; |
| 296 | |
| 297 | LogicalResult |
| 298 | matchAndRewrite(gpu::BallotOp op, gpu::BallotOp::Adaptor adaptor, |
| 299 | ConversionPatternRewriter &rewriter) const override { |
| 300 | Location loc = op->getLoc(); |
| 301 | auto int32Type = IntegerType::get(context: rewriter.getContext(), width: 32); |
| 302 | auto intType = cast<IntegerType>(Val: op.getType()); |
| 303 | unsigned width = intType.getWidth(); |
| 304 | |
| 305 | // NVVM ballot natively returns i32. For i64 results, zero-extend since |
| 306 | // NVIDIA warps have exactly 32 threads, so upper 32 bits are always zero. |
| 307 | if (width != 32 && width != 64) |
| 308 | return rewriter.notifyMatchFailure( |
| 309 | arg&: op, msg: "nvvm.vote.sync ballot only supports i32 and i64 result types" ); |
| 310 | |
| 311 | // Use full mask (-1) so all 32 lanes participate in the ballot. |
| 312 | Value mask = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: int32Type, |
| 313 | value: rewriter.getI32IntegerAttr(value: -1)); |
| 314 | |
| 315 | auto voteKind = NVVM::VoteSyncKindAttr::get(context: rewriter.getContext(), |
| 316 | value: NVVM::VoteSyncKind::ballot); |
| 317 | Value result = NVVM::VoteSyncOp::create(builder&: rewriter, location: loc, res: int32Type, mask, |
| 318 | pred: adaptor.getPredicate(), kind: voteKind); |
| 319 | |
| 320 | if (width == 64) |
| 321 | result = LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: op.getType(), arg: result); |
| 322 | |
| 323 | rewriter.replaceOp(op, newValues: result); |
| 324 | return success(); |
| 325 | } |
| 326 | }; |
| 327 | |
| 328 | /// Lowering of cf.assert into a conditional __assertfail. |
| 329 | struct AssertOpToAssertfailLowering |
| 330 | : public ConvertOpToLLVMPattern<cf::AssertOp> { |
| 331 | using ConvertOpToLLVMPattern<cf::AssertOp>::ConvertOpToLLVMPattern; |
| 332 | |
| 333 | LogicalResult |
| 334 | matchAndRewrite(cf::AssertOp assertOp, cf::AssertOpAdaptor adaptor, |
| 335 | ConversionPatternRewriter &rewriter) const override { |
| 336 | MLIRContext *ctx = rewriter.getContext(); |
| 337 | Location loc = assertOp.getLoc(); |
| 338 | Type i8Type = typeConverter->convertType(t: rewriter.getIntegerType(width: 8)); |
| 339 | Type i32Type = typeConverter->convertType(t: rewriter.getIntegerType(width: 32)); |
| 340 | Type i64Type = typeConverter->convertType(t: rewriter.getIntegerType(width: 64)); |
| 341 | Type ptrType = LLVM::LLVMPointerType::get(context: ctx); |
| 342 | Type voidType = LLVM::LLVMVoidType::get(ctx); |
| 343 | |
| 344 | // Find or create __assertfail function declaration. |
| 345 | auto moduleOp = assertOp->getParentOfType<gpu::GPUModuleOp>(); |
| 346 | auto assertfailType = LLVM::LLVMFunctionType::get( |
| 347 | result: voidType, arguments: {ptrType, ptrType, i32Type, ptrType, i64Type}); |
| 348 | LLVM::LLVMFuncOp assertfailDecl = getOrDefineFunction( |
| 349 | moduleOp, loc, b&: rewriter, name: "__assertfail" , type: assertfailType); |
| 350 | assertfailDecl.setPassthroughAttr( |
| 351 | ArrayAttr::get(context: ctx, value: StringAttr::get(context: ctx, bytes: "noreturn" ))); |
| 352 | |
| 353 | // Split blocks and insert conditional branch. |
| 354 | // ^before: |
| 355 | // ... |
| 356 | // cf.cond_br %condition, ^after, ^assert |
| 357 | // ^assert: |
| 358 | // cf.assert |
| 359 | // cf.br ^after |
| 360 | // ^after: |
| 361 | // ... |
| 362 | Block *beforeBlock = assertOp->getBlock(); |
| 363 | Block *assertBlock = |
| 364 | rewriter.splitBlock(block: beforeBlock, before: assertOp->getIterator()); |
| 365 | Block *afterBlock = |
| 366 | rewriter.splitBlock(block: assertBlock, before: ++assertOp->getIterator()); |
| 367 | rewriter.setInsertionPointToEnd(beforeBlock); |
| 368 | cf::CondBranchOp::create(builder&: rewriter, location: loc, condition: adaptor.getArg(), trueDest: afterBlock, |
| 369 | falseDest: assertBlock); |
| 370 | rewriter.setInsertionPointToEnd(assertBlock); |
| 371 | cf::BranchOp::create(builder&: rewriter, location: loc, dest: afterBlock); |
| 372 | |
| 373 | // Continue cf.assert lowering. |
| 374 | rewriter.setInsertionPoint(assertOp); |
| 375 | |
| 376 | // Populate file name, file number and function name from the location of |
| 377 | // the AssertOp. |
| 378 | StringRef fileName = "(unknown)" ; |
| 379 | StringRef funcName = "(unknown)" ; |
| 380 | int32_t fileLine = 0; |
| 381 | while (auto callSiteLoc = dyn_cast<CallSiteLoc>(Val&: loc)) |
| 382 | loc = callSiteLoc.getCallee(); |
| 383 | if (auto fileLineColLoc = dyn_cast<FileLineColRange>(Val&: loc)) { |
| 384 | fileName = fileLineColLoc.getFilename().strref(); |
| 385 | fileLine = fileLineColLoc.getStartLine(); |
| 386 | } else if (auto nameLoc = dyn_cast<NameLoc>(Val&: loc)) { |
| 387 | funcName = nameLoc.getName().strref(); |
| 388 | if (auto fileLineColLoc = |
| 389 | dyn_cast<FileLineColRange>(Val: nameLoc.getChildLoc())) { |
| 390 | fileName = fileLineColLoc.getFilename().strref(); |
| 391 | fileLine = fileLineColLoc.getStartLine(); |
| 392 | } |
| 393 | } |
| 394 | |
| 395 | // Create constants. |
| 396 | auto getGlobal = [&](LLVM::GlobalOp global) { |
| 397 | // Get a pointer to the format string's first element. |
| 398 | Value globalPtr = LLVM::AddressOfOp::create( |
| 399 | builder&: rewriter, location: loc, res: LLVM::LLVMPointerType::get(context: ctx, addressSpace: global.getAddrSpace()), |
| 400 | global_name: global.getSymNameAttr()); |
| 401 | Value start = |
| 402 | LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: global.getGlobalType(), |
| 403 | basePtr: globalPtr, indices: ArrayRef<LLVM::GEPArg>{0, 0}); |
| 404 | return start; |
| 405 | }; |
| 406 | Value assertMessage = getGlobal(getOrCreateStringConstant( |
| 407 | b&: rewriter, loc, moduleOp, llvmI8: i8Type, namePrefix: "assert_message_" , str: assertOp.getMsg())); |
| 408 | Value assertFile = getGlobal(getOrCreateStringConstant( |
| 409 | b&: rewriter, loc, moduleOp, llvmI8: i8Type, namePrefix: "assert_file_" , str: fileName)); |
| 410 | Value assertFunc = getGlobal(getOrCreateStringConstant( |
| 411 | b&: rewriter, loc, moduleOp, llvmI8: i8Type, namePrefix: "assert_func_" , str: funcName)); |
| 412 | Value assertLine = |
| 413 | LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Type, value: fileLine); |
| 414 | Value c1 = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i64Type, value: 1); |
| 415 | |
| 416 | // Insert function call to __assertfail. |
| 417 | SmallVector<Value> arguments{assertMessage, assertFile, assertLine, |
| 418 | assertFunc, c1}; |
| 419 | rewriter.replaceOpWithNewOp<LLVM::CallOp>(op: assertOp, args&: assertfailDecl, |
| 420 | args&: arguments); |
| 421 | return success(); |
| 422 | } |
| 423 | }; |
| 424 | |
| 425 | struct GPUBarrierOpToNVVMLowering final |
| 426 | : public ConvertOpToLLVMPattern<gpu::BarrierOp> { |
| 427 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 428 | |
| 429 | LogicalResult |
| 430 | matchAndRewrite(gpu::BarrierOp op, gpu::BarrierOp::Adaptor adaptor, |
| 431 | ConversionPatternRewriter &rewriter) const override { |
| 432 | if (Value namedBarrier = adaptor.getNamedBarrier()) { |
| 433 | Location loc = op.getLoc(); |
| 434 | Value barrierId = |
| 435 | LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: namedBarrier, position: 0); |
| 436 | Value numberOfThreads = |
| 437 | LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: namedBarrier, position: 1); |
| 438 | NVVM::BarrierOp::create(builder&: rewriter, location: loc, barrierId, numberOfThreads); |
| 439 | rewriter.eraseOp(op); |
| 440 | return success(); |
| 441 | } |
| 442 | |
| 443 | gpu::BarrierScope scope = op.getScope(); |
| 444 | switch (scope) { |
| 445 | case gpu::BarrierScope::Workgroup: |
| 446 | rewriter.replaceOpWithNewOp<NVVM::BarrierOp>(op); |
| 447 | return success(); |
| 448 | case gpu::BarrierScope::Subgroup: { |
| 449 | // Emit __syncwarp(0xFFFFFFFF) for full-warp sync. |
| 450 | Value mask = |
| 451 | LLVM::ConstantOp::create(builder&: rewriter, location: op.getLoc(), res: rewriter.getI32Type(), |
| 452 | value: rewriter.getI32IntegerAttr(value: 0xFFFFFFFF)); |
| 453 | rewriter.replaceOpWithNewOp<NVVM::SyncWarpOp>(op, args&: mask); |
| 454 | return success(); |
| 455 | } |
| 456 | case gpu::BarrierScope::Cluster: { |
| 457 | UnitAttr aligned = rewriter.getUnitAttr(); |
| 458 | if (op.getAddressSpaces() && op.getAddressSpaces()->empty()) |
| 459 | NVVM::ClusterArriveRelaxedOp::create(builder&: rewriter, location: op.getLoc(), aligned); |
| 460 | else |
| 461 | NVVM::ClusterArriveOp::create(builder&: rewriter, location: op.getLoc(), aligned); |
| 462 | rewriter.replaceOpWithNewOp<NVVM::ClusterWaitOp>(op, args&: aligned); |
| 463 | return success(); |
| 464 | } |
| 465 | } |
| 466 | llvm_unreachable("unknown barrier scope" ); |
| 467 | } |
| 468 | }; |
| 469 | |
| 470 | struct GPUInitializeNamedBarrierOpToNVVMLowering final |
| 471 | : public ConvertOpToLLVMPattern<gpu::InitializeNamedBarrierOp> { |
| 472 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 473 | |
| 474 | LogicalResult |
| 475 | matchAndRewrite(gpu::InitializeNamedBarrierOp op, |
| 476 | gpu::InitializeNamedBarrierOp::Adaptor adaptor, |
| 477 | ConversionPatternRewriter &rewriter) const override { |
| 478 | Location loc = op.getLoc(); |
| 479 | MLIRContext *ctx = rewriter.getContext(); |
| 480 | Type i32 = rewriter.getI32Type(); |
| 481 | Type namedBarrierType = |
| 482 | getTypeConverter()->convertType(t: op.getResult().getType()); |
| 483 | if (!namedBarrierType) |
| 484 | return rewriter.notifyMatchFailure(arg&: op, msg: "failed to convert result type" ); |
| 485 | |
| 486 | FailureOr<StringAttr> maybeGlobalName = |
| 487 | createNVVMNamedBarrierIdGlobal(op, rewriter); |
| 488 | if (failed(Result: maybeGlobalName)) |
| 489 | return failure(); |
| 490 | |
| 491 | auto addressOf = LLVM::AddressOfOp::create( |
| 492 | builder&: rewriter, location: loc, res: LLVM::LLVMPointerType::get(context: ctx), global_name: *maybeGlobalName); |
| 493 | Value barrierId = |
| 494 | LLVM::LoadOp::create(builder&: rewriter, location: loc, type: i32, addr: addressOf.getResult()); |
| 495 | |
| 496 | Value warpSize = LLVM::ConstantOp::create( |
| 497 | builder&: rewriter, location: loc, res: i32, value: rewriter.getI32IntegerAttr(value: kNVVMWarpSize)); |
| 498 | Value numberOfThreads = |
| 499 | LLVM::MulOp::create(builder&: rewriter, location: loc, lhs: adaptor.getMemberCount(), rhs: warpSize); |
| 500 | |
| 501 | Value namedBarrier = |
| 502 | LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: namedBarrierType); |
| 503 | DenseI64ArrayAttr barrierIdPos = rewriter.getDenseI64ArrayAttr(values: {0}); |
| 504 | DenseI64ArrayAttr numberOfThreadsPos = rewriter.getDenseI64ArrayAttr(values: {1}); |
| 505 | namedBarrier = LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: namedBarrier, |
| 506 | value: barrierId, position: barrierIdPos); |
| 507 | namedBarrier = LLVM::InsertValueOp::create( |
| 508 | builder&: rewriter, location: loc, container: namedBarrier, value: numberOfThreads, position: numberOfThreadsPos); |
| 509 | rewriter.replaceOp(op, newValues: namedBarrier); |
| 510 | return success(); |
| 511 | } |
| 512 | }; |
| 513 | |
| 514 | /// A pass that replaces all occurrences of GPU device operations with their |
| 515 | /// corresponding NVVM equivalent. |
| 516 | /// |
| 517 | /// This pass only handles device code and is not meant to be run on GPU host |
| 518 | /// code. |
| 519 | struct LowerGpuOpsToNVVMOpsPass final |
| 520 | : public impl::ConvertGpuOpsToNVVMOpsBase<LowerGpuOpsToNVVMOpsPass> { |
| 521 | using Base::Base; |
| 522 | |
| 523 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 524 | Base::getDependentDialects(registry); |
| 525 | registerConvertToLLVMDependentDialectLoading(registry); |
| 526 | } |
| 527 | |
| 528 | void runOnOperation() override { |
| 529 | gpu::GPUModuleOp m = getOperation(); |
| 530 | |
| 531 | // Request C wrapper emission for externally visible functions only. A |
| 532 | // private function cannot be called from outside its module, so its C |
| 533 | // interface wrapper is unreachable and would only anchor the wrapped |
| 534 | // function against dead-code elimination. |
| 535 | for (auto func : m.getOps<func::FuncOp>()) { |
| 536 | if (func.isPrivate()) |
| 537 | continue; |
| 538 | func->setDiscardableAttr(name: LLVM::LLVMDialect::getEmitCWrapperAttrName(), |
| 539 | value: UnitAttr::get(context: &getContext())); |
| 540 | } |
| 541 | |
| 542 | // Customize the bitwidth used for the device side index computations. |
| 543 | LowerToLLVMOptions options( |
| 544 | m.getContext(), |
| 545 | DataLayout(cast<DataLayoutOpInterface>(Val: m.getOperation()))); |
| 546 | if (indexBitwidth != kDeriveIndexBitwidthFromDataLayout) |
| 547 | options.overrideIndexBitwidth(bitwidth: indexBitwidth); |
| 548 | options.useBarePtrCallConv = useBarePtrCallConv; |
| 549 | |
| 550 | // Apply in-dialect lowering. In-dialect lowering will replace |
| 551 | // ops which need to be lowered further, which is not supported by a |
| 552 | // single conversion pass. |
| 553 | { |
| 554 | RewritePatternSet patterns(m.getContext()); |
| 555 | populateGpuRewritePatterns(patterns); |
| 556 | // Transform N-D vector.from_elements to 1-D vector.from_elements before |
| 557 | // conversion. |
| 558 | vector::populateVectorFromElementsUnrollPatterns(patterns); |
| 559 | if (failed(Result: applyPatternsGreedily(op: m, patterns: std::move(patterns)))) |
| 560 | return signalPassFailure(); |
| 561 | } |
| 562 | |
| 563 | LLVMTypeConverter converter(m.getContext(), options); |
| 564 | configureGpuToNVVMTypeConverter(converter); |
| 565 | RewritePatternSet llvmPatterns(m.getContext()); |
| 566 | LLVMConversionTarget target(getContext()); |
| 567 | |
| 568 | // Set higher benefit, so patterns will run before generic LLVM lowering. |
| 569 | populateGpuToNVVMConversionPatterns(converter, patterns&: llvmPatterns, |
| 570 | /*benefit=*/10); |
| 571 | |
| 572 | llvm::SmallDenseSet<StringRef> allowedDialectsSet(allowedDialects.begin(), |
| 573 | allowedDialects.end()); |
| 574 | for (Dialect *dialect : getContext().getLoadedDialects()) { |
| 575 | // Skip math patterns as nvvm needs custom math lowering. |
| 576 | if (isa<math::MathDialect>(Val: dialect)) |
| 577 | continue; |
| 578 | |
| 579 | bool allowed = allowedDialectsSet.contains(V: dialect->getNamespace()); |
| 580 | // Empty `allowedDialectsSet` means all dialects are allowed. |
| 581 | if (!allowedDialectsSet.empty() && !allowed) |
| 582 | continue; |
| 583 | |
| 584 | auto *iface = dyn_cast<ConvertToLLVMPatternInterface>(Val: dialect); |
| 585 | if (!iface) { |
| 586 | // Error out if dialect was explicily specified but doesn't implement |
| 587 | // conversion interface. |
| 588 | if (allowed) { |
| 589 | m.emitError() |
| 590 | << "dialect does not implement ConvertToLLVMPatternInterface: " |
| 591 | << dialect->getNamespace(); |
| 592 | return signalPassFailure(); |
| 593 | } |
| 594 | continue; |
| 595 | } |
| 596 | |
| 597 | iface->populateConvertToLLVMConversionPatterns(target, typeConverter&: converter, |
| 598 | patterns&: llvmPatterns); |
| 599 | } |
| 600 | |
| 601 | populateGpuWMMAToNVVMConversionPatterns(converter, patterns&: llvmPatterns); |
| 602 | if (this->hasRedux) |
| 603 | populateGpuSubgroupReduceOpLoweringPattern(converter, patterns&: llvmPatterns); |
| 604 | configureGpuToNVVMConversionLegality(target); |
| 605 | ConversionConfig config; |
| 606 | config.allowPatternRollback = allowPatternRollback; |
| 607 | if (failed( |
| 608 | Result: applyPartialConversion(op: m, target, patterns: std::move(llvmPatterns), config))) |
| 609 | signalPassFailure(); |
| 610 | } |
| 611 | }; |
| 612 | |
| 613 | } // namespace |
| 614 | |
| 615 | void mlir::configureGpuToNVVMConversionLegality(ConversionTarget &target) { |
| 616 | target.addIllegalOp<func::FuncOp>(); |
| 617 | target.addIllegalOp<cf::AssertOp>(); |
| 618 | target.addLegalDialect<::mlir::LLVM::LLVMDialect>(); |
| 619 | target.addLegalDialect<::mlir::NVVM::NVVMDialect>(); |
| 620 | target.addIllegalDialect<gpu::GPUDialect>(); |
| 621 | target.addIllegalOp<LLVM::CopySignOp, LLVM::CosOp, LLVM::ExpOp, LLVM::Exp2Op, |
| 622 | LLVM::FAbsOp, LLVM::FCeilOp, LLVM::FFloorOp, LLVM::FRemOp, |
| 623 | LLVM::LogOp, LLVM::Log10Op, LLVM::Log2Op, LLVM::PowOp, |
| 624 | LLVM::RoundEvenOp, LLVM::RoundOp, LLVM::SinOp, |
| 625 | LLVM::SincosOp, LLVM::SqrtOp>(); |
| 626 | |
| 627 | // TODO: Remove once we support replacing non-root ops. |
| 628 | target.addLegalOp<gpu::YieldOp, gpu::GPUModuleOp>(); |
| 629 | } |
| 630 | |
| 631 | void mlir::configureGpuToNVVMTypeConverter(LLVMTypeConverter &converter) { |
| 632 | nvgpu::populateCommonGPUTypeAndAttributeConversions(typeConverter&: converter); |
| 633 | |
| 634 | converter.addConversion(callback: [&](gpu::NamedBarrierType type) -> Type { |
| 635 | Type i32 = IntegerType::get(context: type.getContext(), width: 32); |
| 636 | return LLVM::LLVMStructType::getLiteral(context: type.getContext(), types: {i32, i32}); |
| 637 | }); |
| 638 | |
| 639 | // Lowering for MMAMatrixType. |
| 640 | converter.addConversion(callback: [&](gpu::MMAMatrixType type) -> Type { |
| 641 | return convertMMAToLLVMType(type); |
| 642 | }); |
| 643 | } |
| 644 | |
| 645 | void mlir::populateGpuSubgroupReduceOpLoweringPattern( |
| 646 | const LLVMTypeConverter &converter, RewritePatternSet &patterns, |
| 647 | PatternBenefit benefit) { |
| 648 | patterns.add<GPUSubgroupReduceOpLowering>(arg: converter, args&: benefit); |
| 649 | } |
| 650 | |
| 651 | void mlir::populateGpuToNVVMConversionPatterns( |
| 652 | const LLVMTypeConverter &converter, RewritePatternSet &patterns, |
| 653 | PatternBenefit benefit) { |
| 654 | using gpu::index_lowering::IndexKind; |
| 655 | using gpu::index_lowering::IntrType; |
| 656 | |
| 657 | patterns.add<GPUBarrierOpToNVVMLowering, |
| 658 | GPUInitializeNamedBarrierOpToNVVMLowering, |
| 659 | GPUPrintfOpToVPrintfLowering, AssertOpToAssertfailLowering>( |
| 660 | arg: converter, args&: benefit); |
| 661 | patterns.add< |
| 662 | gpu::index_lowering::OpLowering<gpu::ThreadIdOp, NVVM::ThreadIdXOp, |
| 663 | NVVM::ThreadIdYOp, NVVM::ThreadIdZOp>>( |
| 664 | arg: converter, args: IndexKind::Block, args: IntrType::Id, args&: benefit); |
| 665 | patterns.add< |
| 666 | gpu::index_lowering::OpLowering<gpu::BlockDimOp, NVVM::BlockDimXOp, |
| 667 | NVVM::BlockDimYOp, NVVM::BlockDimZOp>>( |
| 668 | arg: converter, args: IndexKind::Block, args: IntrType::Dim, args&: benefit); |
| 669 | patterns.add< |
| 670 | gpu::index_lowering::OpLowering<gpu::ClusterIdOp, NVVM::ClusterIdXOp, |
| 671 | NVVM::ClusterIdYOp, NVVM::ClusterIdZOp>>( |
| 672 | arg: converter, args: IndexKind::Other, args: IntrType::Id, args&: benefit); |
| 673 | patterns.add<gpu::index_lowering::OpLowering< |
| 674 | gpu::ClusterDimOp, NVVM::ClusterDimXOp, NVVM::ClusterDimYOp, |
| 675 | NVVM::ClusterDimZOp>>(arg: converter, args: IndexKind::Other, args: IntrType::Dim, |
| 676 | args&: benefit); |
| 677 | patterns.add<gpu::index_lowering::OpLowering< |
| 678 | gpu::ClusterBlockIdOp, NVVM::BlockInClusterIdXOp, |
| 679 | NVVM::BlockInClusterIdYOp, NVVM::BlockInClusterIdZOp>>( |
| 680 | arg: converter, args: IndexKind::Cluster, args: IntrType::Id, args&: benefit); |
| 681 | patterns.add<gpu::index_lowering::OpLowering< |
| 682 | gpu::ClusterDimBlocksOp, NVVM::ClusterDimBlocksXOp, |
| 683 | NVVM::ClusterDimBlocksYOp, NVVM::ClusterDimBlocksZOp>>( |
| 684 | arg: converter, args: IndexKind::Cluster, args: IntrType::Dim, args&: benefit); |
| 685 | patterns.add<gpu::index_lowering::OpLowering< |
| 686 | gpu::BlockIdOp, NVVM::BlockIdXOp, NVVM::BlockIdYOp, NVVM::BlockIdZOp>>( |
| 687 | arg: converter, args: IndexKind::Grid, args: IntrType::Id, args&: benefit); |
| 688 | patterns.add<gpu::index_lowering::OpLowering< |
| 689 | gpu::GridDimOp, NVVM::GridDimXOp, NVVM::GridDimYOp, NVVM::GridDimZOp>>( |
| 690 | arg: converter, args: IndexKind::Grid, args: IntrType::Dim, args&: benefit); |
| 691 | patterns.add<GPULaneIdOpToNVVM, GPUBallotOpToNVVM, GPUShuffleOpLowering, |
| 692 | GPUReturnOpLowering>(arg: converter, args&: benefit); |
| 693 | |
| 694 | patterns.add<GPUDynamicSharedMemoryOpLowering>( |
| 695 | arg: converter, args: NVVM::kSharedMemoryAlignmentBit, args&: benefit); |
| 696 | |
| 697 | // Explicitly drop memory space when lowering private memory |
| 698 | // attributions since NVVM models it as `alloca`s in the default |
| 699 | // memory space and does not support `alloca`s with addrspace(5). |
| 700 | patterns.add<GPUFuncOpLowering>( |
| 701 | arg: converter, |
| 702 | args: GPUFuncOpLoweringOptions{ |
| 703 | /*allocaAddrSpace=*/0, |
| 704 | /*workgroupAddrSpace=*/ |
| 705 | static_cast<unsigned>(NVVM::NVVMMemorySpace::Shared), |
| 706 | .kernelAttributeName: StringAttr::get(context: &converter.getContext(), |
| 707 | bytes: NVVM::NVVMDialect::getKernelFuncAttrName()), |
| 708 | .kernelBlockSizeAttributeName: StringAttr::get(context: &converter.getContext(), |
| 709 | bytes: NVVM::NVVMDialect::getMaxntidAttrName()), |
| 710 | .kernelClusterSizeAttributeName: StringAttr::get(context: &converter.getContext(), |
| 711 | bytes: NVVM::NVVMDialect::getClusterDimAttrName())}, |
| 712 | args&: benefit); |
| 713 | |
| 714 | populateLibDeviceConversionPatterns(converter, patterns, benefit); |
| 715 | } |
| 716 | |
| 717 | //===----------------------------------------------------------------------===// |
| 718 | // NVVMTargetAttr convert to LLVM attr interface |
| 719 | //===----------------------------------------------------------------------===// |
| 720 | |
| 721 | namespace { |
| 722 | struct NVVMTargetConvertToLLVMAttrInterface |
| 723 | : public ConvertToLLVMAttrInterface::ExternalModel< |
| 724 | NVVMTargetConvertToLLVMAttrInterface, NVVM::NVVMTargetAttr> { |
| 725 | /// Configure GPU to NVVM. |
| 726 | void populateConvertToLLVMConversionPatterns( |
| 727 | Attribute attr, ConversionTarget &target, |
| 728 | LLVMTypeConverter &typeConverter, RewritePatternSet &patterns) const; |
| 729 | }; |
| 730 | } // namespace |
| 731 | |
| 732 | void NVVMTargetConvertToLLVMAttrInterface:: |
| 733 | populateConvertToLLVMConversionPatterns(Attribute attr, |
| 734 | ConversionTarget &target, |
| 735 | LLVMTypeConverter &typeConverter, |
| 736 | RewritePatternSet &patterns) const { |
| 737 | configureGpuToNVVMConversionLegality(target); |
| 738 | configureGpuToNVVMTypeConverter(converter&: typeConverter); |
| 739 | populateGpuToNVVMConversionPatterns(converter: typeConverter, patterns); |
| 740 | } |
| 741 | |
| 742 | void mlir::NVVM::registerConvertGpuToNVVMInterface(DialectRegistry ®istry) { |
| 743 | registry.addExtension(extensionFn: +[](MLIRContext *ctx, NVVMDialect *dialect) { |
| 744 | NVVMTargetAttr::attachInterface<NVVMTargetConvertToLLVMAttrInterface>(context&: *ctx); |
| 745 | }); |
| 746 | } |
| 747 | |