| 1 | //===- GPUToLLVMSPV.cpp - Convert GPU operations to LLVM dialect ----------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Conversion/GPUToLLVMSPV/GPUToLLVMSPVPass.h" |
| 10 | |
| 11 | #include "../GPUCommon/GPUOpsLowering.h" |
| 12 | #include "mlir/Conversion/GPUCommon/AttrToSPIRVConverter.h" |
| 13 | #include "mlir/Conversion/GPUCommon/GPUCommonPass.h" |
| 14 | #include "mlir/Conversion/LLVMCommon/ConversionTarget.h" |
| 15 | #include "mlir/Conversion/LLVMCommon/LoweringOptions.h" |
| 16 | #include "mlir/Conversion/LLVMCommon/Pattern.h" |
| 17 | #include "mlir/Conversion/LLVMCommon/TypeConverter.h" |
| 18 | #include "mlir/Conversion/SPIRVCommon/AttrToLLVMConverter.h" |
| 19 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 20 | #include "mlir/Dialect/LLVMIR/LLVMAttrs.h" |
| 21 | #include "mlir/Dialect/LLVMIR/LLVMCallOps.h" |
| 22 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 23 | #include "mlir/Dialect/LLVMIR/LLVMConversionOps.h" |
| 24 | #include "mlir/Dialect/LLVMIR/LLVMModuleOps.h" |
| 25 | #include "mlir/Dialect/LLVMIR/LLVMTypes.h" |
| 26 | #include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h" |
| 27 | #include "mlir/IR/BuiltinTypes.h" |
| 28 | #include "mlir/IR/Matchers.h" |
| 29 | #include "mlir/IR/PatternMatch.h" |
| 30 | #include "mlir/IR/SymbolTable.h" |
| 31 | #include "mlir/Pass/Pass.h" |
| 32 | #include "mlir/Support/LLVM.h" |
| 33 | #include "mlir/Transforms/DialectConversion.h" |
| 34 | |
| 35 | #include "llvm/ADT/TypeSwitch.h" |
| 36 | #include "llvm/Support/FormatVariadic.h" |
| 37 | |
| 38 | #define DEBUG_TYPE "gpu-to-llvm-spv" |
| 39 | |
| 40 | using namespace mlir; |
| 41 | |
| 42 | namespace mlir { |
| 43 | #define GEN_PASS_DEF_CONVERTGPUOPSTOLLVMSPVOPS |
| 44 | #include "mlir/Conversion/Passes.h.inc" |
| 45 | } // namespace mlir |
| 46 | |
| 47 | //===----------------------------------------------------------------------===// |
| 48 | // Helper Functions |
| 49 | //===----------------------------------------------------------------------===// |
| 50 | |
| 51 | static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable, |
| 52 | StringRef name, |
| 53 | ArrayRef<Type> paramTypes, |
| 54 | Type resultType, bool isMemNone, |
| 55 | bool isConvergent) { |
| 56 | auto func = dyn_cast_or_null<LLVM::LLVMFuncOp>( |
| 57 | Val: SymbolTable::lookupSymbolIn(op: symbolTable, symbol: name)); |
| 58 | if (!func) { |
| 59 | OpBuilder b(symbolTable->getRegion(index: 0)); |
| 60 | func = LLVM::LLVMFuncOp::create( |
| 61 | builder&: b, location: symbolTable->getLoc(), name, |
| 62 | type: LLVM::LLVMFunctionType::get(result: resultType, arguments: paramTypes)); |
| 63 | func.setCConv(LLVM::cconv::CConv::SPIR_FUNC); |
| 64 | func.setNoUnwind(true); |
| 65 | func.setWillReturn(true); |
| 66 | |
| 67 | if (isMemNone) { |
| 68 | // no externally observable effects |
| 69 | constexpr auto noModRef = mlir::LLVM::ModRefInfo::NoModRef; |
| 70 | auto memAttr = b.getAttr<LLVM::MemoryEffectsAttr>( |
| 71 | /*other=*/args: noModRef, |
| 72 | /*argMem=*/args: noModRef, /*inaccessibleMem=*/args: noModRef, |
| 73 | /*errnoMem=*/args: noModRef, |
| 74 | /*targetMem0=*/args: noModRef, |
| 75 | /*targetMem1=*/args: noModRef); |
| 76 | func.setMemoryEffectsAttr(memAttr); |
| 77 | } |
| 78 | |
| 79 | func.setConvergent(isConvergent); |
| 80 | } |
| 81 | return func; |
| 82 | } |
| 83 | |
| 84 | static LLVM::CallOp createSPIRVBuiltinCall(Location loc, |
| 85 | ConversionPatternRewriter &rewriter, |
| 86 | LLVM::LLVMFuncOp func, |
| 87 | ValueRange args) { |
| 88 | auto call = LLVM::CallOp::create(builder&: rewriter, location: loc, func, args); |
| 89 | call.setCConv(func.getCConv()); |
| 90 | call.setConvergentAttr(func.getConvergentAttr()); |
| 91 | call.setNoUnwindAttr(func.getNoUnwindAttr()); |
| 92 | call.setWillReturnAttr(func.getWillReturnAttr()); |
| 93 | call.setMemoryEffectsAttr(func.getMemoryEffectsAttr()); |
| 94 | return call; |
| 95 | } |
| 96 | |
| 97 | namespace { |
| 98 | //===----------------------------------------------------------------------===// |
| 99 | // Barriers |
| 100 | //===----------------------------------------------------------------------===// |
| 101 | |
| 102 | /// Replace `gpu.barrier` with an `llvm.call` to `barrier` using |
| 103 | /// `CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE`, ensuring that all memory |
| 104 | /// accesses are visible to all work-items in the work-group. |
| 105 | /// ``` |
| 106 | /// // gpu.barrier |
| 107 | /// // 3 = CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE |
| 108 | /// %c3 = llvm.mlir.constant(3: i32) : i32 |
| 109 | /// llvm.call spir_funccc @_Z7barrierj(%c3) : (i32) -> () |
| 110 | /// ``` |
| 111 | struct GPUBarrierConversion final : ConvertOpToLLVMPattern<gpu::BarrierOp> { |
| 112 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 113 | |
| 114 | LogicalResult |
| 115 | matchAndRewrite(gpu::BarrierOp op, OpAdaptor adaptor, |
| 116 | ConversionPatternRewriter &rewriter) const final { |
| 117 | constexpr StringLiteral funcName = "_Z7barrierj" ; |
| 118 | |
| 119 | Operation *moduleOp = op->getParentWithTrait<OpTrait::SymbolTable>(); |
| 120 | assert(moduleOp && "Expecting module" ); |
| 121 | Type flagTy = rewriter.getI32Type(); |
| 122 | Type voidTy = rewriter.getType<LLVM::LLVMVoidType>(); |
| 123 | LLVM::LLVMFuncOp func = |
| 124 | lookupOrCreateSPIRVFn(symbolTable: moduleOp, name: funcName, paramTypes: flagTy, resultType: voidTy, |
| 125 | /*isMemNone=*/false, /*isConvergent=*/true); |
| 126 | |
| 127 | // Values used by SPIR-V backend to represent `CLK_LOCAL_MEM_FENCE` and |
| 128 | // `CLK_GLOBAL_MEM_FENCE`. See `llvm/lib/Target/SPIRV/SPIRVBuiltins.td`. |
| 129 | constexpr int64_t localMemFenceFlag = 1; |
| 130 | constexpr int64_t globalMemFenceFlag = 2; |
| 131 | int64_t memFenceFlag = 0; |
| 132 | std::optional<ArrayAttr> addressSpaces = adaptor.getAddressSpaces(); |
| 133 | if (addressSpaces) { |
| 134 | for (Attribute attr : addressSpaces.value()) { |
| 135 | auto addressSpace = cast<gpu::AddressSpaceAttr>(Val&: attr).getValue(); |
| 136 | switch (addressSpace) { |
| 137 | case gpu::AddressSpace::Global: |
| 138 | memFenceFlag = memFenceFlag | globalMemFenceFlag; |
| 139 | break; |
| 140 | case gpu::AddressSpace::Workgroup: |
| 141 | memFenceFlag = memFenceFlag | localMemFenceFlag; |
| 142 | break; |
| 143 | case gpu::AddressSpace::Private: |
| 144 | case gpu::AddressSpace::Constant: |
| 145 | // Private is thread-local, constant is read-only; no fencing needed. |
| 146 | break; |
| 147 | } |
| 148 | } |
| 149 | } else { |
| 150 | memFenceFlag = localMemFenceFlag | globalMemFenceFlag; |
| 151 | } |
| 152 | Location loc = op->getLoc(); |
| 153 | Value flag = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: flagTy, value: memFenceFlag); |
| 154 | rewriter.replaceOp(op, newOp: createSPIRVBuiltinCall(loc, rewriter, func, args: flag)); |
| 155 | return success(); |
| 156 | } |
| 157 | }; |
| 158 | |
| 159 | //===----------------------------------------------------------------------===// |
| 160 | // SPIR-V Builtins |
| 161 | //===----------------------------------------------------------------------===// |
| 162 | |
| 163 | /// Replace `gpu.*` with an `llvm.call` to the corresponding SPIR-V builtin with |
| 164 | /// a constant argument for the `dimension` attribute. Return type will depend |
| 165 | /// on index width option: |
| 166 | /// ``` |
| 167 | /// // %thread_id_y = gpu.thread_id y |
| 168 | /// %c1 = llvm.mlir.constant(1: i32) : i32 |
| 169 | /// %0 = llvm.call spir_funccc @_Z12get_local_idj(%c1) : (i32) -> i64 |
| 170 | /// ``` |
| 171 | struct LaunchConfigConversion : ConvertToLLVMPattern { |
| 172 | LaunchConfigConversion(StringRef funcName, StringRef rootOpName, |
| 173 | MLIRContext *context, |
| 174 | const LLVMTypeConverter &typeConverter, |
| 175 | PatternBenefit benefit) |
| 176 | : ConvertToLLVMPattern(rootOpName, context, typeConverter, benefit), |
| 177 | funcName(funcName) {} |
| 178 | |
| 179 | virtual gpu::Dimension getDimension(Operation *op) const = 0; |
| 180 | |
| 181 | LogicalResult |
| 182 | matchAndRewrite(Operation *op, ArrayRef<Value> operands, |
| 183 | ConversionPatternRewriter &rewriter) const final { |
| 184 | Operation *moduleOp = op->getParentWithTrait<OpTrait::SymbolTable>(); |
| 185 | assert(moduleOp && "Expecting module" ); |
| 186 | Type dimTy = rewriter.getI32Type(); |
| 187 | Type indexTy = getTypeConverter()->getIndexType(); |
| 188 | LLVM::LLVMFuncOp func = lookupOrCreateSPIRVFn(symbolTable: moduleOp, name: funcName, paramTypes: dimTy, |
| 189 | resultType: indexTy, /*isMemNone=*/true, |
| 190 | /*isConvergent=*/false); |
| 191 | |
| 192 | Location loc = op->getLoc(); |
| 193 | gpu::Dimension dim = getDimension(op); |
| 194 | Value dimVal = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: dimTy, |
| 195 | value: static_cast<int64_t>(dim)); |
| 196 | rewriter.replaceOp(op, newOp: createSPIRVBuiltinCall(loc, rewriter, func, args: dimVal)); |
| 197 | return success(); |
| 198 | } |
| 199 | |
| 200 | StringRef funcName; |
| 201 | }; |
| 202 | |
| 203 | template <typename SourceOp> |
| 204 | struct LaunchConfigOpConversion final : LaunchConfigConversion { |
| 205 | static StringRef getFuncName(); |
| 206 | |
| 207 | explicit LaunchConfigOpConversion(const LLVMTypeConverter &typeConverter, |
| 208 | PatternBenefit benefit = 1) |
| 209 | : LaunchConfigConversion(getFuncName(), SourceOp::getOperationName(), |
| 210 | &typeConverter.getContext(), typeConverter, |
| 211 | benefit) {} |
| 212 | |
| 213 | gpu::Dimension getDimension(Operation *op) const final { |
| 214 | return cast<SourceOp>(op).getDimension(); |
| 215 | } |
| 216 | }; |
| 217 | |
| 218 | template <> |
| 219 | StringRef LaunchConfigOpConversion<gpu::BlockIdOp>::getFuncName() { |
| 220 | return "_Z12get_group_idj" ; |
| 221 | } |
| 222 | |
| 223 | template <> |
| 224 | StringRef LaunchConfigOpConversion<gpu::GridDimOp>::getFuncName() { |
| 225 | return "_Z14get_num_groupsj" ; |
| 226 | } |
| 227 | |
| 228 | template <> |
| 229 | StringRef LaunchConfigOpConversion<gpu::BlockDimOp>::getFuncName() { |
| 230 | return "_Z14get_local_sizej" ; |
| 231 | } |
| 232 | |
| 233 | template <> |
| 234 | StringRef LaunchConfigOpConversion<gpu::ThreadIdOp>::getFuncName() { |
| 235 | return "_Z12get_local_idj" ; |
| 236 | } |
| 237 | |
| 238 | template <> |
| 239 | StringRef LaunchConfigOpConversion<gpu::GlobalIdOp>::getFuncName() { |
| 240 | return "_Z13get_global_idj" ; |
| 241 | } |
| 242 | |
| 243 | //===----------------------------------------------------------------------===// |
| 244 | // Shuffles |
| 245 | //===----------------------------------------------------------------------===// |
| 246 | |
| 247 | /// Replace `gpu.shuffle` with an `llvm.call` to the corresponding SPIR-V |
| 248 | /// builtin for `shuffleResult`, keeping `value` and `offset` arguments, and a |
| 249 | /// `true` constant for the `valid` result type. Conversion will only take place |
| 250 | /// if `width` is constant and equal to the `subgroup` pass option: |
| 251 | /// ``` |
| 252 | /// // %0 = gpu.shuffle idx %value, %offset, %width : f64 |
| 253 | /// %0 = llvm.call spir_funccc @_Z17sub_group_shuffledj(%value, %offset) |
| 254 | /// : (f64, i32) -> f64 |
| 255 | /// ``` |
| 256 | struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> { |
| 257 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 258 | |
| 259 | static StringRef getBaseName(gpu::ShuffleMode mode) { |
| 260 | switch (mode) { |
| 261 | case gpu::ShuffleMode::IDX: |
| 262 | return "sub_group_shuffle" ; |
| 263 | case gpu::ShuffleMode::XOR: |
| 264 | return "sub_group_shuffle_xor" ; |
| 265 | case gpu::ShuffleMode::UP: |
| 266 | return "sub_group_shuffle_up" ; |
| 267 | case gpu::ShuffleMode::DOWN: |
| 268 | return "sub_group_shuffle_down" ; |
| 269 | } |
| 270 | llvm_unreachable("Unhandled shuffle mode" ); |
| 271 | } |
| 272 | |
| 273 | static std::optional<StringRef> getTypeMangling(Type type) { |
| 274 | return TypeSwitch<Type, std::optional<StringRef>>(type) |
| 275 | .Case(caseFn: [](Float16Type) { return "Dhj" ; }) |
| 276 | .Case(caseFn: [](Float32Type) { return "fj" ; }) |
| 277 | .Case(caseFn: [](Float64Type) { return "dj" ; }) |
| 278 | .Case(caseFn: [](IntegerType intTy) -> std::optional<StringRef> { |
| 279 | switch (intTy.getWidth()) { |
| 280 | case 8: |
| 281 | return "cj" ; |
| 282 | case 16: |
| 283 | return "sj" ; |
| 284 | case 32: |
| 285 | return "ij" ; |
| 286 | case 64: |
| 287 | return "lj" ; |
| 288 | } |
| 289 | return std::nullopt; |
| 290 | }) |
| 291 | .Default(std::nullopt); |
| 292 | } |
| 293 | |
| 294 | static std::optional<std::string> getFuncName(gpu::ShuffleMode mode, |
| 295 | Type type) { |
| 296 | StringRef baseName = getBaseName(mode); |
| 297 | std::optional<StringRef> typeMangling = getTypeMangling(type); |
| 298 | if (!typeMangling) |
| 299 | return std::nullopt; |
| 300 | return llvm::formatv(Fmt: "_Z{}{}{}" , Vals: baseName.size(), Vals&: baseName, |
| 301 | Vals&: typeMangling.value()); |
| 302 | } |
| 303 | |
| 304 | /// Get the subgroup size from the target or return a default. |
| 305 | static std::optional<int> getSubgroupSize(Operation *op) { |
| 306 | auto parentFunc = op->getParentOfType<LLVM::LLVMFuncOp>(); |
| 307 | if (!parentFunc) |
| 308 | return std::nullopt; |
| 309 | return parentFunc.getIntelReqdSubGroupSize(); |
| 310 | } |
| 311 | |
| 312 | static bool hasValidWidth(gpu::ShuffleOp op, int subgroupSize) { |
| 313 | llvm::APInt val; |
| 314 | Value width = op.getWidth(); |
| 315 | return matchPattern(value: width, pattern: m_ConstantInt(bind_value: &val)) && val == subgroupSize; |
| 316 | } |
| 317 | |
| 318 | static Value bitcastOrExtBeforeShuffle(Value oldVal, Location loc, |
| 319 | ConversionPatternRewriter &rewriter) { |
| 320 | return TypeSwitch<Type, Value>(oldVal.getType()) |
| 321 | .Case(caseFn: [&](BFloat16Type) { |
| 322 | return LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: rewriter.getI16Type(), |
| 323 | arg: oldVal); |
| 324 | }) |
| 325 | .Case(caseFn: [&](IntegerType intTy) -> Value { |
| 326 | if (intTy.getWidth() == 1) |
| 327 | return LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: rewriter.getI8Type(), |
| 328 | arg: oldVal); |
| 329 | return oldVal; |
| 330 | }) |
| 331 | .Default(defaultResult: oldVal); |
| 332 | } |
| 333 | |
| 334 | static Value bitcastOrTruncAfterShuffle(Value oldVal, Type newTy, |
| 335 | Location loc, |
| 336 | ConversionPatternRewriter &rewriter) { |
| 337 | return TypeSwitch<Type, Value>(newTy) |
| 338 | .Case(caseFn: [&](BFloat16Type) { |
| 339 | return LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: newTy, arg: oldVal); |
| 340 | }) |
| 341 | .Case(caseFn: [&](IntegerType intTy) -> Value { |
| 342 | if (intTy.getWidth() == 1) |
| 343 | return LLVM::TruncOp::create(builder&: rewriter, location: loc, res: newTy, arg: oldVal); |
| 344 | return oldVal; |
| 345 | }) |
| 346 | .Default(defaultResult: oldVal); |
| 347 | } |
| 348 | |
| 349 | LogicalResult |
| 350 | matchAndRewrite(gpu::ShuffleOp op, OpAdaptor adaptor, |
| 351 | ConversionPatternRewriter &rewriter) const final { |
| 352 | auto maybeSubgroupSize = getSubgroupSize(op); |
| 353 | if (maybeSubgroupSize && !hasValidWidth(op, subgroupSize: maybeSubgroupSize.value())) |
| 354 | return rewriter.notifyMatchFailure( |
| 355 | arg&: op, msg: "shuffle width and subgroup size mismatch" ); |
| 356 | |
| 357 | Location loc = op->getLoc(); |
| 358 | Value inValue = |
| 359 | bitcastOrExtBeforeShuffle(oldVal: adaptor.getValue(), loc, rewriter); |
| 360 | std::optional<std::string> funcName = |
| 361 | getFuncName(mode: op.getMode(), type: inValue.getType()); |
| 362 | if (!funcName) |
| 363 | return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported value type" ); |
| 364 | |
| 365 | Operation *moduleOp = op->getParentWithTrait<OpTrait::SymbolTable>(); |
| 366 | assert(moduleOp && "Expecting module" ); |
| 367 | Type valueType = inValue.getType(); |
| 368 | Type offsetType = adaptor.getOffset().getType(); |
| 369 | Type resultType = valueType; |
| 370 | LLVM::LLVMFuncOp func = lookupOrCreateSPIRVFn( |
| 371 | symbolTable: moduleOp, name: funcName.value(), paramTypes: {valueType, offsetType}, resultType, |
| 372 | /*isMemNone=*/false, /*isConvergent=*/true); |
| 373 | |
| 374 | std::array<Value, 2> args{inValue, adaptor.getOffset()}; |
| 375 | Value result = |
| 376 | createSPIRVBuiltinCall(loc, rewriter, func, args).getResult(); |
| 377 | Value resultOrConversion = |
| 378 | bitcastOrTruncAfterShuffle(oldVal: result, newTy: op.getType(i: 0), loc, rewriter); |
| 379 | |
| 380 | Value trueVal = |
| 381 | LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI1Type(), value: true); |
| 382 | rewriter.replaceOp(op, newValues: {resultOrConversion, trueVal}); |
| 383 | return success(); |
| 384 | } |
| 385 | }; |
| 386 | |
| 387 | class MemorySpaceToOpenCLMemorySpaceConverter final : public TypeConverter { |
| 388 | public: |
| 389 | MemorySpaceToOpenCLMemorySpaceConverter(MLIRContext *ctx) { |
| 390 | addConversion(callback: [](Type t) { return t; }); |
| 391 | addConversion(callback: [ctx](BaseMemRefType memRefType) -> std::optional<Type> { |
| 392 | // Attach global addr space attribute to memrefs with no addr space attr |
| 393 | Attribute memSpaceAttr = memRefType.getMemorySpace(); |
| 394 | if (memSpaceAttr) |
| 395 | return std::nullopt; |
| 396 | |
| 397 | unsigned globalAddrspace = storageClassToAddressSpace( |
| 398 | clientAPI: spirv::ClientAPI::OpenCL, storageClass: spirv::StorageClass::CrossWorkgroup); |
| 399 | Attribute addrSpaceAttr = |
| 400 | IntegerAttr::get(type: IntegerType::get(context: ctx, width: 64), value: globalAddrspace); |
| 401 | if (auto rankedType = dyn_cast<MemRefType>(Val&: memRefType)) { |
| 402 | return MemRefType::get(shape: memRefType.getShape(), |
| 403 | elementType: memRefType.getElementType(), |
| 404 | layout: rankedType.getLayout(), memorySpace: addrSpaceAttr); |
| 405 | } |
| 406 | return UnrankedMemRefType::get(elementType: memRefType.getElementType(), |
| 407 | memorySpace: addrSpaceAttr); |
| 408 | }); |
| 409 | addConversion(callback: [this](FunctionType type) { |
| 410 | auto inputs = llvm::map_to_vector( |
| 411 | C: type.getInputs(), F: [this](Type ty) { return convertType(t: ty); }); |
| 412 | auto results = llvm::map_to_vector( |
| 413 | C: type.getResults(), F: [this](Type ty) { return convertType(t: ty); }); |
| 414 | return FunctionType::get(context: type.getContext(), inputs, results); |
| 415 | }); |
| 416 | } |
| 417 | }; |
| 418 | |
| 419 | //===----------------------------------------------------------------------===// |
| 420 | // Subgroup query ops. |
| 421 | //===----------------------------------------------------------------------===// |
| 422 | |
| 423 | template <typename SubgroupOp> |
| 424 | struct GPUSubgroupOpConversion final : ConvertOpToLLVMPattern<SubgroupOp> { |
| 425 | using ConvertOpToLLVMPattern<SubgroupOp>::ConvertOpToLLVMPattern; |
| 426 | using ConvertToLLVMPattern::getTypeConverter; |
| 427 | |
| 428 | LogicalResult |
| 429 | matchAndRewrite(SubgroupOp op, typename SubgroupOp::Adaptor adaptor, |
| 430 | ConversionPatternRewriter &rewriter) const final { |
| 431 | constexpr StringRef funcName = [] { |
| 432 | if constexpr (std::is_same_v<SubgroupOp, gpu::SubgroupIdOp>) { |
| 433 | return "_Z16get_sub_group_id" ; |
| 434 | } else if constexpr (std::is_same_v<SubgroupOp, gpu::LaneIdOp>) { |
| 435 | return "_Z22get_sub_group_local_id" ; |
| 436 | } else if constexpr (std::is_same_v<SubgroupOp, gpu::NumSubgroupsOp>) { |
| 437 | return "_Z18get_num_sub_groups" ; |
| 438 | } else if constexpr (std::is_same_v<SubgroupOp, gpu::SubgroupSizeOp>) { |
| 439 | return "_Z18get_sub_group_size" ; |
| 440 | } |
| 441 | }(); |
| 442 | |
| 443 | Operation *moduleOp = |
| 444 | op->template getParentWithTrait<OpTrait::SymbolTable>(); |
| 445 | Type resultTy = rewriter.getI32Type(); |
| 446 | LLVM::LLVMFuncOp func = |
| 447 | lookupOrCreateSPIRVFn(symbolTable: moduleOp, name: funcName, paramTypes: {}, resultType: resultTy, |
| 448 | /*isMemNone=*/false, /*isConvergent=*/false); |
| 449 | |
| 450 | Location loc = op->getLoc(); |
| 451 | Value result = createSPIRVBuiltinCall(loc, rewriter, func, args: {}).getResult(); |
| 452 | |
| 453 | Type indexTy = getTypeConverter()->getIndexType(); |
| 454 | if (resultTy != indexTy) { |
| 455 | if (indexTy.getIntOrFloatBitWidth() < resultTy.getIntOrFloatBitWidth()) { |
| 456 | return failure(); |
| 457 | } |
| 458 | result = LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: indexTy, arg: result); |
| 459 | } |
| 460 | |
| 461 | rewriter.replaceOp(op, result); |
| 462 | return success(); |
| 463 | } |
| 464 | }; |
| 465 | |
| 466 | //===----------------------------------------------------------------------===// |
| 467 | // GPU To LLVM-SPV Pass. |
| 468 | //===----------------------------------------------------------------------===// |
| 469 | |
| 470 | struct GPUToLLVMSPVConversionPass final |
| 471 | : impl::ConvertGpuOpsToLLVMSPVOpsBase<GPUToLLVMSPVConversionPass> { |
| 472 | using Base::Base; |
| 473 | |
| 474 | void runOnOperation() final { |
| 475 | MLIRContext *context = &getContext(); |
| 476 | RewritePatternSet patterns(context); |
| 477 | |
| 478 | LowerToLLVMOptions options(context); |
| 479 | options.overrideIndexBitwidth(bitwidth: this->use64bitIndex ? 64 : 32); |
| 480 | LLVMTypeConverter converter(context, options); |
| 481 | LLVMConversionTarget target(*context); |
| 482 | |
| 483 | // Force OpenCL address spaces when they are not present |
| 484 | { |
| 485 | MemorySpaceToOpenCLMemorySpaceConverter converter(context); |
| 486 | AttrTypeReplacer replacer; |
| 487 | replacer.addReplacement(callback: [&converter](BaseMemRefType origType) |
| 488 | -> std::optional<BaseMemRefType> { |
| 489 | return converter.convertType<BaseMemRefType>(t: origType); |
| 490 | }); |
| 491 | |
| 492 | replacer.recursivelyReplaceElementsIn(op: getOperation(), |
| 493 | /*replaceAttrs=*/true, |
| 494 | /*replaceLocs=*/false, |
| 495 | /*replaceTypes=*/true); |
| 496 | } |
| 497 | |
| 498 | target.addIllegalOp<gpu::BarrierOp, gpu::BlockDimOp, gpu::BlockIdOp, |
| 499 | gpu::GPUFuncOp, gpu::GlobalIdOp, gpu::GridDimOp, |
| 500 | gpu::LaneIdOp, gpu::NumSubgroupsOp, gpu::ReturnOp, |
| 501 | gpu::ShuffleOp, gpu::SubgroupIdOp, gpu::SubgroupSizeOp, |
| 502 | gpu::ThreadIdOp, gpu::PrintfOp>(); |
| 503 | |
| 504 | populateGpuToLLVMSPVConversionPatterns(converter, patterns); |
| 505 | populateGpuMemorySpaceAttributeConversions(typeConverter&: converter); |
| 506 | patterns.add<GPUPrintfOpToLLVMCallLowering>(arg&: converter, /*addressSpace=*/args: 2, |
| 507 | args: LLVM::cconv::CConv::SPIR_FUNC, |
| 508 | args: "_Z6printfPU3AS2Kcz" ); |
| 509 | |
| 510 | if (failed(Result: applyPartialConversion(op: getOperation(), target, |
| 511 | patterns: std::move(patterns)))) |
| 512 | signalPassFailure(); |
| 513 | } |
| 514 | }; |
| 515 | } // namespace |
| 516 | |
| 517 | //===----------------------------------------------------------------------===// |
| 518 | // GPU To LLVM-SPV Patterns. |
| 519 | //===----------------------------------------------------------------------===// |
| 520 | |
| 521 | namespace mlir { |
| 522 | namespace { |
| 523 | static unsigned |
| 524 | gpuAddressSpaceToOCLAddressSpace(gpu::AddressSpace addressSpace) { |
| 525 | constexpr spirv::ClientAPI clientAPI = spirv::ClientAPI::OpenCL; |
| 526 | return storageClassToAddressSpace(clientAPI, |
| 527 | storageClass: addressSpaceToStorageClass(addressSpace)); |
| 528 | } |
| 529 | } // namespace |
| 530 | |
| 531 | void populateGpuToLLVMSPVConversionPatterns( |
| 532 | const LLVMTypeConverter &typeConverter, RewritePatternSet &patterns) { |
| 533 | patterns.add<GPUBarrierConversion, GPUReturnOpLowering, GPUShuffleConversion, |
| 534 | GPUSubgroupOpConversion<gpu::LaneIdOp>, |
| 535 | GPUSubgroupOpConversion<gpu::NumSubgroupsOp>, |
| 536 | GPUSubgroupOpConversion<gpu::SubgroupIdOp>, |
| 537 | GPUSubgroupOpConversion<gpu::SubgroupSizeOp>, |
| 538 | LaunchConfigOpConversion<gpu::BlockDimOp>, |
| 539 | LaunchConfigOpConversion<gpu::BlockIdOp>, |
| 540 | LaunchConfigOpConversion<gpu::GlobalIdOp>, |
| 541 | LaunchConfigOpConversion<gpu::GridDimOp>, |
| 542 | LaunchConfigOpConversion<gpu::ThreadIdOp>>(arg: typeConverter); |
| 543 | MLIRContext *context = &typeConverter.getContext(); |
| 544 | unsigned privateAddressSpace = |
| 545 | gpuAddressSpaceToOCLAddressSpace(addressSpace: gpu::AddressSpace::Private); |
| 546 | unsigned localAddressSpace = |
| 547 | gpuAddressSpaceToOCLAddressSpace(addressSpace: gpu::AddressSpace::Workgroup); |
| 548 | OperationName llvmFuncOpName(LLVM::LLVMFuncOp::getOperationName(), context); |
| 549 | StringAttr kernelBlockSizeAttributeName = |
| 550 | LLVM::LLVMFuncOp::getReqdWorkGroupSizeAttrName(name: llvmFuncOpName); |
| 551 | patterns.add<GPUFuncOpLowering>( |
| 552 | arg: typeConverter, |
| 553 | args: GPUFuncOpLoweringOptions{ |
| 554 | .allocaAddrSpace: privateAddressSpace, .workgroupAddrSpace: localAddressSpace, |
| 555 | /*kernelAttributeName=*/{}, .kernelBlockSizeAttributeName: kernelBlockSizeAttributeName, |
| 556 | /*kernelClusterSizeAttributeName=*/{}, .kernelCallingConvention: LLVM::CConv::SPIR_KERNEL, |
| 557 | .nonKernelCallingConvention: LLVM::CConv::SPIR_FUNC, |
| 558 | /*encodeWorkgroupAttributionsAsArguments=*/true}); |
| 559 | } |
| 560 | |
| 561 | void populateGpuMemorySpaceAttributeConversions(TypeConverter &typeConverter) { |
| 562 | populateGpuMemorySpaceAttributeConversions(typeConverter, |
| 563 | mapping: gpuAddressSpaceToOCLAddressSpace); |
| 564 | } |
| 565 | } // namespace mlir |
| 566 | |