| 1 | //===- GPUToSPIRVPass.cpp - GPU to SPIR-V Passes --------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements a pass to convert a kernel function in the GPU Dialect |
| 10 | // into a spirv.module operation. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Conversion/GPUToSPIRV/GPUToSPIRVPass.h" |
| 15 | |
| 16 | #include "mlir/Conversion/ArithToSPIRV/ArithToSPIRV.h" |
| 17 | #include "mlir/Conversion/FuncToSPIRV/FuncToSPIRV.h" |
| 18 | #include "mlir/Conversion/GPUToSPIRV/GPUToSPIRV.h" |
| 19 | #include "mlir/Conversion/MemRefToSPIRV/MemRefToSPIRV.h" |
| 20 | #include "mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h" |
| 21 | #include "mlir/Conversion/VectorToSPIRV/VectorToSPIRV.h" |
| 22 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 23 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 24 | #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h" |
| 25 | #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h" |
| 26 | #include "mlir/IR/PatternMatch.h" |
| 27 | |
| 28 | namespace mlir { |
| 29 | #define GEN_PASS_DEF_CONVERTGPUTOSPIRV |
| 30 | #include "mlir/Conversion/Passes.h.inc" |
| 31 | } // namespace mlir |
| 32 | |
| 33 | using namespace mlir; |
| 34 | |
| 35 | namespace { |
| 36 | /// Pass to lower GPU Dialect to SPIR-V. The pass only converts the gpu.func ops |
| 37 | /// inside gpu.module ops. i.e., the function that are referenced in |
| 38 | /// gpu.launch_func ops. For each such function |
| 39 | /// |
| 40 | /// 1) Create a spirv::ModuleOp, and clone the function into spirv::ModuleOp |
| 41 | /// (the original function is still needed by the gpu::LaunchKernelOp, so cannot |
| 42 | /// replace it). |
| 43 | /// |
| 44 | /// 2) Lower the body of the spirv::ModuleOp. |
| 45 | struct GPUToSPIRVPass final : impl::ConvertGPUToSPIRVBase<GPUToSPIRVPass> { |
| 46 | explicit GPUToSPIRVPass(bool mapMemorySpace) |
| 47 | : mapMemorySpace(mapMemorySpace) {} |
| 48 | void runOnOperation() override; |
| 49 | |
| 50 | private: |
| 51 | /// Queries the target environment from 'targets' attribute of the given |
| 52 | /// `moduleOp`. |
| 53 | spirv::TargetEnvAttr lookupTargetEnvInTargets(gpu::GPUModuleOp moduleOp); |
| 54 | |
| 55 | /// Queries the target environment from 'targets' attribute of the given |
| 56 | /// `moduleOp` or returns target environment as returned by |
| 57 | /// `spirv::lookupTargetEnvOrDefault` if not provided by 'targets'. |
| 58 | spirv::TargetEnvAttr lookupTargetEnvOrDefault(gpu::GPUModuleOp moduleOp); |
| 59 | bool mapMemorySpace; |
| 60 | }; |
| 61 | |
| 62 | spirv::TargetEnvAttr |
| 63 | GPUToSPIRVPass::lookupTargetEnvInTargets(gpu::GPUModuleOp moduleOp) { |
| 64 | if (ArrayAttr targets = moduleOp.getTargetsAttr()) { |
| 65 | for (Attribute targetAttr : targets) |
| 66 | if (auto spirvTargetEnvAttr = dyn_cast<spirv::TargetEnvAttr>(Val&: targetAttr)) |
| 67 | return spirvTargetEnvAttr; |
| 68 | } |
| 69 | |
| 70 | return {}; |
| 71 | } |
| 72 | |
| 73 | spirv::TargetEnvAttr |
| 74 | GPUToSPIRVPass::lookupTargetEnvOrDefault(gpu::GPUModuleOp moduleOp) { |
| 75 | if (spirv::TargetEnvAttr targetEnvAttr = lookupTargetEnvInTargets(moduleOp)) |
| 76 | return targetEnvAttr; |
| 77 | |
| 78 | return spirv::lookupTargetEnvOrDefault(op: moduleOp); |
| 79 | } |
| 80 | |
| 81 | void GPUToSPIRVPass::runOnOperation() { |
| 82 | MLIRContext *context = &getContext(); |
| 83 | ModuleOp module = getOperation(); |
| 84 | |
| 85 | SmallVector<Operation *, 1> gpuModules; |
| 86 | OpBuilder builder(context); |
| 87 | |
| 88 | auto targetEnvSupportsKernelCapability = [this](gpu::GPUModuleOp moduleOp) { |
| 89 | auto targetAttr = lookupTargetEnvOrDefault(moduleOp); |
| 90 | spirv::TargetEnv targetEnv(targetAttr); |
| 91 | return targetEnv.allows(spirv::Capability::Kernel); |
| 92 | }; |
| 93 | |
| 94 | module.walk(callback: [&](gpu::GPUModuleOp moduleOp) { |
| 95 | // Clone each GPU kernel module for conversion, given that the GPU |
| 96 | // launch op still needs the original GPU kernel module. |
| 97 | // For Vulkan Shader capabilities, we insert the newly converted SPIR-V |
| 98 | // module right after the original GPU module, as that's the expectation of |
| 99 | // the in-tree SPIR-V CPU runner (the Vulkan runner does not use this pass). |
| 100 | // For OpenCL Kernel capabilities, we insert the newly converted SPIR-V |
| 101 | // module inside the original GPU module, as that's the expectaion of the |
| 102 | // normal GPU compilation pipeline. |
| 103 | if (targetEnvSupportsKernelCapability(moduleOp)) { |
| 104 | builder.setInsertionPointToStart(moduleOp.getBody()); |
| 105 | } else { |
| 106 | builder.setInsertionPoint(moduleOp.getOperation()); |
| 107 | } |
| 108 | gpuModules.push_back(Elt: builder.clone(op&: *moduleOp.getOperation())); |
| 109 | }); |
| 110 | |
| 111 | // Run conversion for each module independently as they can have different |
| 112 | // TargetEnv attributes. |
| 113 | for (Operation *gpuModule : gpuModules) { |
| 114 | spirv::TargetEnvAttr targetAttr = |
| 115 | lookupTargetEnvOrDefault(moduleOp: cast<gpu::GPUModuleOp>(Val: gpuModule)); |
| 116 | |
| 117 | // Map MemRef memory space to SPIR-V storage class first if requested. |
| 118 | if (mapMemorySpace) { |
| 119 | spirv::MemorySpaceToStorageClassMap memorySpaceMap = |
| 120 | targetEnvSupportsKernelCapability( |
| 121 | dyn_cast<gpu::GPUModuleOp>(Val: gpuModule)) |
| 122 | ? spirv::mapMemorySpaceToOpenCLStorageClass |
| 123 | : spirv::mapMemorySpaceToVulkanStorageClass; |
| 124 | spirv::MemorySpaceToStorageClassConverter converter(memorySpaceMap); |
| 125 | spirv::convertMemRefTypesAndAttrs(op: gpuModule, typeConverter&: converter); |
| 126 | |
| 127 | // Check if there are any illegal ops remaining. |
| 128 | std::unique_ptr<ConversionTarget> target = |
| 129 | spirv::getMemorySpaceToStorageClassTarget(*context); |
| 130 | gpuModule->walk(callback: [&target, this](Operation *childOp) { |
| 131 | if (target->isIllegal(op: childOp)) { |
| 132 | childOp->emitOpError(message: "failed to legalize memory space" ); |
| 133 | signalPassFailure(); |
| 134 | return WalkResult::interrupt(); |
| 135 | } |
| 136 | return WalkResult::advance(); |
| 137 | }); |
| 138 | } |
| 139 | |
| 140 | std::unique_ptr<ConversionTarget> target = |
| 141 | SPIRVConversionTarget::get(targetAttr); |
| 142 | |
| 143 | SPIRVConversionOptions options; |
| 144 | options.use64bitIndex = this->use64bitIndex; |
| 145 | SPIRVTypeConverter typeConverter(targetAttr, options); |
| 146 | populateMMAToSPIRVCoopMatrixTypeConversion(typeConverter); |
| 147 | populateGPUNamedBarrierToSPIRVTypeConversion(typeConverter); |
| 148 | |
| 149 | RewritePatternSet patterns(context); |
| 150 | populateGPUToSPIRVPatterns(typeConverter, patterns); |
| 151 | populateGpuWMMAToSPIRVCoopMatrixKHRConversionPatterns(typeConverter, |
| 152 | patterns); |
| 153 | |
| 154 | // TODO: Change SPIR-V conversion to be progressive and remove the following |
| 155 | // patterns. |
| 156 | ScfToSPIRVContext scfContext; |
| 157 | populateSCFToSPIRVPatterns(typeConverter, scfToSPIRVContext&: scfContext, patterns); |
| 158 | mlir::arith::populateArithToSPIRVPatterns(typeConverter, patterns); |
| 159 | populateMemRefToSPIRVPatterns(typeConverter, patterns); |
| 160 | populateFuncToSPIRVPatterns(typeConverter, patterns); |
| 161 | populateVectorToSPIRVPatterns(typeConverter, patterns); |
| 162 | |
| 163 | if (failed(Result: applyFullConversion(op: gpuModule, target: *target, patterns: std::move(patterns)))) |
| 164 | return signalPassFailure(); |
| 165 | } |
| 166 | |
| 167 | // For OpenCL, the gpu.func op in the original gpu.module op needs to be |
| 168 | // replaced with an empty func.func op with the same arguments as the gpu.func |
| 169 | // op. The func.func op needs gpu.kernel attribute set. |
| 170 | module.walk(callback: [&](gpu::GPUModuleOp moduleOp) { |
| 171 | if (targetEnvSupportsKernelCapability(moduleOp)) { |
| 172 | moduleOp.walk(callback: [&](gpu::GPUFuncOp funcOp) { |
| 173 | builder.setInsertionPoint(funcOp); |
| 174 | auto newFuncOp = |
| 175 | func::FuncOp::create(builder, location: funcOp.getLoc(), name: funcOp.getName(), |
| 176 | type: funcOp.getFunctionType()); |
| 177 | auto entryBlock = newFuncOp.addEntryBlock(); |
| 178 | builder.setInsertionPointToEnd(entryBlock); |
| 179 | func::ReturnOp::create(builder, location: funcOp.getLoc()); |
| 180 | newFuncOp->setDiscardableAttr(name: gpu::GPUDialect::getKernelFuncAttrName(), |
| 181 | value: builder.getUnitAttr()); |
| 182 | funcOp.erase(); |
| 183 | }); |
| 184 | } |
| 185 | }); |
| 186 | } |
| 187 | |
| 188 | } // namespace |
| 189 | |
| 190 | std::unique_ptr<OperationPass<ModuleOp>> |
| 191 | mlir::createConvertGPUToSPIRVPass(bool mapMemorySpace) { |
| 192 | return std::make_unique<GPUToSPIRVPass>(args&: mapMemorySpace); |
| 193 | } |
| 194 | |