| 1 | //===- GPUToXeVMPipeline.cpp - Lowering pipeline to XeVM/LLVM -------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements a pass for testing the lowering to XeVM as a generally |
| 10 | // usable sink pass. If XeGPU ops are used, it expects the MLIR code to have |
| 11 | // XeGPU ops already embedded in gpu code. |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #include "mlir/Conversion/AffineToStandard/AffineToStandard.h" |
| 16 | #include "mlir/Conversion/ConvertToLLVM/ToLLVMPass.h" |
| 17 | #include "mlir/Conversion/GPUCommon/GPUCommonPass.h" |
| 18 | #include "mlir/Conversion/GPUToLLVMSPV/GPUToLLVMSPVPass.h" |
| 19 | #include "mlir/Conversion/MathToXeVM/MathToXeVM.h" |
| 20 | #include "mlir/Conversion/ReconcileUnrealizedCasts/ReconcileUnrealizedCasts.h" |
| 21 | #include "mlir/Conversion/SCFToControlFlow/SCFToControlFlow.h" |
| 22 | #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.h" |
| 23 | #include "mlir/Conversion/VectorToSCF/VectorToSCF.h" |
| 24 | #include "mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h" |
| 25 | #include "mlir/Conversion/XeGPUToXeVM/XeGPUToXeVM.h" |
| 26 | #include "mlir/Conversion/XeVMToLLVM/XeVMToLLVM.h" |
| 27 | #include "mlir/Dialect/Arith/Transforms/Passes.h" |
| 28 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 29 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 30 | #include "mlir/Dialect/GPU/Pipelines/Passes.h" |
| 31 | #include "mlir/Dialect/GPU/Transforms/Passes.h" |
| 32 | #include "mlir/Dialect/GPU/Transforms/XeVM/Passes.h" |
| 33 | #include "mlir/Dialect/LLVMIR/Transforms/RequestCWrappers.h" |
| 34 | #include "mlir/Dialect/Math/Transforms/Passes.h" |
| 35 | #include "mlir/Dialect/MemRef/Transforms/Passes.h" |
| 36 | #include "mlir/Dialect/XeGPU/Transforms/Passes.h" |
| 37 | #include "mlir/Pass/PassManager.h" |
| 38 | #include "mlir/Pass/PassOptions.h" |
| 39 | #include "mlir/Target/LLVM/XeVM/Target.h" |
| 40 | #include "mlir/Transforms/Passes.h" |
| 41 | |
| 42 | #include <utility> |
| 43 | |
| 44 | using namespace mlir; |
| 45 | |
| 46 | namespace { |
| 47 | //===----------------------------------------------------------------------===// |
| 48 | // Pre-GPU common pipeline for both Host and GPU. |
| 49 | //===----------------------------------------------------------------------===// |
| 50 | void buildPreGPUCommonPassPipeline( |
| 51 | OpPassManager &pm, const mlir::gpu::GPUToXeVMPipelineOptions &options) { |
| 52 | // builtin.module scope passes. |
| 53 | pm.addPass(pass: createCSEPass()); |
| 54 | { |
| 55 | GpuXeVMAttachTargetOptions xevmTargetOptions; |
| 56 | xevmTargetOptions.moduleMatcher = options.xevmModuleMatcher; |
| 57 | xevmTargetOptions.triple = options.zebinTriple; |
| 58 | xevmTargetOptions.chip = options.zebinChip; |
| 59 | xevmTargetOptions.optLevel = options.optLevel; |
| 60 | xevmTargetOptions.cmdOptions = options.cmdOptions; |
| 61 | pm.addPass(pass: createGpuXeVMAttachTarget(options: std::move(xevmTargetOptions))); |
| 62 | } |
| 63 | pm.addPass(pass: createLowerAffinePass()); |
| 64 | pm.addNestedPass<func::FuncOp>(pass: createGpuAsyncRegionPass()); |
| 65 | } |
| 66 | |
| 67 | //===----------------------------------------------------------------------===// |
| 68 | // GPUModule-specific stuff. |
| 69 | //===----------------------------------------------------------------------===// |
| 70 | void buildGPUPassPipeline(OpPassManager &pm, |
| 71 | const mlir::gpu::GPUToXeVMPipelineOptions &options) { |
| 72 | xegpu::XeGPUPropagateLayoutOptions laneLayoutOptions; |
| 73 | laneLayoutOptions.indexBitWidth = options.use64bitIndex ? 64 : 32; |
| 74 | laneLayoutOptions.layoutKind = "lane" ; |
| 75 | pm.addNestedPass<ModuleOp>(pass: createCSEPass()); |
| 76 | if (options.enableVectorToXeGPU) |
| 77 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createConvertVectorToXeGPU()); |
| 78 | if (options.xegpuOpLevel == "workgroup" ) { |
| 79 | xegpu::XeGPUPropagateLayoutOptions sgLayoutOptions; |
| 80 | sgLayoutOptions.layoutKind = "subgroup" ; |
| 81 | pm.addNestedPass<gpu::GPUModuleOp>( |
| 82 | pass: xegpu::createXeGPUPropagateLayout(options: std::move(sgLayoutOptions))); |
| 83 | pm.addNestedPass<gpu::GPUModuleOp>(pass: xegpu::createXeGPUWgToSgDistribute()); |
| 84 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass()); |
| 85 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createLowerAffinePass()); |
| 86 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass()); |
| 87 | xegpu::XeGPUPropagateLayoutOptions instDataOptions; |
| 88 | instDataOptions.layoutKind = "inst" ; |
| 89 | pm.addNestedPass<gpu::GPUModuleOp>( |
| 90 | pass: xegpu::createXeGPUPropagateLayout(options: std::move(instDataOptions))); |
| 91 | pm.addNestedPass<gpu::GPUModuleOp>(pass: xegpu::createXeGPUBlocking()); |
| 92 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass()); |
| 93 | } |
| 94 | if (options.xegpuOpLevel == "subgroup" || |
| 95 | options.xegpuOpLevel == "workgroup" ) { |
| 96 | pm.addNestedPass<gpu::GPUModuleOp>( |
| 97 | pass: xegpu::createXeGPUPropagateLayout(options: laneLayoutOptions)); |
| 98 | pm.addNestedPass<gpu::GPUModuleOp>(pass: xegpu::createXeGPUPeepHoleOptimizer()); |
| 99 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass()); |
| 100 | pm.addNestedPass<gpu::GPUModuleOp>( |
| 101 | pass: xegpu::createXeGPUPropagateLayout(options: std::move(laneLayoutOptions))); |
| 102 | pm.addNestedPass<gpu::GPUModuleOp>(pass: xegpu::createXeGPUSgToLaneDistribute()); |
| 103 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createCanonicalizerPass()); |
| 104 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass()); |
| 105 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createLoopInvariantCodeMotionPass()); |
| 106 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass()); |
| 107 | pm.addNestedPass<gpu::GPUModuleOp>(pass: xegpu::createXeGPUVectorLinearize()); |
| 108 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createCanonicalizerPass()); |
| 109 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass()); |
| 110 | } |
| 111 | // Break down high-level micro-scaling (MX) ops (arith.scaling_extf and |
| 112 | // arith.scaling_truncf) into standard arith ops (extf/truncf + mulf), and |
| 113 | // expand extf/truncf on f8E8M0FNU into integer bit manipulation. This runs |
| 114 | // before the XeVM/LLVM conversions. The f4E2M1FN expansion patterns are |
| 115 | // intentionally left disabled: f4E2M1FN extf/truncf are lowered by the XeVM |
| 116 | // conversions (xevm.extf), whereas f8E8M0FNU is not supported there and so |
| 117 | // must be expanded here. |
| 118 | { |
| 119 | arith::ArithExpandOpsPassOptions arithExpandOptions; |
| 120 | arithExpandOptions.includeF8E8M0 = true; |
| 121 | pm.addNestedPass<gpu::GPUModuleOp>( |
| 122 | pass: arith::createArithExpandOpsPass(options: std::move(arithExpandOptions))); |
| 123 | } |
| 124 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createConvertMathToXeVM()); |
| 125 | ConvertXeGPUToXeVMPassOptions xegpuToXeVMOptions; |
| 126 | xegpuToXeVMOptions.use64bitIndex = options.use64bitIndex; |
| 127 | pm.addNestedPass<gpu::GPUModuleOp>( |
| 128 | pass: createConvertXeGPUToXeVMPass(options: std::move(xegpuToXeVMOptions))); |
| 129 | { |
| 130 | ConvertGpuOpsToLLVMSPVOpsOptions gpuToLLVMSPVOptions; |
| 131 | gpuToLLVMSPVOptions.use64bitIndex = options.use64bitIndex; |
| 132 | pm.addNestedPass<gpu::GPUModuleOp>( |
| 133 | pass: createConvertGpuOpsToLLVMSPVOps(options: std::move(gpuToLLVMSPVOptions))); |
| 134 | } |
| 135 | // Legalize math/arith ops on floating-point types that the XeVM target |
| 136 | // cannot handle natively (e.g. bf16) by wrapping them with extf/truncf |
| 137 | // around a supported type (defaulting to f32). |
| 138 | { |
| 139 | math::MathExtendToSupportedTypesOptions mathExtendOptions; |
| 140 | mathExtendOptions.extraTypeStrs.assign(in_start: options.mathExtendExtraTypes.begin(), |
| 141 | in_end: options.mathExtendExtraTypes.end()); |
| 142 | mathExtendOptions.targetTypeStr = options.supportedTargetTypes; |
| 143 | pm.addNestedPass<gpu::GPUModuleOp>( |
| 144 | pass: math::createMathExtendToSupportedTypes(options: std::move(mathExtendOptions))); |
| 145 | } |
| 146 | { |
| 147 | arith::ArithEmulateUnsupportedFloatsOptions arithEmulateOptions; |
| 148 | arithEmulateOptions.sourceTypeStrs.assign( |
| 149 | in_start: options.unsupportedSourceTypes.begin(), |
| 150 | in_end: options.unsupportedSourceTypes.end()); |
| 151 | arithEmulateOptions.targetTypeStr = options.supportedTargetTypes; |
| 152 | pm.addNestedPass<gpu::GPUModuleOp>( |
| 153 | pass: arith::createArithEmulateUnsupportedFloats( |
| 154 | options: std::move(arithEmulateOptions))); |
| 155 | } |
| 156 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass()); |
| 157 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createReconcileUnrealizedCastsPass()); |
| 158 | } |
| 159 | |
| 160 | //===----------------------------------------------------------------------===// |
| 161 | // Post-GPU pipeline for both Host and GPU. |
| 162 | //===----------------------------------------------------------------------===// |
| 163 | void buildPostGPUCommonPassPipeline( |
| 164 | OpPassManager &pm, const mlir::gpu::GPUToXeVMPipelineOptions &options) { |
| 165 | // builtin.module scope passes. |
| 166 | pm.addPass(pass: createConvertVectorToSCFPass()); |
| 167 | pm.addPass(pass: createSCFToControlFlowPass()); |
| 168 | pm.addPass(pass: memref::createExpandStridedMetadataPass()); |
| 169 | { |
| 170 | GpuToLLVMConversionPassOptions gpuToLLVMOptions; |
| 171 | gpuToLLVMOptions.hostBarePtrCallConv = options.hostBarePtrCallConv; |
| 172 | gpuToLLVMOptions.kernelBarePtrCallConv = options.kernelBarePtrCallConv; |
| 173 | pm.addPass(pass: createGpuToLLVMConversionPass(options: std::move(gpuToLLVMOptions))); |
| 174 | } |
| 175 | pm.addPass(pass: createLowerAffinePass()); |
| 176 | pm.addPass(pass: createConvertVectorToLLVMPass()); |
| 177 | pm.addPass(pass: createConvertToLLVMPass()); |
| 178 | pm.addPass(pass: createReconcileUnrealizedCastsPass()); |
| 179 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createCanonicalizerPass()); |
| 180 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass()); |
| 181 | // XeVM-to-LLVM must be the last pass before gpu-module-to-binary. |
| 182 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createConvertXeVMToLLVMPass()); |
| 183 | // gpu-module-to-binary |
| 184 | { |
| 185 | GpuModuleToBinaryPassOptions gpuToModuleBinOptions; |
| 186 | gpuToModuleBinOptions.compilationTarget = options.binaryFormat; |
| 187 | gpuToModuleBinOptions.cmdOptions = options.cmdOptions; |
| 188 | std::string skipTarget("skip" ); |
| 189 | if (options.binaryFormat.compare(str: skipTarget) != 0) |
| 190 | pm.addPass(pass: createGpuModuleToBinaryPass(options: std::move(gpuToModuleBinOptions))); |
| 191 | } |
| 192 | } |
| 193 | } // namespace |
| 194 | |
| 195 | void mlir::gpu::buildLowerToXeVMPassPipeline( |
| 196 | OpPassManager &pm, const GPUToXeVMPipelineOptions &options) { |
| 197 | // Pre-GPU common pipelines. |
| 198 | buildPreGPUCommonPassPipeline(pm, options); |
| 199 | |
| 200 | // GPUModule-specific stuff. |
| 201 | buildGPUPassPipeline(pm, options); |
| 202 | |
| 203 | // Post-GPU pipeline for both Host and GPU. |
| 204 | buildPostGPUCommonPassPipeline(pm, options); |
| 205 | } |
| 206 | |
| 207 | void mlir::gpu::registerGPUToXeVMPipeline() { |
| 208 | PassPipelineRegistration<GPUToXeVMPipelineOptions>( |
| 209 | "gpu-lower-to-xevm-pipeline" , |
| 210 | "The default GPU to XeVM lowering pipeline. It starts by lowering GPU " |
| 211 | "code to the " |
| 212 | "specified compilation target (default is fatbin) then lowers the host " |
| 213 | "code." , |
| 214 | buildLowerToXeVMPassPipeline); |
| 215 | } |
| 216 | |