1//===- GPUToXeVMPipeline.cpp - Lowering pipeline to XeVM/LLVM -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a pass for testing the lowering to XeVM as a generally
10// usable sink pass. If XeGPU ops are used, it expects the MLIR code to have
11// XeGPU ops already embedded in gpu code.
12//
13//===----------------------------------------------------------------------===//
14
15#include "mlir/Conversion/AffineToStandard/AffineToStandard.h"
16#include "mlir/Conversion/ConvertToLLVM/ToLLVMPass.h"
17#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
18#include "mlir/Conversion/GPUToLLVMSPV/GPUToLLVMSPVPass.h"
19#include "mlir/Conversion/MathToXeVM/MathToXeVM.h"
20#include "mlir/Conversion/ReconcileUnrealizedCasts/ReconcileUnrealizedCasts.h"
21#include "mlir/Conversion/SCFToControlFlow/SCFToControlFlow.h"
22#include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.h"
23#include "mlir/Conversion/VectorToSCF/VectorToSCF.h"
24#include "mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h"
25#include "mlir/Conversion/XeGPUToXeVM/XeGPUToXeVM.h"
26#include "mlir/Conversion/XeVMToLLVM/XeVMToLLVM.h"
27#include "mlir/Dialect/Arith/Transforms/Passes.h"
28#include "mlir/Dialect/Func/IR/FuncOps.h"
29#include "mlir/Dialect/GPU/IR/GPUDialect.h"
30#include "mlir/Dialect/GPU/Pipelines/Passes.h"
31#include "mlir/Dialect/GPU/Transforms/Passes.h"
32#include "mlir/Dialect/GPU/Transforms/XeVM/Passes.h"
33#include "mlir/Dialect/LLVMIR/Transforms/RequestCWrappers.h"
34#include "mlir/Dialect/Math/Transforms/Passes.h"
35#include "mlir/Dialect/MemRef/Transforms/Passes.h"
36#include "mlir/Dialect/XeGPU/Transforms/Passes.h"
37#include "mlir/Pass/PassManager.h"
38#include "mlir/Pass/PassOptions.h"
39#include "mlir/Target/LLVM/XeVM/Target.h"
40#include "mlir/Transforms/Passes.h"
41
42#include <utility>
43
44using namespace mlir;
45
46namespace {
47//===----------------------------------------------------------------------===//
48// Pre-GPU common pipeline for both Host and GPU.
49//===----------------------------------------------------------------------===//
50void buildPreGPUCommonPassPipeline(
51 OpPassManager &pm, const mlir::gpu::GPUToXeVMPipelineOptions &options) {
52 // builtin.module scope passes.
53 pm.addPass(pass: createCSEPass());
54 {
55 GpuXeVMAttachTargetOptions xevmTargetOptions;
56 xevmTargetOptions.moduleMatcher = options.xevmModuleMatcher;
57 xevmTargetOptions.triple = options.zebinTriple;
58 xevmTargetOptions.chip = options.zebinChip;
59 xevmTargetOptions.optLevel = options.optLevel;
60 xevmTargetOptions.cmdOptions = options.cmdOptions;
61 pm.addPass(pass: createGpuXeVMAttachTarget(options: std::move(xevmTargetOptions)));
62 }
63 pm.addPass(pass: createLowerAffinePass());
64 pm.addNestedPass<func::FuncOp>(pass: createGpuAsyncRegionPass());
65}
66
67//===----------------------------------------------------------------------===//
68// GPUModule-specific stuff.
69//===----------------------------------------------------------------------===//
70void buildGPUPassPipeline(OpPassManager &pm,
71 const mlir::gpu::GPUToXeVMPipelineOptions &options) {
72 xegpu::XeGPUPropagateLayoutOptions laneLayoutOptions;
73 laneLayoutOptions.indexBitWidth = options.use64bitIndex ? 64 : 32;
74 laneLayoutOptions.layoutKind = "lane";
75 pm.addNestedPass<ModuleOp>(pass: createCSEPass());
76 if (options.enableVectorToXeGPU)
77 pm.addNestedPass<gpu::GPUModuleOp>(pass: createConvertVectorToXeGPU());
78 if (options.xegpuOpLevel == "workgroup") {
79 xegpu::XeGPUPropagateLayoutOptions sgLayoutOptions;
80 sgLayoutOptions.layoutKind = "subgroup";
81 pm.addNestedPass<gpu::GPUModuleOp>(
82 pass: xegpu::createXeGPUPropagateLayout(options: std::move(sgLayoutOptions)));
83 pm.addNestedPass<gpu::GPUModuleOp>(pass: xegpu::createXeGPUWgToSgDistribute());
84 pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass());
85 pm.addNestedPass<gpu::GPUModuleOp>(pass: createLowerAffinePass());
86 pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass());
87 xegpu::XeGPUPropagateLayoutOptions instDataOptions;
88 instDataOptions.layoutKind = "inst";
89 pm.addNestedPass<gpu::GPUModuleOp>(
90 pass: xegpu::createXeGPUPropagateLayout(options: std::move(instDataOptions)));
91 pm.addNestedPass<gpu::GPUModuleOp>(pass: xegpu::createXeGPUBlocking());
92 pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass());
93 }
94 if (options.xegpuOpLevel == "subgroup" ||
95 options.xegpuOpLevel == "workgroup") {
96 pm.addNestedPass<gpu::GPUModuleOp>(
97 pass: xegpu::createXeGPUPropagateLayout(options: laneLayoutOptions));
98 pm.addNestedPass<gpu::GPUModuleOp>(pass: xegpu::createXeGPUPeepHoleOptimizer());
99 pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass());
100 pm.addNestedPass<gpu::GPUModuleOp>(
101 pass: xegpu::createXeGPUPropagateLayout(options: std::move(laneLayoutOptions)));
102 pm.addNestedPass<gpu::GPUModuleOp>(pass: xegpu::createXeGPUSgToLaneDistribute());
103 pm.addNestedPass<gpu::GPUModuleOp>(pass: createCanonicalizerPass());
104 pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass());
105 pm.addNestedPass<gpu::GPUModuleOp>(pass: createLoopInvariantCodeMotionPass());
106 pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass());
107 pm.addNestedPass<gpu::GPUModuleOp>(pass: xegpu::createXeGPUVectorLinearize());
108 pm.addNestedPass<gpu::GPUModuleOp>(pass: createCanonicalizerPass());
109 pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass());
110 }
111 // Break down high-level micro-scaling (MX) ops (arith.scaling_extf and
112 // arith.scaling_truncf) into standard arith ops (extf/truncf + mulf), and
113 // expand extf/truncf on f8E8M0FNU into integer bit manipulation. This runs
114 // before the XeVM/LLVM conversions. The f4E2M1FN expansion patterns are
115 // intentionally left disabled: f4E2M1FN extf/truncf are lowered by the XeVM
116 // conversions (xevm.extf), whereas f8E8M0FNU is not supported there and so
117 // must be expanded here.
118 {
119 arith::ArithExpandOpsPassOptions arithExpandOptions;
120 arithExpandOptions.includeF8E8M0 = true;
121 pm.addNestedPass<gpu::GPUModuleOp>(
122 pass: arith::createArithExpandOpsPass(options: std::move(arithExpandOptions)));
123 }
124 pm.addNestedPass<gpu::GPUModuleOp>(pass: createConvertMathToXeVM());
125 ConvertXeGPUToXeVMPassOptions xegpuToXeVMOptions;
126 xegpuToXeVMOptions.use64bitIndex = options.use64bitIndex;
127 pm.addNestedPass<gpu::GPUModuleOp>(
128 pass: createConvertXeGPUToXeVMPass(options: std::move(xegpuToXeVMOptions)));
129 {
130 ConvertGpuOpsToLLVMSPVOpsOptions gpuToLLVMSPVOptions;
131 gpuToLLVMSPVOptions.use64bitIndex = options.use64bitIndex;
132 pm.addNestedPass<gpu::GPUModuleOp>(
133 pass: createConvertGpuOpsToLLVMSPVOps(options: std::move(gpuToLLVMSPVOptions)));
134 }
135 // Legalize math/arith ops on floating-point types that the XeVM target
136 // cannot handle natively (e.g. bf16) by wrapping them with extf/truncf
137 // around a supported type (defaulting to f32).
138 {
139 math::MathExtendToSupportedTypesOptions mathExtendOptions;
140 mathExtendOptions.extraTypeStrs.assign(in_start: options.mathExtendExtraTypes.begin(),
141 in_end: options.mathExtendExtraTypes.end());
142 mathExtendOptions.targetTypeStr = options.supportedTargetTypes;
143 pm.addNestedPass<gpu::GPUModuleOp>(
144 pass: math::createMathExtendToSupportedTypes(options: std::move(mathExtendOptions)));
145 }
146 {
147 arith::ArithEmulateUnsupportedFloatsOptions arithEmulateOptions;
148 arithEmulateOptions.sourceTypeStrs.assign(
149 in_start: options.unsupportedSourceTypes.begin(),
150 in_end: options.unsupportedSourceTypes.end());
151 arithEmulateOptions.targetTypeStr = options.supportedTargetTypes;
152 pm.addNestedPass<gpu::GPUModuleOp>(
153 pass: arith::createArithEmulateUnsupportedFloats(
154 options: std::move(arithEmulateOptions)));
155 }
156 pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass());
157 pm.addNestedPass<gpu::GPUModuleOp>(pass: createReconcileUnrealizedCastsPass());
158}
159
160//===----------------------------------------------------------------------===//
161// Post-GPU pipeline for both Host and GPU.
162//===----------------------------------------------------------------------===//
163void buildPostGPUCommonPassPipeline(
164 OpPassManager &pm, const mlir::gpu::GPUToXeVMPipelineOptions &options) {
165 // builtin.module scope passes.
166 pm.addPass(pass: createConvertVectorToSCFPass());
167 pm.addPass(pass: createSCFToControlFlowPass());
168 pm.addPass(pass: memref::createExpandStridedMetadataPass());
169 {
170 GpuToLLVMConversionPassOptions gpuToLLVMOptions;
171 gpuToLLVMOptions.hostBarePtrCallConv = options.hostBarePtrCallConv;
172 gpuToLLVMOptions.kernelBarePtrCallConv = options.kernelBarePtrCallConv;
173 pm.addPass(pass: createGpuToLLVMConversionPass(options: std::move(gpuToLLVMOptions)));
174 }
175 pm.addPass(pass: createLowerAffinePass());
176 pm.addPass(pass: createConvertVectorToLLVMPass());
177 pm.addPass(pass: createConvertToLLVMPass());
178 pm.addPass(pass: createReconcileUnrealizedCastsPass());
179 pm.addNestedPass<gpu::GPUModuleOp>(pass: createCanonicalizerPass());
180 pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass());
181 // XeVM-to-LLVM must be the last pass before gpu-module-to-binary.
182 pm.addNestedPass<gpu::GPUModuleOp>(pass: createConvertXeVMToLLVMPass());
183 // gpu-module-to-binary
184 {
185 GpuModuleToBinaryPassOptions gpuToModuleBinOptions;
186 gpuToModuleBinOptions.compilationTarget = options.binaryFormat;
187 gpuToModuleBinOptions.cmdOptions = options.cmdOptions;
188 std::string skipTarget("skip");
189 if (options.binaryFormat.compare(str: skipTarget) != 0)
190 pm.addPass(pass: createGpuModuleToBinaryPass(options: std::move(gpuToModuleBinOptions)));
191 }
192}
193} // namespace
194
195void mlir::gpu::buildLowerToXeVMPassPipeline(
196 OpPassManager &pm, const GPUToXeVMPipelineOptions &options) {
197 // Pre-GPU common pipelines.
198 buildPreGPUCommonPassPipeline(pm, options);
199
200 // GPUModule-specific stuff.
201 buildGPUPassPipeline(pm, options);
202
203 // Post-GPU pipeline for both Host and GPU.
204 buildPostGPUCommonPassPipeline(pm, options);
205}
206
207void mlir::gpu::registerGPUToXeVMPipeline() {
208 PassPipelineRegistration<GPUToXeVMPipelineOptions>(
209 "gpu-lower-to-xevm-pipeline",
210 "The default GPU to XeVM lowering pipeline. It starts by lowering GPU "
211 "code to the "
212 "specified compilation target (default is fatbin) then lowers the host "
213 "code.",
214 buildLowerToXeVMPassPipeline);
215}
216