1//===- XeGPUPeepHoleOptimizer.cpp - XeGPU optimize block loads -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Arith/IR/Arith.h"
10#include "mlir/Dialect/GPU/IR/GPUDialect.h"
11#include "mlir/Dialect/MemRef/IR/MemRef.h"
12#include "mlir/Dialect/SCF/Transforms/Patterns.h"
13#include "mlir/Dialect/Utils/IndexingUtils.h"
14#include "mlir/Dialect/Utils/StaticValueUtils.h"
15#include "mlir/Dialect/Vector/IR/VectorOps.h"
16#include "mlir/Dialect/XeGPU/IR/XeGPU.h"
17#include "mlir/Dialect/XeGPU/Transforms/Passes.h"
18#include "mlir/Dialect/XeGPU/Transforms/Transforms.h"
19#include "mlir/Dialect/XeGPU/Transforms/XeGPULayoutImpl.h"
20#include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
21#include "mlir/Dialect/XeGPU/uArch/uArchBase.h"
22#include "mlir/Dialect/XeGPU/uArch/uArchCommon.h"
23#include "mlir/IR/BuiltinTypes.h"
24#include "mlir/IR/OpDefinition.h"
25#include "mlir/IR/Types.h"
26#include "mlir/IR/Value.h"
27#include "mlir/Transforms/DialectConversion.h"
28#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
29#include "llvm/ADT/STLExtras.h"
30#include "llvm/ADT/SmallVector.h"
31#include <optional>
32
33namespace mlir {
34namespace xegpu {
35#define GEN_PASS_DEF_XEGPUPEEPHOLEOPTIMIZER
36#include "mlir/Dialect/XeGPU/Transforms/Passes.h.inc"
37} // namespace xegpu
38} // namespace mlir
39
40#define DEBUG_TYPE "xegpu-optimize-peephole"
41#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE "]: ")
42
43using namespace mlir;
44
45namespace {
46
47/// Get the 2D lane data from a tensor desc type if it exists.
48static std::optional<SmallVector<int64_t>>
49getMaybeLaneData(xegpu::TensorDescType tdescType) {
50 auto layout = tdescType.getLayoutAttr();
51 if (!layout)
52 return std::nullopt;
53 return xegpu::getInner2DIfUnitLeadingDims(vals: layout.getEffectiveLaneDataAsInt());
54}
55
56/// Get the 2D lane layout from a tensor desc type if it exists.
57static std::optional<SmallVector<int64_t>>
58getMaybeLaneLayout(xegpu::TensorDescType tdescType) {
59 auto layout = tdescType.getLayoutAttr();
60 if (!layout)
61 return std::nullopt;
62 return xegpu::getInner2DIfUnitLeadingDims(
63 vals: layout.getEffectiveLaneLayoutAsInt());
64}
65
66/// A layout can be optimized if its lane layout is transposed (lane[0] != 1 &&
67/// lane[1] == 1), but inner lane data is not equal to [1, 1].
68/// Example:
69/// !xegpu.tensor_desc<16x16xf16,
70/// #xegpu.layout<lane_layout = [16, 1], lane_data = [1, 2]>>
71/// In this case, lane layout is transposed (from the usual [1, SG_SIZE] form)
72/// indicating that this is a load that requires transpose effect. However,
73/// lane data is [1, 2], meaning that each lane must grab 2 f16 elements from
74/// the inner dimension. We convert this to a optimized form by converting the
75/// tensor_desc to i32 type such that lane data becomes [1, 1]. This makes the
76/// later lowering easily use the load with transpose instruction.
77static bool canBeOptimizedForTranspose(ArrayRef<int64_t> laneLayout,
78 ArrayRef<int64_t> laneData) {
79 if (laneLayout.size() != 2 || laneData.size() != 2)
80 return false;
81 if (laneLayout[0] == 1 || laneLayout[1] != 1)
82 return false;
83 if (laneData[0] != 1 || laneData[1] == 1)
84 return false;
85 return true;
86}
87
88/// A tensor desc type can be optimized if its element type is less than 32 bits
89/// and its layout can be optimized.
90static bool canBeOptimizedForTranspose(xegpu::TensorDescType tdescType) {
91 // If the dtype is greater or equal to 32 bits, layout must be valid.
92 int elementTyBitwidth = tdescType.getElementType().getIntOrFloatBitWidth();
93 if (elementTyBitwidth >= 32)
94 return false;
95 auto maybeLaneLayout = getMaybeLaneLayout(tdescType);
96 auto maybeLaneData = getMaybeLaneData(tdescType);
97 if (!maybeLaneData || !maybeLaneLayout)
98 return false;
99 return canBeOptimizedForTranspose(laneLayout: *maybeLaneLayout, laneData: *maybeLaneData);
100}
101
102/// Check if a tensor desc type can be optimized for transpose, if so return the
103/// new optimized tensor desc type with a valid transpose layout.
104static xegpu::TensorDescType
105tryOptimize(xegpu::TensorDescType tdescType,
106 const xegpu::uArch::uArch *targetuArch) {
107 if (!canBeOptimizedForTranspose(tdescType))
108 return tdescType;
109 auto laneData = getMaybeLaneData(tdescType)
110 .value(); // Lane data must exist if we reach here.
111 int64_t innerLaneData = laneData[1];
112 int elementTyBitwidth = tdescType.getElementType().getIntOrFloatBitWidth();
113 // Required shape is total shape of the vector result that this tensor desc
114 // must eventually load after adjusting for the new bitwidth and array
115 // length.
116 SmallVector<int64_t> requiredShape(tdescType.getShape());
117 requiredShape.back() =
118 requiredShape.back() * tdescType.getArrayLength() / innerLaneData;
119 int newBitWidth = elementTyBitwidth * innerLaneData;
120 Type newElemTy = IntegerType::get(context: tdescType.getContext(), width: newBitWidth);
121 // Supported shape is the max transpose shape that can be supported by
122 // hardware that is less than or equal to required shape.
123 auto *blockLoadTarget =
124 dyn_cast<xegpu::uArch::Subgroup2DBlockLoadInstruction>(
125 Val: targetuArch->getInstruction(
126 instKind: xegpu::uArch::InstructionKind::Subgroup2DBlockLoad));
127 auto maybeHWParams = blockLoadTarget->getBlockWidthHeightCount(
128 elemTy: newElemTy, /** has transform */ hasTransform: false, /** has transpose */ hasTranspose: true);
129 // If no HW params found, return the original type.
130 if (!maybeHWParams)
131 return tdescType;
132 auto [widths, heights, counts] = maybeHWParams.value();
133 // TODO: Currently we expect array length to be 1 for transpose case.
134 if (counts.size() != 1 || counts[0] != 1)
135 return tdescType;
136 int arrayLen = counts[0];
137 // The transpose acts on the innermost 2 dims; any leading dims are unit
138 // batch.
139 int64_t rank = requiredShape.size();
140 int supportedHeight = xegpu::getLargestDivisor(
141 dim: static_cast<int>(requiredShape[rank - 2]), candidates: heights);
142 int supportedWidth = xegpu::getLargestDivisor(
143 dim: static_cast<int>(requiredShape[rank - 1]), candidates: widths);
144 // If no supported height or width found, return the original type.
145 if (supportedHeight == -1 || supportedWidth == -1)
146 return tdescType;
147
148 // Preserve leading (unit) batch dims; only the inner 2 dims are reshaped.
149 SmallVector<int64_t> supportedShape(requiredShape.begin(),
150 requiredShape.end() - 2);
151 supportedShape.push_back(Elt: supportedHeight);
152 supportedShape.push_back(Elt: supportedWidth);
153 auto ctx = tdescType.getContext();
154 auto origLayout = tdescType.getLayoutAttr();
155 auto laneLayoutI64 = origLayout.getEffectiveLaneLayoutAsInt();
156 SmallVector<int32_t> laneLayoutI32(laneLayoutI64.begin(),
157 laneLayoutI64.end());
158
159 xegpu::LayoutAttr newLayout = xegpu::LayoutAttr::get(
160 context: ctx, /*lane_layout=*/DenseI32ArrayAttr::get(context: ctx, content: laneLayoutI32),
161 /*lane_data=*/
162 DenseI32ArrayAttr::get(context: ctx,
163 content: SmallVector<int32_t>(laneLayoutI32.size(), 1)),
164 /*order=*/origLayout.getOrder());
165
166 // Array length can not be larger than 1 for transpose case.
167 return xegpu::TensorDescType::get(shape: supportedShape, elementType: newElemTy, array_length: arrayLen,
168 boundary_check: tdescType.getBoundaryCheck(),
169 memory_space: tdescType.getMemorySpace(), layout: newLayout);
170}
171
172/// Helper to convert an OpFoldResult to Value.
173static Value convertToValue(ConversionPatternRewriter &rewriter, Location loc,
174 OpFoldResult ofr) {
175 std::optional<int64_t> mayBeInt = getConstantIntValue(ofr);
176 if (mayBeInt)
177 return arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: *mayBeInt).getResult();
178 return llvm::cast<Value>(Val&: ofr);
179}
180
181/// Helper to divide a Value by a constant integer.
182static Value divideByConstant(ConversionPatternRewriter &rewriter, Location loc,
183 Value val, int64_t constant) {
184 // If the constant is a power of 2, use right shift for division.
185 if (llvm::isPowerOf2_64(Value: constant)) {
186 int64_t shiftAmount = llvm::Log2_64(Value: constant);
187 return arith::ShRUIOp::create(
188 builder&: rewriter, location: loc, lhs: val,
189 rhs: arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: shiftAmount)
190 .getResult())
191 .getResult();
192 }
193 auto constantOp =
194 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: constant).getResult();
195 return arith::DivUIOp::create(builder&: rewriter, location: loc, lhs: val, rhs: constantOp).getResult();
196}
197
198/// This function takes a larger register block `data` and generates multiple
199/// smaller loads (size given by `newTensorDesc`) to fill in the `data` block
200/// starting from `offsets`.
201static Value generateLoads(ConversionPatternRewriter &rewriter,
202 TypedValue<VectorType> data,
203 SmallVector<OpFoldResult> offsets,
204 TypedValue<xegpu::TensorDescType> newTensorDesc,
205 xegpu::LoadNdOp origLoadOp) {
206 Location loc = data.getLoc();
207 assert(offsets.size() >= 2 && "Expecting at least 2 offsets for 2D LoadNdOp");
208 int64_t rank = data.getType().getRank();
209 Value offsetDim0 = convertToValue(rewriter, loc, ofr: offsets[offsets.size() - 2]);
210 Value offsetDim1 = convertToValue(rewriter, loc, ofr: offsets[offsets.size() - 1]);
211 SmallVector<int64_t> supportedShape(newTensorDesc.getType().getShape());
212 // Compute the ratio between original shape and supported shape. We need to
213 // generate loads in this ratio arrangement.
214 auto shapeRatio = computeShapeRatio(shape: data.getType().getShape(),
215 subShape: supportedShape)
216 .value(); // `ratio` must be defined if we reach here.
217 // The loop below only walks the last 2 entries of `shapeRatio`; leading
218 // (batch) dims are unit, so their ratio is 1 and their offsets pass through
219 // unchanged.
220 int64_t suppDim0 = supportedShape[rank - 2];
221 int64_t suppDim1 = supportedShape[rank - 1];
222 for (int64_t h = 0; h < shapeRatio[rank - 2]; ++h) {
223 for (int64_t w = 0; w < shapeRatio[rank - 1]; ++w) {
224 int64_t localOffsetDim0 = h * suppDim0;
225 int64_t localOffsetDim1 = w * suppDim1;
226 Value loadOffsetX = arith::AddIOp::create(
227 builder&: rewriter, location: loc, lhs: offsetDim0,
228 rhs: arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: localOffsetDim0)
229 .getResult());
230 Value loadOffsetY = arith::AddIOp::create(
231 builder&: rewriter, location: loc, lhs: offsetDim1,
232 rhs: arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: localOffsetDim1)
233 .getResult());
234 // Keep the leading (batch) offsets; replace only the inner 2.
235 SmallVector<OpFoldResult> loadOffsets(offsets.begin(), offsets.end());
236 loadOffsets[loadOffsets.size() - 2] = loadOffsetX;
237 loadOffsets[loadOffsets.size() - 1] = loadOffsetY;
238 auto loadOp = xegpu::LoadNdOp::create(
239 builder&: rewriter, location: loc,
240 value: VectorType::get(shape: supportedShape, elementType: data.getType().getElementType()),
241 TensorDesc: newTensorDesc, offsets: loadOffsets, packed: origLoadOp.getPackedAttr(),
242 transpose: origLoadOp.getTransposeAttr(), l1_hint: origLoadOp.getL1HintAttr(),
243 l2_hint: origLoadOp.getL2HintAttr(), l3_hint: origLoadOp.getL3HintAttr(),
244 layout: origLoadOp.getLayoutAttr());
245 // Set the layout for the loadOp.
246 auto layoutAttr = newTensorDesc.getType().getLayoutAttr();
247 loadOp.setAnchorLayout(layoutAttr);
248 // Insert the loaded block into the right position in data (leading dims
249 // at 0, inner 2 dims at the local tile offset).
250 SmallVector<int64_t> insertPos(rank, 0);
251 insertPos[rank - 2] = localOffsetDim0;
252 insertPos[rank - 1] = localOffsetDim1;
253 SmallVector<int64_t> insertStrides(rank, 1);
254 auto insertOp = vector::InsertStridedSliceOp::create(
255 builder&: rewriter, location: loc, valueToStore: loadOp.getResult(), dest: data, offsets: insertPos, strides: insertStrides);
256 // InsertOp must have the same layout as newTensorDesc.
257 xegpu::setTemporaryLayout(operandOrResult: insertOp->getOpResult(idx: 0), layout: layoutAttr);
258 data = insertOp.getResult();
259 }
260 }
261 return data;
262}
263
264/// Checks if a CreateNdDescOp can be optimized for transpose, if so creates a
265/// new CreateNdDescOp with optimized tensor desc type. This involves extracting
266/// the base pointer from the original memory source and adjusting the shape and
267/// strides of the tensor desc to fit with the new optimized transpose layout.
268class XeGPUCreateNdDescOpPattern final
269 : public OpConversionPattern<xegpu::CreateNdDescOp> {
270public:
271 using OpConversionPattern<xegpu::CreateNdDescOp>::OpConversionPattern;
272 LogicalResult
273 matchAndRewrite(xegpu::CreateNdDescOp createNdOp, OpAdaptor adaptor,
274 ConversionPatternRewriter &rewriter) const override {
275 auto tdescTy = createNdOp.getType();
276 // Get the target uArch info.
277 auto chipStr = xegpu::getChipStr(op: createNdOp);
278 // Check if the chip is supported.
279 assert(chipStr &&
280 (chipStr.value() == "pvc" || chipStr.value() == "bmg" ||
281 chipStr.value() == "cri") &&
282 "Expecting target chip to be pvc, bmg or cri for transpose "
283 "optimization.");
284 const auto *targetuArch = xegpu::uArch::getUArch(archName: chipStr.value());
285
286 auto convertType = tryOptimize(tdescType: tdescTy, targetuArch);
287 if (convertType == tdescTy)
288 return failure();
289 Location loc = createNdOp.getLoc();
290 Value source = createNdOp.getSource();
291 auto memrefType = dyn_cast<MemRefType>(Val: source.getType());
292
293 SmallVector<OpFoldResult> mixedSizes;
294 SmallVector<OpFoldResult> mixedStrides;
295 memref::ExtractStridedMetadataOp meta;
296 if (memrefType) {
297 // extract_aligned_pointer_as_index returns the base pointer without the
298 // memref offset, so always extract strided metadata to fold it in.
299 meta = memref::ExtractStridedMetadataOp::create(builder&: rewriter, location: loc, source);
300 mixedSizes = meta.getConstifiedMixedSizes();
301 mixedStrides = meta.getConstifiedMixedStrides();
302 } else {
303 mixedSizes = createNdOp.getMixedSizes();
304 mixedStrides = createNdOp.getMixedStrides();
305 }
306
307 auto maybeConstInnerStride = getConstantIntValue(ofr: mixedStrides.back());
308 // Only row-major memrefs are expected for now.
309 if (!maybeConstInnerStride || *maybeConstInnerStride != 1)
310 return rewriter.notifyMatchFailure(
311 arg&: createNdOp, msg: "Expecting row-major memref for transpose optimization.");
312 auto optionalLaneData = getMaybeLaneData(tdescType: tdescTy);
313 assert(optionalLaneData && "Expected 2D lane data");
314 auto laneData = optionalLaneData.value();
315 int64_t innerLaneData = laneData[1];
316 // Inner dimension of the shape must be adjusted based on innerLaneData.
317 SmallVector<OpFoldResult> modifiedShape(mixedSizes);
318 modifiedShape.back() = divideByConstant(
319 rewriter, loc, val: convertToValue(rewriter, loc, ofr: modifiedShape.back()),
320 constant: innerLaneData);
321 // Repacking to a wider element rescales every stride but the innermost.
322 assert(mixedStrides.size() >= 2 &&
323 "Expected at least 2 strides for CreateNdDescOp");
324 SmallVector<OpFoldResult> modifiedStrides(mixedStrides);
325 for (size_t i = 0; i + 1 < modifiedStrides.size(); ++i)
326 modifiedStrides[i] = divideByConstant(
327 rewriter, loc, val: convertToValue(rewriter, loc, ofr: modifiedStrides[i]),
328 constant: innerLaneData);
329
330 if (memrefType) {
331 // Compute base = aligned pointer + offset (in bytes).
332 Value alignedPtr = memref::ExtractAlignedPointerAsIndexOp::create(
333 builder&: rewriter, location: loc, source: meta.getBaseBuffer());
334 Value baseIdx = xegpu::addElemOffsetInBytes(builder&: rewriter, loc, baseAddr: alignedPtr,
335 elemOffset: meta.getOffset(), memrefTy: memrefType);
336 source = arith::IndexCastOp::create(builder&: rewriter, location: loc, out: rewriter.getI64Type(),
337 in: baseIdx);
338 }
339 // Create a new CreateNdDescOp with the modified shape and converted type.
340 auto newCreateNdDescOp = xegpu::CreateNdDescOp::create(
341 builder&: rewriter, location: loc, tdesc: convertType, source, shape: modifiedShape, strides: modifiedStrides);
342 rewriter.replaceOp(op: createNdOp, newValues: newCreateNdDescOp.getResult());
343 return success();
344 }
345};
346
347/// Checks if a LoadNdOp consumes a tensor desc type that was rewritten for
348/// tranpose optimization. If so, rewrites the LoadNdOp to to align with the
349/// adjusted tensor desc type. This can result in multiple LoadNdOps being
350/// generated to fill in the original load shape.
351class XeGPULoadNdDescOpPattern final
352 : public OpConversionPattern<xegpu::LoadNdOp> {
353public:
354 using OpConversionPattern<xegpu::LoadNdOp>::OpConversionPattern;
355 LogicalResult
356 matchAndRewrite(xegpu::LoadNdOp loadNdOp, OpAdaptor adaptor,
357 ConversionPatternRewriter &rewriter) const override {
358 auto origTensorDescType = loadNdOp.getTensorDescType();
359 auto adaptorType =
360 cast<xegpu::TensorDescType>(Val: adaptor.getTensorDesc().getType());
361 if (adaptorType == origTensorDescType)
362 return failure();
363 // Offsets must be adjusted based on innerLaneData.
364 auto laneData = getMaybeLaneData(tdescType: loadNdOp.getTensorDescType()).value();
365 int64_t innerLaneData = laneData[1];
366 auto offsets = loadNdOp.getMixedOffsets();
367 if (offsets.empty())
368 return rewriter.notifyMatchFailure(arg&: loadNdOp,
369 msg: "Expecting offsets in LoadNd");
370 SmallVector<OpFoldResult> modifiedOffsets(offsets);
371 modifiedOffsets.back() = divideByConstant(
372 rewriter, loc: loadNdOp.getLoc(),
373 val: convertToValue(rewriter, loc: loadNdOp.getLoc(), ofr: modifiedOffsets.back()),
374 constant: innerLaneData);
375 // Get the 2D data shape of this loadNdOp in its original type including
376 // array length.
377 SmallVector<int64_t> origDataShape(origTensorDescType.getShape());
378 // Adjust the data shape based on innerLaneData.
379 origDataShape.back() /= innerLaneData;
380 // HW supported shape is the new tensor desc shape after conversion.
381 SmallVector<int64_t> hwSupportedShape(adaptorType.getShape());
382 VectorType origVectorType =
383 VectorType::get(shape: origDataShape, elementType: adaptorType.getElementType());
384 Value data;
385 // Orig data shape is 3D for the array length case.
386 if (origTensorDescType.getArrayLength() > 1) {
387 SmallVector<Value> arraySlices;
388 for (int64_t i = 0; i < origTensorDescType.getArrayLength(); ++i) {
389 Value slice = arith::ConstantOp::create(
390 builder&: rewriter, location: loadNdOp->getLoc(), result: origVectorType,
391 value: rewriter.getZeroAttr(type: origVectorType));
392 // Increase the Y offset for each array slice.
393 Value offsetY = convertToValue(rewriter, loc: loadNdOp->getLoc(),
394 ofr: modifiedOffsets.back());
395 modifiedOffsets.back() =
396 arith::AddIOp::create(
397 builder&: rewriter, location: loadNdOp->getLoc(), lhs: offsetY,
398 rhs: arith::ConstantIndexOp::create(builder&: rewriter, location: loadNdOp->getLoc(),
399 value: i * origDataShape[1])
400 .getResult())
401 .getResult();
402 slice = generateLoads(
403 rewriter, data: cast<TypedValue<VectorType>>(Val&: slice), offsets: modifiedOffsets,
404 newTensorDesc: cast<TypedValue<xegpu::TensorDescType>>(Val: adaptor.getTensorDesc()),
405 origLoadOp: loadNdOp);
406 // BitCast back to original load shape without array length.
407 auto bitcastType = VectorType::get(shape: origTensorDescType.getShape(),
408 elementType: origTensorDescType.getElementType());
409 auto bitCastOp = vector::BitCastOp::create(builder&: rewriter, location: loadNdOp->getLoc(),
410 result: bitcastType, source: slice);
411 // BitCastOp must have the same layout as the original loadNdOp.
412 xegpu::setTemporaryLayout(operandOrResult: bitCastOp->getOpResult(idx: 0),
413 layout: origTensorDescType.getLayoutAttr());
414 arraySlices.push_back(Elt: bitCastOp.getResult());
415 }
416 rewriter.replaceOpWithMultiple(op: loadNdOp, newValues: {arraySlices});
417 return success();
418 }
419 data = arith::ConstantOp::create(
420 builder&: rewriter, location: loadNdOp->getLoc(),
421 result: VectorType::get(shape: origDataShape, elementType: adaptorType.getElementType()),
422 value: rewriter.getZeroAttr(type: origVectorType));
423 data = generateLoads(
424 rewriter, data: cast<TypedValue<VectorType>>(Val&: data), offsets: modifiedOffsets,
425 newTensorDesc: cast<TypedValue<xegpu::TensorDescType>>(Val: adaptor.getTensorDesc()),
426 origLoadOp: loadNdOp);
427 auto bitCastOp = vector::BitCastOp::create(builder&: rewriter, location: loadNdOp->getLoc(),
428 result: loadNdOp.getType(), source: data);
429 // BitCastOp must have the same layout as the original loadNdOp.
430 xegpu::setTemporaryLayout(operandOrResult: bitCastOp->getOpResult(idx: 0),
431 layout: origTensorDescType.getLayoutAttr());
432 rewriter.replaceOp(op: loadNdOp, newOp: bitCastOp);
433 return success();
434 }
435};
436
437/// Vector ExtractOp must be processed if the original tensor desc type has
438/// array length greater than 1. In this case, the LoadNdOp is replaced with
439/// multiple LoadNdOps for each array slice making the extraction unnecessary.
440/// In this case, we simply remove the ExtractOp.
441class VectorExtractOpPattern final
442 : public OpConversionPattern<vector::ExtractOp> {
443public:
444 using OpConversionPattern<vector::ExtractOp>::OpConversionPattern;
445 LogicalResult
446 matchAndRewrite(vector::ExtractOp extractOp, OneToNOpAdaptor adaptor,
447 ConversionPatternRewriter &rewriter) const override {
448 // Check if the source of the extraction is split to multiple values.
449 if (adaptor.getSource().size() == 1)
450 return failure();
451 auto mixedPos = extractOp.getMixedPosition();
452 if (mixedPos.size() != 1)
453 return failure();
454 auto mayBeInt = getConstantIntValue(ofr: mixedPos[0]);
455 if (!mayBeInt)
456 return failure();
457 rewriter.replaceOp(op: extractOp, newValues: adaptor.getSource()[*mayBeInt]);
458 return success();
459 }
460};
461
462/// Performs a reduction over 2 dimensions by decomposing it into two 1D
463/// reductions ordered based on layout to minimize cross-lane communication.
464class MultiRed2dOpPattern
465 : public OpConversionPattern<vector::MultiDimReductionOp> {
466 using Base::Base;
467 LogicalResult
468 matchAndRewrite(vector::MultiDimReductionOp reductionOp, OpAdaptor adaptor,
469 ConversionPatternRewriter &rewriter) const override {
470 auto sourceVecType = reductionOp.getSourceVectorType();
471 if (reductionOp.getReductionDims().size() != 2)
472 return rewriter.notifyMatchFailure(arg&: reductionOp, msg: "Expected 2D reduction");
473 auto resLayout = xegpu::getDistributeLayoutAttr(value: reductionOp.getResult());
474 // Retrieve and order dims for 1D decomposition (prefer intra-lane first).
475 auto dims = llvm::to_vector(Range: reductionOp.getReductionDims());
476 auto [intraLaneDim, crossLaneDim] = getReductionDimOrder(reductionDims: dims, layout: resLayout);
477 // Order does not matter
478 if (intraLaneDim == -1 || crossLaneDim == -1) {
479 intraLaneDim = dims[0];
480 crossLaneDim = dims[1];
481 }
482 auto loc = reductionOp.getLoc();
483 auto acc = reductionOp.getAcc();
484
485 // The decomposition below splits the 2D reduction into an intra-lane
486 // then a cross-lane 1D reduction. The natural result layout of the
487 // decomposed sequence (a doubly-sliced layout) differs from the
488 // original 2D reduction's result layout that the rest of the IR was
489 // written/propagated against. To keep the post-peephole IR
490 // self-consistent without depending on a follow-up layout
491 // propagation pass, we always insert a bridge xegpu.convert_layout
492 // from the natural post-decomposition layout to the original
493 // reduction's result layout. Trivial bridges fold away in
494 // canonicalization.
495 xegpu::DistributeLayoutAttr postDecompLayout;
496 if (resLayout) {
497 // Derive the source vector's layout.
498 xegpu::DistributeLayoutAttr srcLayoutForCvt;
499 if (auto resSlice = dyn_cast_if_present<xegpu::SliceAttr>(Val&: resLayout))
500 srcLayoutForCvt = resSlice.getParent();
501 if (!srcLayoutForCvt)
502 srcLayoutForCvt =
503 xegpu::getDistributeLayoutAttr(value: reductionOp.getSource());
504 if (srcLayoutForCvt) {
505 // The natural layout of the post-decomposition reduction result
506 // is a nested SliceAttr: REDUCE_1 (reduces `intraLaneDim` from
507 // the source) yields `slice<src, [intraLaneDim]>`; REDUCE_2
508 // then reduces `adjCrossLaneDim` from that intermediate, giving
509 // `slice<slice<src, [intraLaneDim]>, [adjCrossLaneDim]>`.
510 MLIRContext *ctx = reductionOp.getContext();
511 int64_t adjCrossLaneDim =
512 crossLaneDim > intraLaneDim ? crossLaneDim - 1 : crossLaneDim;
513 auto intermediateLayout = xegpu::SliceAttr::get(
514 context: ctx, parent: srcLayoutForCvt, dims: DenseI64ArrayAttr::get(context: ctx, content: {intraLaneDim}));
515 postDecompLayout = xegpu::SliceAttr::get(
516 context: ctx, parent: intermediateLayout,
517 dims: DenseI64ArrayAttr::get(context: ctx, content: {adjCrossLaneDim}));
518 }
519 }
520
521 SmallVector<int64_t> accShape(sourceVecType.getShape());
522 accShape.erase(CI: accShape.begin() + intraLaneDim);
523 Type eTy = sourceVecType.getElementType();
524 Value constNeutralVal = xegpu::createReductionNeutralValue(
525 builder&: rewriter, loc, type: VectorType::get(shape: accShape, elementType: eTy), kind: reductionOp.getKind(),
526 fastmath: reductionOp.getFastmathAttr());
527
528 Value intraLaneReduced = vector::MultiDimReductionOp::create(
529 builder&: rewriter, location: loc, kind: reductionOp.getKind(), source: reductionOp.getSource(),
530 acc: constNeutralVal, reduction_dims: ArrayRef<int64_t>(intraLaneDim),
531 fastmath: reductionOp.getFastmath());
532
533 // Adjust crossLaneDim after the first reduction.
534 if (crossLaneDim > intraLaneDim)
535 crossLaneDim -= 1;
536 Value crossLaneReduced = vector::MultiDimReductionOp::create(
537 builder&: rewriter, location: loc, kind: reductionOp.getKind(), source: intraLaneReduced, acc,
538 reduction_dims: ArrayRef<int64_t>(crossLaneDim), fastmath: reductionOp.getFastmath());
539 assert(crossLaneReduced.getType() == reductionOp.getResult().getType() &&
540 "Type mismatch");
541
542 Value replacement = crossLaneReduced;
543 if (resLayout && postDecompLayout) {
544 // Bridge from the natural post-decomposition layout to the
545 // original reduction's result layout. This preserves the contract
546 // any consumer (convert_layout, anchor op, or otherwise) was
547 // written against, so the rewrite is correct independent of
548 // whether layout propagation runs afterwards.
549 auto bridgeOp = xegpu::ConvertLayoutOp::create(
550 builder&: rewriter, location: loc, result: crossLaneReduced.getType(), source: crossLaneReduced,
551 input_layout: postDecompLayout, target_layout: resLayout);
552 replacement = bridgeOp.getResult();
553 }
554
555 rewriter.replaceOp(op: reductionOp, newValues: replacement);
556 return success();
557 }
558
559private:
560 std::pair<int64_t, int64_t>
561 getReductionDimOrder(ArrayRef<int64_t> reductionDims,
562 xegpu::DistributeLayoutAttr layout) const {
563 assert(layout.isForSubgroup() && "Must know the lane layout");
564 assert(reductionDims.size() == 2 && "Expected 2D reduction");
565 int64_t intra = -1, cross = -1;
566 xegpu::LayoutAttr layoutAttr = dyn_cast<xegpu::LayoutAttr>(Val&: layout);
567 if (auto layoutSliceAttr = dyn_cast<xegpu::SliceAttr>(Val&: layout))
568 layoutAttr =
569 dyn_cast<xegpu::LayoutAttr>(Val: layoutSliceAttr.flatten().getParent());
570 assert(layoutAttr);
571 SmallVector<int64_t> laneLayout = layoutAttr.getEffectiveLaneLayoutAsInt();
572
573 assert(laneLayout.size() && "Expected a non-empty layout");
574 // try to pick a dim that does not communicate
575 for (auto dim : reductionDims) {
576 if (laneLayout[dim] == 1)
577 intra = dim;
578 else
579 cross = dim;
580 }
581 return {intra, cross};
582 }
583};
584
585} // namespace
586
587void xegpu::populateXeGPUPeepHoleOptimizerPatterns(
588 RewritePatternSet &patterns) {
589 patterns.add<XeGPUCreateNdDescOpPattern, XeGPULoadNdDescOpPattern,
590 VectorExtractOpPattern, MultiRed2dOpPattern>(
591 arg: patterns.getContext());
592}
593
594namespace {
595
596struct XeGPUPeepHoleOptimizerPass final
597 : public xegpu::impl::XeGPUPeepHoleOptimizerBase<
598 XeGPUPeepHoleOptimizerPass> {
599 void runOnOperation() override {
600 MLIRContext &context = getContext();
601 TypeConverter converter;
602 RewritePatternSet patterns(&context);
603 ConversionTarget target(context);
604
605 // This pass is only meant for PVC, BMG or CRI targets. If unsupported
606 // target is found, exit early.
607 bool isTargetSupported = false;
608 getOperation()->walk(callback: [&](gpu::GPUFuncOp funcOp) {
609 auto chipStr = xegpu::getChipStr(op: funcOp);
610 if (chipStr && (chipStr.value() == "pvc" || chipStr.value() == "bmg" ||
611 chipStr.value() == "cri"))
612 isTargetSupported = true;
613 });
614
615 if (!isTargetSupported) {
616 DBGS() << "XeGPUPeepHoleOptimizerPass only supports PVC, BMG targets."
617 << "\n";
618 return;
619 }
620
621 // Run array length optimization patterns first so that subsequent transpose
622 // peephole patterns operate on the array-length-optimized tensor descs.
623 {
624 RewritePatternSet arrayLenPatterns(&context);
625 xegpu::populateXeGPUArrayLengthOptimizationPatterns(patterns&: arrayLenPatterns);
626 if (failed(Result: applyPatternsGreedily(op: getOperation(),
627 patterns: std::move(arrayLenPatterns)))) {
628 DBGS() << "Array length optimization patterns failed.\n";
629 return signalPassFailure();
630 }
631 }
632
633 // CreateNdDescOp and LoadNdOp with optimizable tensor desc types must be
634 // converted.
635 target.addDynamicallyLegalOp<xegpu::CreateNdDescOp>(
636 callback: [&](xegpu::CreateNdDescOp createNdOp) {
637 return !canBeOptimizedForTranspose(tdescType: createNdOp.getType());
638 });
639 target.addDynamicallyLegalOp<xegpu::LoadNdOp>(
640 callback: [&](xegpu::LoadNdOp loadNdOp) {
641 return !canBeOptimizedForTranspose(tdescType: loadNdOp.getTensorDescType());
642 });
643 // Vector ExtractOps can have optimizable layouts if they extract from
644 // LoadNdOps with array length greater than 1. These ExtractOps must be
645 // converted.
646 target.addDynamicallyLegalOp<vector::ExtractOp>(
647 callback: [&](vector::ExtractOp extractOp) {
648 auto layout = xegpu::getTemporaryLayout(
649 operandOrResult: dyn_cast<OpResult>(Val: extractOp.getResult()));
650 if (!layout)
651 return true;
652 auto laneLayout = layout.getEffectiveLaneLayoutAsInt();
653 auto laneData = layout.getEffectiveLaneDataAsInt();
654 return !canBeOptimizedForTranspose(laneLayout, laneData);
655 });
656
657 target.addDynamicallyLegalOp<vector::MultiDimReductionOp>(
658 callback: [=](Operation *op) -> bool {
659 auto layout = xegpu::getDistributeLayoutAttr(value: op->getResult(idx: 0));
660 if (!layout || !layout.isForSubgroup())
661 return true;
662 if (auto reductionOp = dyn_cast<vector::MultiDimReductionOp>(Val: op))
663 return reductionOp.getReductionDims().size() != 2;
664 return true;
665 });
666
667 converter.addConversion(callback: [](Type type) { return type; });
668
669 target.addLegalDialect<arith::ArithDialect, memref::MemRefDialect,
670 vector::VectorDialect>();
671 // xegpu.convert_layout is left untouched by this pass; mark it legal
672 // so in-place updates don't trigger re-legalization failures.
673 target.addLegalOp<xegpu::ConvertLayoutOp>();
674 scf::populateSCFStructuralTypeConversionsAndLegality(typeConverter: converter, patterns,
675 target);
676 xegpu::populateXeGPUPeepHoleOptimizerPatterns(patterns);
677 if (failed(Result: applyPartialConversion(op: getOperation(), target,
678 patterns: std::move(patterns)))) {
679 DBGS() << "Optimize block loads pass failed.\n";
680 return signalPassFailure();
681 }
682
683 // Apply folding for cleaning up IR.
684 MLIRContext *ctx = &getContext();
685 RewritePatternSet emptyPatterns(ctx);
686 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(emptyPatterns));
687
688 xegpu::removeTemporaryLayoutAttrs(op: getOperation());
689 }
690};
691
692} // namespace
693