1//===- XeGPUUnroll.cpp - patterns to do unrolling ---------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains patterns for unrolling XeGPU operations. It follows a
10// similar concept and design as vector unroll patterns, serving as a complement
11// to them.
12//
13//===----------------------------------------------------------------------===//
14
15#include "mlir/Dialect/Arith/IR/Arith.h"
16#include "mlir/Dialect/MemRef/IR/MemRef.h"
17#include "mlir/Dialect/Utils/IndexingUtils.h"
18#include "mlir/Dialect/Vector/IR/VectorOps.h"
19#include "mlir/Dialect/XeGPU/IR/XeGPU.h"
20#include "mlir/Dialect/XeGPU/Transforms/Transforms.h"
21#include "mlir/Dialect/XeGPU/Transforms/XeGPULayoutImpl.h"
22#include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/Support/DebugLog.h"
25
26namespace mlir {
27namespace xegpu {
28#define GEN_PASS_DEF_XEGPUUNROLL
29#include "mlir/Dialect/XeGPU/Transforms/Passes.h.inc"
30} // namespace xegpu
31} // namespace mlir
32
33#define DEBUG_TYPE "xegpu-unroll"
34
35using namespace mlir;
36
37namespace {
38
39// Forward declaration for use inside UnrollPattern below.
40SmallVector<Value>
41unrollByTile(SmallVector<OpFoldResult> mixedOffsets,
42 xegpu::TensorDescType tdescTy, ArrayRef<int64_t> targetShape,
43 const std::function<Value(SmallVector<OpFoldResult>)> &createOp,
44 Location loc, PatternRewriter &rewriter);
45
46template <typename SourceOp>
47struct UnrollPattern : public OpRewritePattern<SourceOp> {
48 UnrollPattern(MLIRContext *context, const xegpu::UnrollOptions &options,
49 PatternBenefit benefit = 1)
50 : OpRewritePattern<SourceOp>(context, benefit), options(options) {}
51
52protected:
53 /// Return the target shape for the given `op`. Return std::nullopt if the
54 /// op shouldn't be or cannot be unrolled.
55 std::optional<SmallVector<int64_t>> getTargetShape(Operation *op) const {
56 LDBG() << "Get unroll shape for: " << *op;
57
58 if (options.filterConstraint && failed(Result: options.filterConstraint(op))) {
59 LDBG() << "--no filter constraint -> BAIL";
60 return std::nullopt;
61 }
62
63 assert(options.nativeShape &&
64 "expects the native shape for native shape call back function.");
65 auto nativeShape = options.nativeShape(op);
66 return nativeShape;
67 }
68
69 SmallVector<Type> getUnrolledTypes(ShapedType type,
70 ArrayRef<int64_t> tileShape) const {
71 return options.getUnrolledTypes(type, tileShape);
72 }
73
74 /// Emulate the the unpack behavior using insert_strided_slice for VectorType
75 /// values and unrealized_conversion_cast for TensorDescType values.
76 Value unpack(ValueRange srcs, Type destTy, ArrayRef<int64_t> blockSize,
77 Location loc, PatternRewriter &rewriter) const {
78 if (auto vecTy = dyn_cast<VectorType>(Val&: destTy)) {
79 auto shape = vecTy.getShape();
80 return xegpu::createVectorWithShapeFromValues(builder&: rewriter, loc, values: srcs, shape);
81 }
82
83 if (isa<xegpu::TensorDescType>(Val: destTy)) {
84 auto attr = NamedAttribute(rewriter.getStringAttr(bytes: unpackAttrName),
85 rewriter.getUnitAttr());
86 auto blkAttr = NamedAttribute(rewriter.getStringAttr(bytes: blockAttrName),
87 rewriter.getDenseI64ArrayAttr(values: blockSize));
88 auto castOp = UnrealizedConversionCastOp::create(
89 builder&: rewriter, location: loc, resultTypes: destTy, operands: srcs,
90 attributes: ArrayRef<NamedAttribute>({attr, blkAttr}));
91 return castOp.getResult(i: 0);
92 }
93
94 llvm_unreachable("Unexpected destTy.");
95 return Value();
96 }
97
98 /// Emulate the the pack behavior using extract_strided_slice for VectorType
99 /// values and unrealized_conversion_cast for TensorDescType values.
100 SmallVector<Value> pack(Value src, TypeRange destTypes,
101 ArrayRef<int64_t> blockSize, Location loc,
102 PatternRewriter &rewriter) const {
103 if (auto vecTy = dyn_cast<VectorType>(Val: src.getType())) {
104 return xegpu::extractVectorsWithShapeFromValue(builder&: rewriter, loc, value: src,
105 shape: blockSize);
106 }
107
108 if (isa<xegpu::TensorDescType>(Val: src.getType())) {
109 auto attr = NamedAttribute(rewriter.getStringAttr(bytes: packAttrName),
110 rewriter.getUnitAttr());
111 auto blkAttr = NamedAttribute(rewriter.getStringAttr(bytes: blockAttrName),
112 rewriter.getDenseI64ArrayAttr(values: blockSize));
113 auto castOp = UnrealizedConversionCastOp::create(
114 builder&: rewriter, location: loc, resultTypes: destTypes, operands: src,
115 attributes: ArrayRef<NamedAttribute>({attr, blkAttr}));
116 return castOp.getResults();
117 }
118
119 llvm_unreachable("Unexpected src type.");
120 return SmallVector<Value>();
121 }
122
123 /// Helper to pack operands for DPAS-like operations with early return if
124 /// no unrolling is needed.
125 SmallVector<Value> packOperandForDpas(Value operand,
126 ArrayRef<int64_t> blockSize,
127 Location loc,
128 PatternRewriter &rewriter) const {
129 auto vecType = cast<VectorType>(Val: operand.getType());
130 std::optional<SmallVector<int64_t>> grids =
131 computeShapeRatio(shape: vecType.getShape(), subShape: blockSize);
132 assert(grids && "Expecting grids to be computed.");
133 auto numNewOps = computeProduct(basis: *grids);
134 if (numNewOps == 1)
135 return SmallVector<Value>({operand});
136 VectorType newVecTy =
137 vecType.cloneWith(shape: blockSize, elementType: vecType.getElementType());
138 SmallVector<Type> convertedTypes(numNewOps, newVecTy);
139 return pack(src: operand, destTypes: convertedTypes, blockSize, loc, rewriter);
140 }
141
142private:
143 const char *const packAttrName = "__xegpu_blocking_pack__";
144 const char *const unpackAttrName = "__xegpu_blocking_unpack__";
145 const char *const blockAttrName = "__xegpu_blocking_tile_shape__";
146
147 xegpu::UnrollOptions options;
148};
149
150// Walks tile offsets within the tensor descriptor shape and emits one op per
151// tile by calling `createOp` with the per-tile offsets. Used by LoadNd,
152// StoreNd, CreateNdDesc, and PrefetchNd unrollers, which all need to adjust
153// their explicit offsets for each unrolled tile.
154SmallVector<Value>
155unrollByTile(SmallVector<OpFoldResult> mixedOffsets,
156 xegpu::TensorDescType tdescTy, ArrayRef<int64_t> targetShape,
157 const std::function<Value(SmallVector<OpFoldResult>)> &createOp,
158 Location loc, PatternRewriter &rewriter) {
159 int64_t rank = tdescTy.getRank();
160 ArrayRef<int64_t> shape = tdescTy.getShape();
161
162 auto addi = [&](OpFoldResult a, int64_t b) -> Value {
163 std::optional<int64_t> maybeInt = getConstantIntValue(ofr: a);
164 if (maybeInt) {
165 return arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: *maybeInt + b);
166 } else {
167 auto aV = llvm::cast<Value>(Val&: a);
168 auto bV = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: b);
169 return rewriter.createOrFold<arith::AddIOp>(location: loc, args&: aV, args&: bV);
170 }
171 };
172
173 SmallVector<OpFoldResult> oldOffsets = llvm::to_vector(
174 Range: llvm::drop_begin(RangeOrContainer&: mixedOffsets, N: mixedOffsets.size() - rank));
175 auto validIdxes =
176 llvm::seq<int64_t>(Begin: mixedOffsets.size() - rank, End: mixedOffsets.size());
177
178 SmallVector<Value> newOps;
179 for (SmallVector<int64_t> offsets :
180 StaticTileOffsetRange(shape, targetShape)) {
181
182 for (auto [idx, oldOff, offset] :
183 llvm::zip(t&: validIdxes, u&: oldOffsets, args&: offsets))
184 mixedOffsets[idx] = addi(oldOff, offset);
185
186 auto newOp = createOp(mixedOffsets);
187 newOps.push_back(Elt: newOp);
188 }
189 return newOps;
190}
191
192struct UnrollCreateNdOp : public UnrollPattern<xegpu::CreateNdDescOp> {
193 using UnrollPattern<xegpu::CreateNdDescOp>::UnrollPattern;
194 LogicalResult matchAndRewrite(xegpu::CreateNdDescOp op,
195 PatternRewriter &rewriter) const override {
196 Location loc = op.getLoc();
197 xegpu::TensorDescType tdescTy = op.getType();
198
199 std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op);
200 if (!targetShape)
201 return failure();
202
203 // Keep the high-D source; only the tile shape shrinks.
204 Value src = op.getSource();
205 auto makeCreateNd = [&](Type tdesc) -> Value {
206 auto ndTy = cast<xegpu::TensorDescType>(Val&: tdesc);
207 if (isa<MemRefType>(Val: src.getType()))
208 return xegpu::CreateNdDescOp::create(builder&: rewriter, location: loc, tdesc: ndTy,
209 source: cast<TypedValue<MemRefType>>(Val&: src));
210 return xegpu::CreateNdDescOp::create(
211 builder&: rewriter, location: loc, tdesc: ndTy, source: src, shape: op.getMixedSizes(), strides: op.getMixedStrides());
212 };
213
214 SmallVector<Type> newTdescTys = getUnrolledTypes(type: tdescTy, tileShape: *targetShape);
215 SmallVector<Value> newOps;
216 if (tdescTy.getRank() <= 2) {
217 // 2D: one tdesc, broadcast across tiles by pack/unpack.
218 newOps.push_back(Elt: makeCreateNd(newTdescTys[0]));
219 } else {
220 // >2D: one tdesc per tile, so the source count matches the pack count.
221 for (Type t : newTdescTys)
222 newOps.push_back(Elt: makeCreateNd(t));
223 }
224 Value castOp = unpack(srcs: newOps, destTy: tdescTy, blockSize: *targetShape, loc, rewriter);
225 rewriter.replaceOp(op, newValues: castOp);
226 return success();
227 }
228};
229
230struct UnrollPrefetchNdOp : public UnrollPattern<xegpu::PrefetchNdOp> {
231 using UnrollPattern<xegpu::PrefetchNdOp>::UnrollPattern;
232 LogicalResult matchAndRewrite(xegpu::PrefetchNdOp op,
233 PatternRewriter &rewriter) const override {
234 Location loc = op.getLoc();
235 xegpu::TensorDescType tdescTy = op.getTensorDescType();
236
237 std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op);
238 if (!targetShape)
239 return failure();
240
241 xegpu::DistributeLayoutAttr layout = op.getLayoutAttr();
242 if (layout)
243 layout = layout.dropInstData();
244
245 // Batch (leading) dims unroll to unit tiles; one tdesc serves all.
246 SmallVector<Type> convertedTdescTypes =
247 getUnrolledTypes(type: tdescTy, tileShape: *targetShape);
248 SmallVector<Value> convertedTdesc = pack(
249 src: op.getTensorDesc(), destTypes: convertedTdescTypes, blockSize: *targetShape, loc, rewriter);
250
251 auto createPrefetch = [&](SmallVector<OpFoldResult> offsets) -> Value {
252 xegpu::PrefetchNdOp::create(builder&: rewriter, location: loc, TensorDesc: convertedTdesc[0], offsets,
253 l1_hint: op.getL1HintAttr(), l2_hint: op.getL2HintAttr(),
254 l3_hint: op.getL3HintAttr(), layout);
255 return nullptr;
256 };
257 unrollByTile(mixedOffsets: op.getMixedOffsets(), tdescTy, targetShape: *targetShape, createOp: createPrefetch,
258 loc, rewriter);
259
260 rewriter.eraseOp(op);
261 return success();
262 }
263};
264
265struct UnrollLoadNdOp : public UnrollPattern<xegpu::LoadNdOp> {
266 using UnrollPattern<xegpu::LoadNdOp>::UnrollPattern;
267 LogicalResult matchAndRewrite(xegpu::LoadNdOp op,
268 PatternRewriter &rewriter) const override {
269
270 Location loc = op.getLoc();
271 VectorType valueTy = op.getType();
272 xegpu::TensorDescType tdescTy = op.getTensorDescType();
273
274 std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op);
275 if (!targetShape)
276 return failure();
277
278 xegpu::DistributeLayoutAttr layout = op.getLayoutAttr();
279 if (layout)
280 layout = layout.dropInstData();
281
282 Type elemTy = tdescTy.getElementType();
283 VectorType newValueTy = valueTy.cloneWith(shape: *targetShape, elementType: elemTy);
284
285 SmallVector<Value> newOps;
286
287 // Batch (leading) dims unroll to unit tiles; one tdesc serves all.
288 SmallVector<Type> convertedTdescTypes =
289 getUnrolledTypes(type: tdescTy, tileShape: *targetShape);
290 SmallVector<Value> convertedTdescs = pack(
291 src: op.getTensorDesc(), destTypes: convertedTdescTypes, blockSize: *targetShape, loc, rewriter);
292
293 auto createLoad = [&](SmallVector<OpFoldResult> offsets) -> Value {
294 return xegpu::LoadNdOp::create(
295 builder&: rewriter, location: loc, value: newValueTy, TensorDesc: convertedTdescs[0], offsets,
296 packed: op.getPackedAttr(), transpose: op.getTransposeAttr(), l1_hint: op.getL1HintAttr(),
297 l2_hint: op.getL2HintAttr(), l3_hint: op.getL3HintAttr(), layout);
298 };
299 newOps = unrollByTile(mixedOffsets: op.getMixedOffsets(), tdescTy, targetShape: *targetShape,
300 createOp: createLoad, loc, rewriter);
301
302 Value castOp = unpack(srcs: newOps, destTy: op.getType(), blockSize: *targetShape, loc, rewriter);
303 rewriter.replaceOp(op, newValues: castOp);
304 return success();
305 }
306};
307
308struct UnrollStoreNdOp : public UnrollPattern<xegpu::StoreNdOp> {
309 using UnrollPattern<xegpu::StoreNdOp>::UnrollPattern;
310 LogicalResult matchAndRewrite(xegpu::StoreNdOp op,
311 PatternRewriter &rewriter) const override {
312 Location loc = op.getLoc();
313 VectorType valueTy = op.getValueType();
314 xegpu::TensorDescType tdescTy = op.getTensorDescType();
315
316 std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op);
317 if (!targetShape)
318 return failure();
319
320 xegpu::DistributeLayoutAttr layout = op.getLayoutAttr();
321 if (layout)
322 layout = layout.dropInstData();
323
324 SmallVector<Type> convertedValTypes =
325 getUnrolledTypes(type: valueTy, tileShape: *targetShape);
326
327 SmallVector<Value> convertedValues =
328 pack(src: op.getValue(), destTypes: convertedValTypes, blockSize: *targetShape, loc, rewriter);
329
330 size_t valueIndex = 0;
331
332 // Batch (leading) dims unroll to unit tiles like any other dim. valueIndex
333 // advances in unrollByTile's tile order, staying in sync with the packed
334 // values.
335 SmallVector<Type> convertedTdescTypes =
336 getUnrolledTypes(type: tdescTy, tileShape: *targetShape);
337 SmallVector<Value> convertedTdescs = pack(
338 src: op.getTensorDesc(), destTypes: convertedTdescTypes, blockSize: *targetShape, loc, rewriter);
339
340 auto createStore = [&](SmallVector<OpFoldResult> offsets) {
341 xegpu::StoreNdOp::create(builder&: rewriter, location: loc, value: convertedValues[valueIndex++],
342 TensorDesc: convertedTdescs[0], offsets, l1_hint: op.getL1HintAttr(),
343 l2_hint: op.getL2HintAttr(), l3_hint: op.getL3HintAttr(), layout);
344 return (Value) nullptr;
345 };
346 unrollByTile(mixedOffsets: op.getMixedOffsets(), tdescTy, targetShape: *targetShape, createOp: createStore, loc,
347 rewriter);
348
349 rewriter.eraseOp(op);
350 return success();
351 }
352};
353
354struct UnrollDpasOp : public UnrollPattern<xegpu::DpasOp> {
355 using UnrollPattern<xegpu::DpasOp>::UnrollPattern;
356 LogicalResult matchAndRewrite(xegpu::DpasOp op,
357 PatternRewriter &rewriter) const override {
358 Location loc = op.getLoc();
359
360 std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op);
361 if (!targetShape || targetShape->size() < 3)
362 return failure();
363
364 // targetShape is [batch..., M, K, N]
365 int64_t tsRank = targetShape->size();
366 auto M = (*targetShape)[tsRank - 3];
367 auto K = (*targetShape)[tsRank - 2];
368 auto N = (*targetShape)[tsRank - 1];
369 ArrayRef<int64_t> batchDims(targetShape->data(), tsRank - 3);
370
371 // Build block sizes including batch dimensions.
372 SmallVector<int64_t> aBlockSize(batchDims);
373 aBlockSize.push_back(Elt: M);
374 aBlockSize.push_back(Elt: K);
375 SmallVector<int64_t> bBlockSize(batchDims);
376 bBlockSize.push_back(Elt: K);
377 bBlockSize.push_back(Elt: N);
378 SmallVector<int64_t> cBlockSize(batchDims);
379 cBlockSize.push_back(Elt: M);
380 cBlockSize.push_back(Elt: N);
381
382 auto a = op.getLhs();
383 auto b = op.getRhs();
384 auto c = op.getAcc();
385
386 SmallVector<Value> aVals = packOperandForDpas(operand: a, blockSize: aBlockSize, loc, rewriter);
387 SmallVector<Value> bVals = packOperandForDpas(operand: b, blockSize: bBlockSize, loc, rewriter);
388 SmallVector<Value> cVals;
389 if (c)
390 cVals = packOperandForDpas(operand: c, blockSize: cBlockSize, loc, rewriter);
391
392 auto ranges = c ? SmallVector<ValueRange>({aVals, bVals, cVals})
393 : SmallVector<ValueRange>({aVals, bVals});
394 if (llvm::any_of(Range&: ranges, P: [](auto &v) { return v.size() == 0; }) ||
395 llvm::all_of(Range&: ranges, P: [](auto &v) { return v.size() == 1; }))
396 return failure();
397
398 VectorType resultTy = op.getResult().getType();
399 auto vecTy = VectorType::get(shape: cBlockSize, elementType: resultTy.getElementType());
400
401 auto aShape = a.getType().getShape();
402 auto bShape = b.getType().getShape();
403
404 // Compute iteration counts. Batch dims only iterate over M and N (not
405 // K-reduction), so compute batch iterations from the C block size.
406 int64_t batchRank = batchDims.size();
407 int64_t mIters = aShape[batchRank] / M;
408 int64_t kIters = aShape[batchRank + 1] / K;
409 int64_t nIters = bShape[batchRank + 1] / N;
410
411 // Compute batch iterations (product of batch dim ratios).
412 int64_t batchIters = 1;
413 for (int64_t d = 0; d < batchRank; ++d)
414 batchIters *= aShape[d] / batchDims[d];
415
416 SmallVector<Value> newOps;
417 for (int64_t batch = 0; batch < batchIters; ++batch) {
418 for (int64_t i = 0; i < mIters; ++i) {
419 for (int64_t j = 0; j < nIters; ++j) {
420 Value tmpC;
421 if (c)
422 tmpC = cVals[batch * (mIters * nIters) + i * nIters + j];
423
424 for (int64_t k = 0; k < kIters; ++k) {
425 Value aVec = aVals[batch * (mIters * kIters) + i * kIters + k];
426 Value bVec = bVals[batch * (kIters * nIters) + k * nIters + j];
427 SmallVector<Value> operands({aVec, bVec});
428 if (tmpC)
429 operands.push_back(Elt: tmpC);
430
431 auto newDpasOp = xegpu::DpasOp::create(
432 builder&: rewriter, location: loc, resultTypes: TypeRange{vecTy}, operands, properties: op.getProperties(),
433 discardableAttributes: xegpu::dropInstDataOnAttrs(
434 attrs: op->getDiscardableAttrDictionary().getValue()));
435 xegpu::dropInstDataOnInherentAttrs(op: newDpasOp);
436 tmpC = newDpasOp.getResult();
437 }
438 newOps.push_back(Elt: tmpC);
439 }
440 }
441 }
442 Value castOp = unpack(srcs: newOps, destTy: resultTy, blockSize: cBlockSize, loc, rewriter);
443 rewriter.replaceOp(op, newValues: castOp);
444 return success();
445 }
446};
447
448struct UnrollDpasMxOp : public UnrollPattern<xegpu::DpasMxOp> {
449 using UnrollPattern<xegpu::DpasMxOp>::UnrollPattern;
450 LogicalResult matchAndRewrite(xegpu::DpasMxOp op,
451 PatternRewriter &rewriter) const override {
452 Location loc = op.getLoc();
453
454 std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op);
455 if (!targetShape || targetShape->size() < 4)
456 return failure();
457
458 // targetShape is [batch..., M, K, N, S]
459 int64_t tsRank = targetShape->size();
460 auto M = (*targetShape)[tsRank - 4];
461 auto K = (*targetShape)[tsRank - 3];
462 auto N = (*targetShape)[tsRank - 2];
463 auto S = (*targetShape)[tsRank - 1];
464 ArrayRef<int64_t> batchDims(targetShape->data(), tsRank - 4);
465
466 SmallVector<int64_t> aBlockSize(batchDims);
467 aBlockSize.push_back(Elt: M);
468 aBlockSize.push_back(Elt: K);
469 SmallVector<int64_t> bBlockSize(batchDims);
470 bBlockSize.push_back(Elt: K);
471 bBlockSize.push_back(Elt: N);
472 SmallVector<int64_t> cBlockSize(batchDims);
473 cBlockSize.push_back(Elt: M);
474 cBlockSize.push_back(Elt: N);
475 SmallVector<int64_t> aScaleBlockSize(batchDims);
476 aScaleBlockSize.push_back(Elt: M);
477 aScaleBlockSize.push_back(Elt: S);
478 SmallVector<int64_t> bScaleBlockSize(batchDims);
479 bScaleBlockSize.push_back(Elt: S);
480 bScaleBlockSize.push_back(Elt: N);
481
482 auto a = op.getA();
483 auto b = op.getB();
484 auto c = op.getAcc();
485 auto ascale = dyn_cast<TypedValue<VectorType>>(Val: op.getScaleA());
486 auto bscale = dyn_cast<TypedValue<VectorType>>(Val: op.getScaleB());
487
488 SmallVector<Value> aVals = packOperandForDpas(operand: a, blockSize: aBlockSize, loc, rewriter);
489 SmallVector<Value> bVals = packOperandForDpas(operand: b, blockSize: bBlockSize, loc, rewriter);
490 SmallVector<Value> cVals;
491 if (c)
492 cVals = packOperandForDpas(operand: c, blockSize: cBlockSize, loc, rewriter);
493 SmallVector<Value> aScaleVals;
494 if (ascale)
495 aScaleVals = packOperandForDpas(operand: ascale, blockSize: aScaleBlockSize, loc, rewriter);
496 SmallVector<Value> bScaleVals;
497 if (bscale)
498 bScaleVals = packOperandForDpas(operand: bscale, blockSize: bScaleBlockSize, loc, rewriter);
499
500 VectorType resultTy = op.getResult().getType();
501 auto vecTy = VectorType::get(shape: cBlockSize, elementType: resultTy.getElementType());
502
503 auto aShape = a.getType().getShape();
504 auto bShape = b.getType().getShape();
505 int64_t batchRank = batchDims.size();
506 int64_t mIters = aShape[batchRank] / M;
507 int64_t kIters = aShape[batchRank + 1] / K;
508 int64_t nIters = bShape[batchRank + 1] / N;
509
510 int64_t batchIters = 1;
511 for (int64_t d = 0; d < batchRank; ++d)
512 batchIters *= aShape[d] / batchDims[d];
513
514 SmallVector<Value> newOps;
515 xegpu::DpasMxOp newDpasMxOp;
516 for (int64_t batch = 0; batch < batchIters; ++batch) {
517 for (int64_t i = 0; i < mIters; ++i) {
518 for (int64_t j = 0; j < nIters; ++j) {
519 Value tmpC;
520 if (c)
521 tmpC = cVals[batch * (mIters * nIters) + i * nIters + j];
522
523 for (int64_t k = 0; k < kIters; ++k) {
524 Value aVec = aVals[batch * (mIters * kIters) + i * kIters + k];
525 Value bVec = bVals[batch * (kIters * nIters) + k * nIters + j];
526 SmallVector<Value> operands({aVec, bVec});
527 if (tmpC)
528 operands.push_back(Elt: tmpC);
529 if (ascale)
530 operands.push_back(
531 Elt: aScaleVals[batch * (mIters * kIters) + i * kIters + k]);
532 if (bscale)
533 operands.push_back(
534 Elt: bScaleVals[batch * (kIters * nIters) + k * nIters + j]);
535
536 newDpasMxOp = xegpu::DpasMxOp::create(
537 builder&: rewriter, location: loc, resultTypes: TypeRange{vecTy}, operands, properties: op.getProperties(),
538 discardableAttributes: xegpu::dropInstDataOnAttrs(
539 attrs: op->getDiscardableAttrDictionary().getValue()));
540 xegpu::dropInstDataOnInherentAttrs(op: newDpasMxOp);
541 tmpC = newDpasMxOp.getResult();
542 }
543 newOps.push_back(Elt: newDpasMxOp);
544 }
545 }
546 }
547 Value castOp = unpack(srcs: newOps, destTy: resultTy, blockSize: cBlockSize, loc, rewriter);
548 rewriter.replaceOp(op, newValues: castOp);
549 return success();
550 }
551};
552
553/// This pattern handles the unrolling of LoadGatherOp with offsets (gathered
554/// load).
555/// It unrolls the offsets and mask operands accordingly, and creates multiple
556/// LoadGatherOp with the unrolled operands.
557struct UnrollLoadGatherOp : public UnrollPattern<xegpu::LoadGatherOp> {
558 using UnrollPattern<xegpu::LoadGatherOp>::UnrollPattern;
559 LogicalResult matchAndRewrite(xegpu::LoadGatherOp op,
560 PatternRewriter &rewriter) const override {
561 Location loc = op.getLoc();
562 VectorType valueTy = llvm::dyn_cast<VectorType>(Val: op.getType());
563 Value offsets = op.getOffsets();
564 Value mask = op.getMask();
565
566 std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op);
567 if (!targetShape)
568 return failure();
569
570 SmallVector<int64_t> targetMaskShape(*targetShape);
571
572 // Unroll mask and offsets with correct shape
573 VectorType maskTy = llvm::dyn_cast<VectorType>(Val: mask.getType());
574 VectorType offsetsTy = llvm::dyn_cast<VectorType>(Val: offsets.getType());
575 Type elemTy = valueTy.getElementType();
576 VectorType newValueTy = VectorType::get(shape: *targetShape, elementType: elemTy);
577
578 SmallVector<Type> convertedMaskTypes =
579 getUnrolledTypes(type: maskTy, tileShape: targetMaskShape);
580 SmallVector<Value> convertedMasks =
581 pack(src: mask, destTypes: convertedMaskTypes, blockSize: targetMaskShape, loc, rewriter);
582
583 SmallVector<Type> convertedOffsetTypes =
584 getUnrolledTypes(type: offsetsTy, tileShape: *targetShape);
585 SmallVector<Value> convertedOffsets =
586 pack(src: offsets, destTypes: convertedOffsetTypes, blockSize: *targetShape, loc, rewriter);
587
588 auto layout = op.getLayoutAttr();
589 if (layout)
590 layout = layout.dropInstData();
591
592 SmallVector<Value> newOps;
593 for (auto [o, m] : llvm::zip(t&: convertedOffsets, u&: convertedMasks)) {
594 auto newOp = xegpu::LoadGatherOp::create(
595 builder&: rewriter, location: loc, value: newValueTy, source: op.getSource(), offsets: o, mask: m, l1_hint: op.getL1HintAttr(),
596 l2_hint: op.getL2HintAttr(), l3_hint: op.getL3HintAttr(), layout,
597 /*contiguity=*/nullptr);
598 newOps.push_back(Elt: newOp);
599 }
600
601 Value castOp = unpack(srcs: newOps, destTy: op.getType(), blockSize: *targetShape, loc, rewriter);
602 rewriter.replaceOp(op, newValues: castOp);
603 return success();
604 }
605};
606
607/// This pattern handles the unrolling of StoreScatterOp with offsets (scattered
608/// store).
609/// It unrolls the offsets and mask operands accordingly, and creates multiple
610/// StoreScatterOp with the unrolled operands.
611struct UnrollStoreScatterOp : public UnrollPattern<xegpu::StoreScatterOp> {
612 using UnrollPattern<xegpu::StoreScatterOp>::UnrollPattern;
613 LogicalResult matchAndRewrite(xegpu::StoreScatterOp op,
614 PatternRewriter &rewriter) const override {
615 Location loc = op.getLoc();
616 VectorType valueTy = llvm::dyn_cast<VectorType>(Val: op.getValue().getType());
617 Value offsets = op.getOffsets();
618 Value mask = op.getMask();
619
620 std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op);
621 if (!targetShape)
622 return failure();
623
624 SmallVector<int64_t> targetMaskShape(*targetShape);
625 VectorType maskTy = llvm::dyn_cast<VectorType>(Val: mask.getType());
626 VectorType offsetsTy = llvm::dyn_cast<VectorType>(Val: offsets.getType());
627
628 SmallVector<Type> convertedMaskTypes =
629 getUnrolledTypes(type: maskTy, tileShape: targetMaskShape);
630 SmallVector<Value> convertedMasks =
631 pack(src: mask, destTypes: convertedMaskTypes, blockSize: targetMaskShape, loc, rewriter);
632
633 SmallVector<Type> convertedOffsetTypes =
634 getUnrolledTypes(type: offsetsTy, tileShape: *targetShape);
635 SmallVector<Value> convertedOffsets =
636 pack(src: offsets, destTypes: convertedOffsetTypes, blockSize: *targetShape, loc, rewriter);
637
638 SmallVector<Type> convertedValTypes =
639 getUnrolledTypes(type: valueTy, tileShape: *targetShape);
640 SmallVector<Value> convertedValues =
641 pack(src: op.getValue(), destTypes: convertedValTypes, blockSize: *targetShape, loc, rewriter);
642
643 auto layout = op.getLayoutAttr();
644 if (layout)
645 layout = layout.dropInstData();
646
647 for (auto [v, o, m] :
648 llvm::zip(t&: convertedValues, u&: convertedOffsets, args&: convertedMasks)) {
649 xegpu::StoreScatterOp::create(builder&: rewriter, location: loc, value: v, dest: op.getDest(), offsets: o, mask: m,
650 l1_hint: op.getL1HintAttr(), l2_hint: op.getL2HintAttr(),
651 l3_hint: op.getL3HintAttr(), layout,
652 /*contiguity=*/nullptr);
653 }
654
655 rewriter.eraseOp(op);
656 return success();
657 }
658};
659
660struct UnrollLoadMatrixOp : public UnrollPattern<xegpu::LoadMatrixOp> {
661 using UnrollPattern<xegpu::LoadMatrixOp>::UnrollPattern;
662 LogicalResult matchAndRewrite(xegpu::LoadMatrixOp op,
663 PatternRewriter &rewriter) const override {
664 Location loc = op.getLoc();
665 VectorType valueTy = llvm::dyn_cast<VectorType>(Val: op.getType());
666 assert(valueTy && "the value type must be vector type!");
667
668 std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op);
669 if (!targetShape || targetShape->size() != (size_t)valueTy.getRank())
670 return failure();
671
672 Type elemTy = valueTy.getElementType();
673 ArrayRef<int64_t> shape = valueTy.getShape();
674 xegpu::DistributeLayoutAttr layout = op.getLayoutAttr();
675
676 VectorType newValueTy = valueTy.cloneWith(shape: *targetShape, elementType: elemTy);
677
678 SmallVector<OpFoldResult> mixedOffsets = op.getMixedOffsets();
679 SmallVector<SmallVector<OpFoldResult>> offsetsList;
680 for (SmallVector<int64_t> offsets :
681 StaticTileOffsetRange(shape, *targetShape)) {
682 auto adds = xegpu::addElementwise(
683 builder&: rewriter, loc, lhs: mixedOffsets,
684 rhs: getAsIndexOpFoldResult(ctx: op.getContext(), values: offsets));
685 offsetsList.push_back(Elt: adds);
686 }
687
688 SmallVector<Value> newOps;
689 if (layout)
690 layout = layout.dropInstData();
691 for (SmallVector<OpFoldResult> offsets : offsetsList) {
692 auto newOp = xegpu::LoadMatrixOp::create(
693 builder&: rewriter, location: op.getLoc(), res: newValueTy, mem_desc: op.getMemDesc(), offsets, layout);
694 newOps.push_back(Elt: newOp);
695 }
696 Value castOp = unpack(srcs: newOps, destTy: op.getType(), blockSize: *targetShape, loc, rewriter);
697 rewriter.replaceOp(op, newValues: castOp);
698 return success();
699 }
700};
701
702struct UnrollStoreMatrixOp : public UnrollPattern<xegpu::StoreMatrixOp> {
703 using UnrollPattern<xegpu::StoreMatrixOp>::UnrollPattern;
704 LogicalResult matchAndRewrite(xegpu::StoreMatrixOp op,
705 PatternRewriter &rewriter) const override {
706 std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op);
707 if (!targetShape)
708 return failure();
709
710 Location loc = op.getLoc();
711 VectorType valueTy = llvm::dyn_cast<VectorType>(Val: op.getData().getType());
712 assert(valueTy && "the value type must be vector type!");
713 ArrayRef<int64_t> shape = valueTy.getShape();
714 xegpu::DistributeLayoutAttr layout = op.getLayoutAttr();
715 if (layout)
716 layout = layout.dropInstData();
717
718 SmallVector<Type> convertedValTypes =
719 getUnrolledTypes(type: valueTy, tileShape: *targetShape);
720 SmallVector<Value> convertedValues =
721 pack(src: op.getData(), destTypes: convertedValTypes, blockSize: *targetShape, loc, rewriter);
722
723 SmallVector<OpFoldResult> mixedOffsets = op.getMixedOffsets();
724 SmallVector<SmallVector<OpFoldResult>> offsetsList;
725 for (SmallVector<int64_t> offsets :
726 StaticTileOffsetRange(shape, *targetShape)) {
727 auto adds = xegpu::addElementwise(
728 builder&: rewriter, loc, lhs: mixedOffsets,
729 rhs: getAsIndexOpFoldResult(ctx: op.getContext(), values: offsets));
730 offsetsList.push_back(Elt: adds);
731 }
732
733 for (auto [v, offsets] : llvm::zip_equal(t&: convertedValues, u&: offsetsList))
734 xegpu::StoreMatrixOp::create(builder&: rewriter, location: loc, data: v, mem_desc: op.getMemDesc(), offsets,
735 layout);
736
737 rewriter.eraseOp(op);
738 return success();
739 }
740};
741
742/// UnrollConvertLayoutOp pattern for unrolling xegpu::ConvertLayoutOp
743/// operations. It first check whether the convert layout op has valid layouts
744/// after inst_data stripped. If it does, it will unroll the vector into
745/// multiple smaller vectors according to the target shape, and create multiple
746/// ConvertLayoutOp with the unrolled vectors and the stripped layouts.
747///
748/// When the input and target layouts have different inst_data, the source is
749/// extracted at the input inst_data granularity and the result is inserted at
750/// the target inst_data granularity, enabling slice cancellation during
751/// canonicalization.
752struct UnrollConvertLayoutOp : public UnrollPattern<xegpu::ConvertLayoutOp> {
753 using UnrollPattern<xegpu::ConvertLayoutOp>::UnrollPattern;
754
755 /// Extracts source in `inTile` slices, regroups into `convTile`-sized
756 /// ConvertLayoutOps, and inserts the result in `outTile` slices.
757 /// Returns failure if the tiles do not evenly divide.
758 LogicalResult
759 rewriteWithRegrouping(xegpu::ConvertLayoutOp op, VectorType valueTy,
760 ArrayRef<int64_t> convTile, ArrayRef<int64_t> inTile,
761 ArrayRef<int64_t> outTile,
762 xegpu::DistributeLayoutAttr inputLayout,
763 xegpu::DistributeLayoutAttr targetLayout, Location loc,
764 PatternRewriter &rewriter) const {
765 ArrayRef<int64_t> vecShape = valueTy.getShape();
766 if (!computeShapeRatio(shape: vecShape, subShape: convTile) ||
767 !computeShapeRatio(shape: convTile, subShape: inTile) ||
768 !computeShapeRatio(shape: convTile, subShape: outTile))
769 return failure();
770
771 Type elemTy = valueTy.getElementType();
772 int64_t rank = valueTy.getRank();
773 VectorType convTy = VectorType::get(shape: convTile, elementType: elemTy);
774 SmallVector<int64_t> strides(rank, 1);
775
776 Value source = op.getSource();
777 auto zeroOf = [&](VectorType ty) -> Value {
778 return arith::ConstantOp::create(
779 builder&: rewriter, location: loc, result: ty,
780 value: DenseElementsAttr::get(type: ty, values: rewriter.getZeroAttr(type: elemTy)));
781 };
782 auto addOffsets = [](ArrayRef<int64_t> a,
783 ArrayRef<int64_t> b) -> SmallVector<int64_t> {
784 SmallVector<int64_t> res(a);
785 for (auto [r, v] : llvm::zip_equal(t&: res, u&: b))
786 r += v;
787 return res;
788 };
789
790 Value result = zeroOf(valueTy);
791 for (SmallVector<int64_t> convOff :
792 StaticTileOffsetRange(vecShape, convTile)) {
793 // Build the convert tile from inTile-sized slices of the source.
794 Value conv;
795 if (convTile == inTile) {
796 conv = vector::ExtractStridedSliceOp::create(
797 builder&: rewriter, location: loc, source, offsets: convOff, sizes: convTile, strides);
798 } else {
799 conv = zeroOf(convTy);
800 for (SmallVector<int64_t> inLocal :
801 StaticTileOffsetRange(convTile, inTile)) {
802 Value piece = vector::ExtractStridedSliceOp::create(
803 builder&: rewriter, location: loc, source, offsets: addOffsets(convOff, inLocal), sizes: inTile,
804 strides);
805 conv = vector::InsertStridedSliceOp::create(builder&: rewriter, location: loc, valueToStore: piece,
806 dest: conv, offsets: inLocal, strides);
807 }
808 }
809
810 conv = xegpu::ConvertLayoutOp::create(builder&: rewriter, location: loc, result: convTy, source: conv,
811 input_layout: inputLayout, target_layout: targetLayout);
812
813 // Write the converted tile into the result as outTile-sized slices.
814 if (convTile == outTile) {
815 result = vector::InsertStridedSliceOp::create(builder&: rewriter, location: loc, valueToStore: conv,
816 dest: result, offsets: convOff, strides);
817 } else {
818 for (SmallVector<int64_t> outLocal :
819 StaticTileOffsetRange(convTile, outTile)) {
820 Value piece = vector::ExtractStridedSliceOp::create(
821 builder&: rewriter, location: loc, source: conv, offsets: outLocal, sizes: outTile, strides);
822 result = vector::InsertStridedSliceOp::create(
823 builder&: rewriter, location: loc, valueToStore: piece, dest: result, offsets: addOffsets(convOff, outLocal),
824 strides);
825 }
826 }
827 }
828
829 rewriter.replaceOp(op, newValues: result);
830 return success();
831 }
832
833 LogicalResult matchAndRewrite(xegpu::ConvertLayoutOp op,
834 PatternRewriter &rewriter) const override {
835 Location loc = op.getLoc();
836 Type valType = op.getType();
837
838 xegpu::DistributeLayoutAttr inputLayout = op.getEffectiveInputLayout();
839 xegpu::DistributeLayoutAttr targetLayout = op.getTargetLayoutAttr();
840 if (!inputLayout || !targetLayout)
841 return rewriter.notifyMatchFailure(arg&: op, msg: "missing layout attributes.");
842
843 if (valType.isIntOrFloat()) {
844 rewriter.replaceOp(op, newValues: op.getSource());
845 return success();
846 }
847
848 // Capture inst_data granularities before stripping them.
849 SmallVector<int64_t> inTile = inputLayout.getEffectiveInstDataAsInt();
850 SmallVector<int64_t> outTile = targetLayout.getEffectiveInstDataAsInt();
851 if (inTile.empty() || outTile.empty())
852 return rewriter.notifyMatchFailure(arg&: op, msg: "Not a target ConvertLayoutOp.");
853
854 inputLayout = inputLayout.dropInstData();
855 targetLayout = targetLayout.dropInstData();
856
857 VectorType valueTy = llvm::dyn_cast<VectorType>(Val: op.getType());
858 assert(valueTy && "the value type must be vector type!");
859
860 std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op);
861 if (!targetShape || targetShape->size() != (size_t)valueTy.getRank())
862 return failure();
863
864 // Nothing to convert if layouts match after stripping inst_data.
865 if (!inputLayout || !targetLayout || inputLayout.isEqualTo(other: targetLayout)) {
866 rewriter.replaceOp(op, newValues: op.getSource());
867 return success();
868 }
869
870 // Try regrouping: extract at inTile, convert, insert at outTile.
871 if (succeeded(Result: rewriteWithRegrouping(op, valueTy, convTile: *targetShape, inTile,
872 outTile, inputLayout, targetLayout, loc,
873 rewriter)))
874 return success();
875
876 // Fallback: pack/unpack at the convert tile granularity.
877 SmallVector<Type> convertedValTypes =
878 getUnrolledTypes(type: valueTy, tileShape: *targetShape);
879 SmallVector<Value> convertedValues =
880 pack(src: op.getOperand(), destTypes: convertedValTypes, blockSize: *targetShape, loc, rewriter);
881 SmallVector<Value> newOps;
882 for (auto [v, t] : llvm::zip(t&: convertedValues, u&: convertedValTypes)) {
883 auto newOp = xegpu::ConvertLayoutOp::create(builder&: rewriter, location: loc, result: t, source: v,
884 input_layout: inputLayout, target_layout: targetLayout);
885 newOps.push_back(Elt: newOp);
886 }
887 Value newSource = unpack(srcs: newOps, destTy: op.getType(), blockSize: *targetShape, loc, rewriter);
888
889 rewriter.replaceOp(op, newValues: newSource);
890 return success();
891 }
892};
893
894/// Unrolls vector.multi_reduction by sequentially reducing tiles with
895/// elementwise arith operations first, then a single multi_reduction
896/// per non-reduced tile position. This avoids generating long chains of
897/// multi_reduction ops (as the upstream pattern does) and is more efficient.
898///
899/// Example:
900/// vector.multi_reduction <32x64xf16> to <32xf16> (tile_shape=32, 32)
901/// -- Upstream pattern generates:
902/// %tmp1 = vector.multi_reduction %tile0, %zero_acc <32x32xf16> to <32xf16>
903/// %res = vector.multi_reduction %tmp1, %tile1 <32x32xf16> to <32xf16>
904/// -- This pattern generates:
905/// %tmp1 = arith.reduction %tile0, %tile1 <32x32xf16> -> <32x32xf16> //
906/// elementwise %res = vector.multi_reduction %tmp1, %zero_acc <32x32xf16> to
907/// <32xf16>
908struct UnrollMultiReductionOp
909 : public UnrollPattern<vector::MultiDimReductionOp> {
910 UnrollMultiReductionOp(MLIRContext *context,
911 const xegpu::UnrollOptions &options,
912 PatternBenefit benefit = 2)
913 : UnrollPattern<vector::MultiDimReductionOp>(context, options, benefit) {}
914
915 LogicalResult matchAndRewrite(vector::MultiDimReductionOp reductionOp,
916 PatternRewriter &rewriter) const override {
917 VectorType srcTy = reductionOp.getSourceVectorType();
918 ArrayRef<int64_t> srcShape = srcTy.getShape();
919 int64_t srcRank = srcTy.getRank();
920
921 Location loc = reductionOp.getLoc();
922 Value source = reductionOp.getSource();
923 Value acc = reductionOp.getAcc();
924 vector::CombiningKind kind = reductionOp.getKind();
925
926 // Result must be a vector (not scalar).
927 auto resultType = dyn_cast<VectorType>(Val: reductionOp.getDestType());
928 if (!resultType)
929 return failure();
930
931 std::optional<SmallVector<int64_t>> targetShapeOpt =
932 getTargetShape(op: reductionOp);
933 if (!targetShapeOpt ||
934 static_cast<int64_t>(targetShapeOpt->size()) != srcRank)
935 return failure();
936
937 SmallVector<int64_t> targetShape = *targetShapeOpt;
938
939 // Check divisibility for all dimensions.
940 for (int64_t i = 0; i < srcRank; ++i) {
941 if (srcShape[i] % targetShape[i] != 0)
942 return failure();
943 }
944
945 SmallVector<bool> reductionMask = reductionOp.getReductionMask();
946 // Identify reduced and kept dimensions from the reduction mask.
947 SmallVector<int64_t> reducedDims, keptDims;
948 for (int64_t i = 0; i < srcRank; ++i) {
949 if (reductionMask[i])
950 reducedDims.push_back(Elt: i);
951 else
952 keptDims.push_back(Elt: i);
953 }
954
955 // Compute the number of tiles along each reduced dimension and their
956 // product
957 SmallVector<int64_t> numReducedTilesPerDim;
958 for (int64_t d : reducedDims)
959 numReducedTilesPerDim.push_back(Elt: srcShape[d] / targetShape[d]);
960
961 // Build kept shapes for iterating over non-reduced dimensions.
962 SmallVector<int64_t> keptShape, keptTileShape;
963 for (int64_t d : keptDims) {
964 keptShape.push_back(Elt: srcShape[d]);
965 keptTileShape.push_back(Elt: targetShape[d]);
966 }
967
968 // Initialize the result vector for assembly.
969 Value result = arith::ConstantOp::create(builder&: rewriter, location: loc, result: resultType,
970 value: rewriter.getZeroAttr(type: resultType));
971
972 // Iterate over all tile positions in the kept dimensions.
973 // Ex: [off0, off1, _ _ off4]
974 // blanks are offsets for the reduced dims, they will be
975 // generated in the inner loop below
976 for (SmallVector<int64_t> keptOffsets :
977 StaticTileOffsetRange(keptShape, keptTileShape)) {
978
979 // Reconstruct full-rank base offsets with 0 for reduced dims.
980 // Ex: [off0, off1, 0, 0, off4]
981 SmallVector<int64_t> baseOffsets(srcRank, 0);
982 for (auto [idx, dim] : llvm::enumerate(First&: keptDims))
983 baseOffsets[dim] = keptOffsets[idx];
984
985 // Generate the full tile indices for the reduced dimensions.
986 // Ex: if reduceDimShapes = [32, 64] and
987 // reducedDimTargetShapes = [16, 16], then reducedTileCoords:
988 // [(0, 0), (0, 1), (0, 2), (0, 3),
989 // (1, 0), (1, 1), (1, 2), (1, 3)]
990 auto reducedTileCoords = StaticTileOffsetRange(
991 numReducedTilesPerDim, SmallVector<int64_t>(reducedDims.size(), 1));
992
993 // Step 1: Fill "blanks" in the offsets for the reduced dimensions
994 // using 'reducedTileCoords' and extract according tiles.
995 // Ex: tiles = [source[off0, off1, off2_red, off3_red, off4], ...]
996 SmallVector<Value> tiles;
997 for (SmallVector<int64_t> reducedTileIdx : reducedTileCoords) {
998 SmallVector<int64_t> offsets(baseOffsets);
999 for (auto [idx, dim] : llvm::enumerate(First&: reducedDims))
1000 offsets[dim] = reducedTileIdx[idx] * targetShape[dim];
1001 SmallVector<int64_t> strides(srcRank, 1);
1002 Value tile = vector::ExtractStridedSliceOp::create(
1003 builder&: rewriter, location: loc, source, offsets, sizes: targetShape, strides);
1004 tiles.push_back(Elt: tile);
1005 }
1006
1007 // Step 2: Sequentially reduce tiles using elementwise arith operations.
1008 Value reduced = tiles[0];
1009 for (size_t i = 1; i < tiles.size(); ++i)
1010 reduced =
1011 vector::makeArithReduction(b&: rewriter, loc, kind, v1: reduced, acc: tiles[i],
1012 fastmath: reductionOp.getFastmathAttr());
1013
1014 // Step 3: Perform a single multi_reduction with the accumulator slice.
1015 SmallVector<int64_t> accStrides(keptTileShape.size(), 1);
1016 Value accSlice = vector::ExtractStridedSliceOp::create(
1017 builder&: rewriter, location: loc, source: acc, offsets: keptOffsets, sizes: keptTileShape, strides: accStrides);
1018
1019 auto newReduction = vector::MultiDimReductionOp::create(
1020 builder&: rewriter, location: loc, source: reduced, acc: accSlice, reductionMask, kind,
1021 fastMathFlags: reductionOp.getFastmath());
1022
1023 // Step 4: Insert the reduced result into the output vector.
1024 SmallVector<int64_t> dstStrides(keptTileShape.size(), 1);
1025 result = vector::InsertStridedSliceOp::create(
1026 builder&: rewriter, location: loc, valueToStore: newReduction, dest: result, offsets: keptOffsets, strides: dstStrides);
1027 }
1028
1029 rewriter.replaceOp(op: reductionOp, newValues: result);
1030 return success();
1031 }
1032};
1033
1034} // namespace
1035
1036void mlir::xegpu::populateXeGPUUnrollPatterns(
1037 RewritePatternSet &patterns, const xegpu::UnrollOptions &options) {
1038 patterns
1039 .add<UnrollCreateNdOp, UnrollPrefetchNdOp, UnrollLoadNdOp,
1040 UnrollStoreNdOp, UnrollDpasOp, UnrollDpasMxOp, UnrollLoadMatrixOp,
1041 UnrollStoreMatrixOp, UnrollLoadGatherOp, UnrollStoreScatterOp,
1042 UnrollConvertLayoutOp, UnrollMultiReductionOp>(arg: patterns.getContext(),
1043 args: options);
1044}
1045