1//===---- XeGPUBlocking.cpp ---- XeGPU Blocking Pass ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/XeGPU/Transforms/Passes.h"
10
11#include "mlir/Dialect/Index/IR/IndexDialect.h"
12#include "mlir/Dialect/SCF/IR/SCF.h"
13#include "mlir/Dialect/SCF/Transforms/Patterns.h"
14#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
15#include "mlir/Dialect/XeGPU/IR/XeGPU.h"
16#include "mlir/Dialect/XeGPU/Transforms/Transforms.h"
17#include "mlir/Dialect/XeGPU/Transforms/XeGPULayoutImpl.h"
18#include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
19#include "mlir/Interfaces/LoopLikeInterface.h"
20#include "mlir/Pass/PassManager.h"
21#include "mlir/Transforms/DialectConversion.h"
22#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/ADT/SetVector.h"
25#include "llvm/Support/DebugLog.h"
26
27namespace mlir {
28namespace xegpu {
29#define GEN_PASS_DEF_XEGPUBLOCKING
30#include "mlir/Dialect/XeGPU/Transforms/Passes.h.inc"
31} // namespace xegpu
32} // namespace mlir
33
34#define DEBUG_TYPE "xegpu-blocking"
35
36using namespace mlir;
37
38namespace {
39
40//===------------------------------------------------------------------------===//
41// The XeGPUBlockingPass leverages the unroll patterns for XeGPU and Vector ops
42// to partition operations that process large shapes into multiple operations on
43// smaller shapes, as specified by the inst_data in the layout attribute. This
44// enables each resulting operation to be efficiently mapped to a hardware
45// instruction.
46//===------------------------------------------------------------------------===//
47
48class XeGPUBlockingPass final
49 : public xegpu::impl::XeGPUBlockingBase<XeGPUBlockingPass> {
50public:
51 void runOnOperation() override;
52
53private:
54 // Get the tile shape for a given OpOperand or OpResult by examining the
55 // corresponding layout attribute. If layout is not present or is not a
56 // subgroup level layout, it returns std::nullopt.
57 template <typename T,
58 typename = std::enable_if_t<std::is_same_v<T, OpOperand> ||
59 std::is_same_v<T, OpResult>>>
60 std::optional<SmallVector<int64_t>>
61 getTileShape(const T &operandOrResult) const;
62
63 // Get the tile shape for a given operation.
64 std::optional<SmallVector<int64_t>> getTileShape(Operation *op) const;
65
66 // Determine if the operation requires unrolling. Return false if all operands
67 // and results have tile shapes identical to their original types. Otherwise,
68 // return true.
69 bool needsUnroll(Operation *op) const;
70};
71} // namespace
72
73template <typename T, typename>
74std::optional<SmallVector<int64_t>>
75XeGPUBlockingPass::getTileShape(const T &operandOrResult) const {
76 Value value;
77 if constexpr (std::is_same_v<T, OpOperand>) {
78 value = operandOrResult.get();
79 } else {
80 value = (Value)operandOrResult;
81 }
82
83 xegpu::DistributeLayoutAttr layout =
84 xegpu::getDistributeLayoutAttr(operandOrResult);
85 if (layout && layout.isForSubgroup()) {
86 if (!layout.getEffectiveInstDataAsInt().empty()) {
87 SmallVector<int64_t> instData = layout.getEffectiveInstDataAsInt();
88 return instData;
89 }
90 if (auto type = dyn_cast<ShapedType>(Val: value.getType()))
91 return llvm::to_vector(Range: type.getShape());
92 }
93 LDBG() << "failed to getTileShape for: " << value;
94 return std::nullopt;
95}
96
97std::optional<SmallVector<int64_t>>
98XeGPUBlockingPass::getTileShape(Operation *op) const {
99 if (isa<xegpu::CreateNdDescOp, xegpu::LoadMatrixOp>(Val: op))
100 return getTileShape(operandOrResult: op->getOpResult(idx: 0));
101 if (isa<xegpu::PrefetchNdOp, xegpu::LoadNdOp, xegpu::PrefetchOp,
102 xegpu::StoreMatrixOp>(Val: op))
103 return getTileShape(operandOrResult: op->getOpOperand(idx: 0));
104 if (isa<xegpu::StoreNdOp>(Val: op))
105 return getTileShape(operandOrResult: op->getOpOperand(idx: 1));
106
107 if (isa<xegpu::LoadGatherOp>(Val: op))
108 return getTileShape(operandOrResult: op->getOpResult(idx: 0));
109
110 if (auto convertLayoutOp = dyn_cast<xegpu::ConvertLayoutOp>(Val: op)) {
111 auto inputInstData =
112 convertLayoutOp.getEffectiveInputLayout().getEffectiveInstDataAsInt();
113 auto targetInstData =
114 convertLayoutOp.getTargetLayout().getEffectiveInstDataAsInt();
115 assert(inputInstData.size() == targetInstData.size() &&
116 "convert_layout layouts must both carry inst_data of the same rank");
117 SmallVector<int64_t> tile(inputInstData.size());
118 for (size_t i = 0; i < tile.size(); ++i)
119 tile[i] = std::max(a: inputInstData[i], b: targetInstData[i]);
120 return tile;
121 }
122
123 if (isa<xegpu::StoreScatterOp>(Val: op))
124 return getTileShape(operandOrResult: op->getOpOperand(idx: 0));
125
126 // Helper lambda to validate and get A/B tiles
127 auto validateABTiles = [&](Operation *op)
128 -> std::optional<std::pair<SmallVector<int64_t>, SmallVector<int64_t>>> {
129 std::optional<SmallVector<int64_t>> aTile =
130 getTileShape(operandOrResult: op->getOpOperand(idx: 0));
131 std::optional<SmallVector<int64_t>> bTile =
132 getTileShape(operandOrResult: op->getOpOperand(idx: 1));
133
134 if (!aTile || aTile->size() < 2 || !bTile || bTile->size() < 2)
135 return std::nullopt;
136
137 // Both must have the same number of batch dimensions.
138 int64_t aBatchRank = aTile->size() - 2;
139 int64_t bBatchRank = bTile->size() - 2;
140 if (aBatchRank != bBatchRank)
141 return std::nullopt;
142
143 // Batch dimensions must match.
144 for (int64_t i = 0; i < aBatchRank; ++i) {
145 if ((*aTile)[i] != (*bTile)[i])
146 return std::nullopt;
147 }
148
149 // Semantic check for A and B: K dimension must match.
150 // A[..., M, K] x B[..., K, N]
151 if ((*aTile).back() != (*bTile)[bBatchRank])
152 return std::nullopt;
153
154 return std::make_pair(x&: *aTile, y&: *bTile);
155 };
156
157 // Helper lambda to validate C tile
158 auto validateCTile = [&](Operation *op, unsigned cOperandIdx,
159 const SmallVector<int64_t> &aTile,
160 const SmallVector<int64_t> &bTile) -> bool {
161 if (op->getNumOperands() <= cOperandIdx)
162 return true;
163
164 std::optional<SmallVector<int64_t>> cTile =
165 getTileShape(operandOrResult: op->getOpOperand(idx: cOperandIdx));
166 if (!cTile)
167 return false;
168 // Expected C tile: batch dims from A + [M, N]
169 int64_t aBatchRank = aTile.size() - 2;
170 SmallVector<int64_t> expectedCTile(aTile.begin(),
171 aTile.begin() + aBatchRank);
172 expectedCTile.push_back(Elt: aTile[aBatchRank]); // M from A
173 expectedCTile.push_back(Elt: bTile.back()); // N from B
174 if (!llvm::equal(LRange&: *cTile, RRange&: expectedCTile))
175 return false;
176 return true;
177 };
178
179 // Helper lambda to validate scale A tile for DpasMxOp
180 auto validateScaleATile =
181 [&](Operation *op, unsigned scaleAOperandIdx,
182 const SmallVector<int64_t> &aTile) -> std::optional<int64_t> {
183 std::optional<SmallVector<int64_t>> aScaleTile =
184 getTileShape(operandOrResult: op->getOpOperand(idx: scaleAOperandIdx));
185
186 if (!aScaleTile || aScaleTile->size() < 2)
187 return std::nullopt;
188
189 // Validate scale_a tile: [batch..., M_tile, K_scale]
190 // M dimension (second-to-last) must match A's M dimension
191 int64_t scaleRank = aScaleTile->size();
192 int64_t aBatchRank = aTile.size() - 2;
193 if ((*aScaleTile)[scaleRank - 2] != aTile[aBatchRank])
194 return std::nullopt;
195
196 // Return the K scale factor (last dim)
197 return aScaleTile->back();
198 };
199
200 // Helper lambda to validate scale B tile for DpasMxOp
201 auto validateScaleBTile =
202 [&](Operation *op, unsigned scaleBOperandIdx,
203 const SmallVector<int64_t> &bTile) -> std::optional<int64_t> {
204 std::optional<SmallVector<int64_t>> bScaleTile =
205 getTileShape(operandOrResult: op->getOpOperand(idx: scaleBOperandIdx));
206
207 if (!bScaleTile || bScaleTile->size() < 2)
208 return std::nullopt;
209
210 // Validate scale_b tile: [batch..., K_scale, N_tile]
211 // N dimension (last) must match B's N dimension (last)
212 if (bScaleTile->back() != bTile.back())
213 return std::nullopt;
214
215 // Return the K scale factor (second-to-last dim)
216 int64_t scaleRank = bScaleTile->size();
217 return (*bScaleTile)[scaleRank - 2];
218 };
219
220 if (isa<xegpu::DpasOp>(Val: op)) {
221 auto abTiles = validateABTiles(op);
222 if (!abTiles)
223 return std::nullopt;
224
225 auto [aTile, bTile] = *abTiles;
226
227 // Semantic check for C.
228 if (!validateCTile(op, 2, aTile, bTile))
229 return std::nullopt;
230
231 // Return [batch..., M, K, N] as the target shape for unrolling.
232 int64_t aBatchRank = aTile.size() - 2;
233 SmallVector<int64_t> tileShape(aTile.begin(), aTile.begin() + aBatchRank);
234 tileShape.push_back(Elt: aTile[aBatchRank]); // M
235 tileShape.push_back(Elt: aTile[aBatchRank + 1]); // K
236 tileShape.push_back(Elt: bTile.back()); // N
237 return tileShape;
238 }
239
240 if (auto dpasMxOp = dyn_cast<xegpu::DpasMxOp>(Val: op)) {
241 auto abTiles = validateABTiles(op);
242 if (!abTiles)
243 return std::nullopt;
244
245 auto [aTile, bTile] = *abTiles;
246
247 // Validate C tile if present using op-specific accessor
248 if (dpasMxOp.getAcc()) {
249 unsigned accOperandIdx = 2; // acc is the 3rd operand
250 if (!validateCTile(op, accOperandIdx, aTile, bTile))
251 return std::nullopt;
252 }
253
254 // Validate scale tiles if present using op-specific accessors
255 int64_t kScaleFactor = 1;
256 std::optional<int64_t> scaleAFactor;
257 std::optional<int64_t> scaleBFactor;
258
259 if (dpasMxOp.getScaleA()) {
260 unsigned scaleAOperandIdx = 2 + (dpasMxOp.getAcc() ? 1 : 0);
261 scaleAFactor = validateScaleATile(op, scaleAOperandIdx, aTile);
262 if (!scaleAFactor)
263 return std::nullopt;
264 }
265
266 if (dpasMxOp.getScaleB()) {
267 unsigned scaleBOperandIdx =
268 2 + (dpasMxOp.getAcc() ? 1 : 0) + (dpasMxOp.getScaleA() ? 1 : 0);
269 scaleBFactor = validateScaleBTile(op, scaleBOperandIdx, bTile);
270 if (!scaleBFactor)
271 return std::nullopt;
272 }
273
274 // If both scales are present, their K dimensions must match
275 if (scaleAFactor && scaleBFactor) {
276 if (*scaleAFactor != *scaleBFactor)
277 return std::nullopt;
278 kScaleFactor = *scaleAFactor;
279 } else if (scaleAFactor) {
280 kScaleFactor = *scaleAFactor;
281 } else if (scaleBFactor) {
282 kScaleFactor = *scaleBFactor;
283 }
284
285 // Return [batch..., M, K, N, S] as the target shape for unrolling.
286 int64_t aBatchRank = aTile.size() - 2;
287 SmallVector<int64_t> tileShape(aTile.begin(), aTile.begin() + aBatchRank);
288 tileShape.push_back(Elt: aTile[aBatchRank]); // M
289 tileShape.push_back(Elt: aTile[aBatchRank + 1]); // K
290 tileShape.push_back(Elt: bTile.back()); // N
291 tileShape.push_back(Elt: kScaleFactor); // S
292 return tileShape;
293 }
294
295 if (OpTrait::hasElementwiseMappableTraits(op) && op->getNumResults() == 1)
296 return getTileShape(operandOrResult: op->getOpResult(idx: 0));
297
298 if (isa<vector::MultiDimReductionOp>(Val: op))
299 return getTileShape(operandOrResult: op->getOpOperand(idx: 0));
300
301 if (isa<vector::TransposeOp, vector::BroadcastOp, vector::StepOp,
302 vector::ShapeCastOp, vector::ConstantMaskOp, vector::CreateMaskOp,
303 vector::BitCastOp, vector::InterleaveOp, vector::DeinterleaveOp>(Val: op))
304 return getTileShape(operandOrResult: op->getOpResult(idx: 0));
305
306 return std::nullopt;
307}
308
309bool XeGPUBlockingPass::needsUnroll(Operation *op) const {
310 // skip the op if any of its operands or results has workgroup level layouts
311 bool hasWgLayoutOperands =
312 llvm::any_of(Range: op->getOpOperands(), P: [](OpOperand &opr) {
313 xegpu::DistributeLayoutAttr layout =
314 xegpu::getDistributeLayoutAttr(opr);
315 return layout && layout.isForWorkgroup();
316 });
317 bool hasWgLayoutResults =
318 llvm::any_of(Range: op->getOpResults(), P: [](OpResult result) {
319 xegpu::DistributeLayoutAttr layout =
320 xegpu::getDistributeLayoutAttr(value: result);
321 return layout && layout.isForWorkgroup();
322 });
323 if (hasWgLayoutOperands || hasWgLayoutResults) {
324 LDBG() << "skip unrolling for op with workgroup level layout: " << *op;
325 return false;
326 }
327
328 auto isUnrollable = [](Value value, ArrayRef<int64_t> tileShape) {
329 Type valTy = value.getType();
330 if (auto tdescTy = dyn_cast<xegpu::TensorDescType>(Val&: valTy)) {
331 xegpu::DistributeLayoutAttr layout = tdescTy.getLayoutAttr();
332 return layout && !layout.getEffectiveInstDataAsInt().empty();
333 }
334 auto shapedType = dyn_cast<ShapedType>(Val&: valTy);
335 return shapedType && !llvm::equal(LRange&: tileShape, RRange: shapedType.getShape());
336 };
337
338 bool hasUnrollableOperands =
339 llvm::any_of(Range: op->getOpOperands(), P: [&](OpOperand &opr) {
340 std::optional<SmallVector<int64_t>> tileShape = getTileShape(operandOrResult: opr);
341 return tileShape.has_value() && isUnrollable(opr.get(), *tileShape);
342 });
343 bool hasUnrollableResults =
344 llvm::any_of(Range: op->getOpResults(), P: [&](OpResult result) {
345 std::optional<SmallVector<int64_t>> tileShape = getTileShape(operandOrResult: result);
346 return tileShape.has_value() && isUnrollable(result, *tileShape);
347 });
348 // ConvertLayoutOp must be processed to drop the inst_data in the layout
349 bool isConvertLayoutWithInstData = false;
350 if (auto convertLayoutOp = dyn_cast<xegpu::ConvertLayoutOp>(Val: op)) {
351 auto targettLayout = convertLayoutOp.getTargetLayout();
352 if (targettLayout && !targettLayout.getEffectiveInstDataAsInt().empty()) {
353 isConvertLayoutWithInstData = true;
354 }
355 }
356 return hasUnrollableOperands || hasUnrollableResults ||
357 isConvertLayoutWithInstData;
358}
359
360void XeGPUBlockingPass::runOnOperation() {
361 MLIRContext *ctx = &getContext();
362 Operation *op = getOperation();
363
364 if (!xegpu::recoverTemporaryLayouts(rootOp: op)) {
365 signalPassFailure();
366 return;
367 }
368
369 auto getTileShapeAndCount = [](llvm::ArrayRef<int64_t> shape,
370 xegpu::DistributeLayoutAttr layout) {
371 int count = 1;
372 SmallVector<int64_t> tileShape(shape);
373 if (layout && !layout.getEffectiveInstDataAsInt().empty()) {
374 tileShape = layout.getEffectiveInstDataAsInt();
375 count = computeProduct(basis: shape) / computeProduct(basis: tileShape);
376 }
377 assert(count >= 1 && "count must be at least 1");
378 return std::make_pair(x&: tileShape, y&: count);
379 };
380
381 // Perform context-aware type conversion for SCF structural ops.
382 // Inspects Values to find inst_data layout information for 1:N conversion.
383 llvm::SmallSetVector<UnrealizedConversionCastOp, 8> existingCasts;
384 op->walk(
385 callback: [&](UnrealizedConversionCastOp castOp) { existingCasts.insert(X: castOp); });
386
387 {
388 TypeConverter converter;
389 converter.addConversion(callback: [](Type type) -> Type { return type; });
390
391 // TensorDescType 1:N converter (type-based, layout is in the type).
392 converter.addConversion(
393 callback: [&](xegpu::TensorDescType type,
394 SmallVectorImpl<Type> &result) -> std::optional<LogicalResult> {
395 Type elemTy = type.getElementType();
396 ArrayRef<int64_t> shape = type.getShape();
397
398 xegpu::DistributeLayoutAttr layout = type.getLayoutAttr();
399 if (layout && layout.isForWorkgroup())
400 return failure();
401
402 int count;
403 SmallVector<int64_t> subShape;
404 std::tie(args&: subShape, args&: count) = getTileShapeAndCount(shape, layout);
405
406 if (layout)
407 layout = layout.dropInstData();
408
409 auto newTy = xegpu::TensorDescType::get(
410 context: type.getContext(), shape: subShape, elementType: elemTy, encoding: type.getEncoding(), layout);
411 result.append(NumInputs: count, Elt: newTy);
412 return success();
413 });
414
415 // Context-aware VectorType conversion based on inst_data (1:1
416 // shape-changing or 1:N).
417 auto getSubShapeAndCount = [&](VectorType vecTy,
418 xegpu::DistributeLayoutAttr layout)
419 -> std::pair<SmallVector<int64_t>, int> {
420 return getTileShapeAndCount(vecTy.getShape(), layout);
421 };
422 auto loopArgTypes =
423 xegpu::precomputeLoopBlockArgTypes(topLevelOp: op, getSubShapeAndCount);
424 xegpu::addVectorTypeConversion(converter, getSubShapeAndCount,
425 loopArgTypes: std::move(loopArgTypes));
426
427 // Loop-carried types are now in the converter's map, so the transient
428 // per-position layout attrs on SCF loop ops are no longer needed. Strip
429 // them before converting: the SCF converters copy old attrs onto the new
430 // op (ConvertForOpTypes::setAttrs), and after 1:N result expansion a stale
431 // `layout_result_N` lands on the wrong (renumbered) result, corrupting the
432 // count invariant and leaving the loop illegal.
433 op->walk(callback: [](Operation *loopOp) {
434 if (!isa<scf::ForOp, scf::WhileOp, scf::ConditionOp, scf::IfOp>(Val: loopOp))
435 return;
436 SmallVector<StringRef> toRemove;
437 for (const NamedAttribute &attr :
438 loopOp->getDiscardableAttrDictionary().getValue()) {
439 StringRef name = attr.getName().strref();
440 if (name.starts_with(Prefix: "layout_operand_") ||
441 name.starts_with(Prefix: "layout_result_"))
442 toRemove.push_back(Elt: name);
443 }
444 for (StringRef name : toRemove)
445 loopOp->removeDiscardableAttr(name);
446 });
447
448 // Source (N:1) and target (1:1) materializations using
449 // UnrealizedConversionCastOp.
450 auto materializeCast = [](OpBuilder &builder, Type type, ValueRange inputs,
451 Location loc) -> Value {
452 return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: type, operands: inputs)
453 .getResult(i: 0);
454 };
455 converter.addSourceMaterialization(callback&: materializeCast);
456 converter.addTargetMaterialization(callback&: materializeCast);
457 // Blocking runs SCF conversion separately (not combined with XeGPU
458 // patterns), so it also needs a 1:N target materialization.
459 converter.addTargetMaterialization(
460 callback: [](mlir::OpBuilder &builder, mlir::TypeRange types,
461 mlir::ValueRange inputs, mlir::Location loc) -> SmallVector<Value> {
462 auto castOp =
463 UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: types, operands: inputs);
464 return SmallVector<Value>(castOp.getResults());
465 });
466
467 ConversionTarget target(*ctx);
468 target.addLegalOp<UnrealizedConversionCastOp>();
469 target.markUnknownOpDynamicallyLegal(fn: [](Operation *) { return true; });
470
471 RewritePatternSet scfPatterns(ctx);
472 scf::populateSCFStructuralTypeConversionsAndLegality(typeConverter: converter, patterns&: scfPatterns,
473 target);
474 if (failed(Result: applyPartialConversion(op, target, patterns: std::move(scfPatterns))))
475 return signalPassFailure();
476
477 // Fold cancelling cast chains and erase dead casts.
478 xegpu::cleanupUnrealizedConversionCasts(root: op, existingCasts);
479 }
480
481 xegpu::UnrollOptions options;
482 options.setFilterConstraint(
483 [&](Operation *op) -> LogicalResult { return success(IsSuccess: needsUnroll(op)); });
484
485 options.setNativeShapeFn([&](Operation *op) { return getTileShape(op); });
486
487 options.setUnrolledTypesFn([&](ShapedType type, ArrayRef<int64_t> tileShape) {
488 Type elemTy = type.getElementType();
489
490 if (auto tdescTy = dyn_cast<xegpu::TensorDescType>(Val&: type)) {
491
492 Attribute encoding = tdescTy.getEncoding();
493
494 xegpu::TensorDescType newTy =
495 xegpu::TensorDescType::get(context: ctx, shape: tileShape, elementType: elemTy, encoding,
496 layout: tdescTy.getLayoutAttr().dropInstData());
497 // Compute the product of batch (higher) dimensions.
498 ArrayRef<int64_t> shape = type.getShape();
499 int64_t batchCount =
500 shape.size() > 2 ? computeProduct(basis: shape.drop_back(N: 2)) : 1;
501 return SmallVector<Type>(batchCount, newTy);
502 }
503 Type newTy = VectorType::get(shape: tileShape, elementType: elemTy);
504
505 std::optional<SmallVector<int64_t>> ratio =
506 computeShapeRatio(shape: type.getShape(), subShape: tileShape);
507 assert(ratio && "The shape of the type must be a multiple of tileShape.");
508 return SmallVector<Type>(computeProduct(basis: *ratio), newTy);
509 });
510
511 RewritePatternSet patterns(ctx);
512 vector::UnrollVectorOptions vectorOptions;
513 vectorOptions.setNativeShapeFn(options.nativeShape);
514
515 populateXeGPUUnrollPatterns(patterns, options);
516 vector::populateVectorUnrollPatterns(patterns, options: vectorOptions);
517
518 // Note: The pattern driver does op folding as well and clean up.
519 // But intermediate insert/extract strided slice ops with
520 // unrealized conversion cast ops in the middle does not get
521 // cleaned up in this step. One more round of folding is needed
522 // after the walk to resolve those unrealized conversion cast ops.
523 (void)applyPatternsGreedily(op, patterns: std::move(patterns));
524
525 op->walk(callback: [](Operation *op) {
526 // Remove the layout attributes cached per operands.
527 for (OpOperand &opr : op->getOpOperands()) {
528 std::string name = xegpu::getTemporaryLayoutName(operand: opr);
529 if (op->hasDiscardableAttrOfType<xegpu::DistributeLayoutAttr>(name))
530 op->removeDiscardableAttr(name);
531 }
532
533 // Update the layout attributes per result.
534 for (OpResult result : op->getOpResults()) {
535 std::string name = xegpu::getTemporaryLayoutName(result);
536 if (auto layout =
537 op->getDiscardableAttrOfType<xegpu::DistributeLayoutAttr>(name)) {
538 op->removeDiscardableAttr(name);
539 if (!isa<LoopLikeOpInterface>(Val: op))
540 xegpu::setDistributeLayoutAttr(Result: result, layout: layout.dropInstData());
541 }
542 }
543
544 // Drop left-over inst_data if the unroll pattern does not being applied,
545 // say, inst_data just matches their shape.
546 SmallVector<NamedAttribute> newAttrs = xegpu::dropInstDataOnAttrs(
547 attrs: op->getDiscardableAttrDictionary().getValue());
548 op->setDiscardableAttrs(newAttrs);
549 xegpu::dropInstDataOnInherentAttrs(op);
550 });
551
552 // Resolve UnrealizedConversionCastOps generated by SCF structural type
553 // conversion and by XeGPU/Vector unrolling (cancelling cast chains and
554 // unpaired pack/unpack casts).
555 xegpu::cleanupUnrealizedConversionCasts(root: op, existingCasts);
556
557 // One more round of folding to clean up the intermediate
558 // insert/extract strided slice ops.
559 RewritePatternSet emptyPatterns(ctx);
560 (void)applyPatternsGreedily(op, patterns: std::move(emptyPatterns));
561}
562