1//===---- XeGPUUtils.cpp - MLIR Utilities for XeGPUOps ------------------===//
2//
3// Part of the MLIR Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements utility methods for working with the XeGPU dialect.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
14#include "mlir/Dialect/Arith/Utils/Utils.h"
15#include "mlir/Dialect/GPU/IR/GPUDialect.h"
16#include "mlir/Dialect/LLVMIR/XeVMDialect.h"
17#include "mlir/Dialect/SCF/IR/SCF.h"
18#include "mlir/Dialect/Utils/IndexingUtils.h"
19#include "mlir/Dialect/Vector/IR/VectorOps.h"
20#include "mlir/Dialect/XeGPU/IR/XeGPU.h"
21#include "mlir/Dialect/XeGPU/uArch/uArchCommon.h"
22#include "mlir/IR/Builders.h"
23#include "mlir/IR/BuiltinOps.h"
24#include "mlir/IR/Operation.h"
25#include "mlir/IR/ValueRange.h"
26#include "mlir/Interfaces/ControlFlowInterfaces.h"
27#include "mlir/Interfaces/LoopLikeInterface.h"
28#include "mlir/Transforms/DialectConversion.h"
29#include "llvm/Support/Casting.h"
30#include "llvm/Support/FormatVariadic.h"
31#include <cstdint>
32#include <numeric>
33
34using namespace mlir;
35
36/// convert ArrayRef<ValueRange> into SmallVector<Value>
37SmallVector<Value> xegpu::flattenValues(ArrayRef<ValueRange> values) {
38 SmallVector<Value> result;
39 for (const auto &vals : values)
40 llvm::append_range(C&: result, R: vals);
41 return result;
42}
43
44FailureOr<VectorType>
45mlir::xegpu::getDistributedVectorType(xegpu::TensorDescType tdescTy) {
46 auto layout = llvm::dyn_cast_if_present<LayoutAttr>(Val: tdescTy.getLayout());
47 // It only works for subgroup level layout, which only has lane_layout
48 // and lane_data, and is to distribute a SIMD code into SIMT code.
49 if (!layout || !layout.isForSubgroup())
50 return failure();
51
52 SmallVector<int64_t> laneData(layout.getLaneData().asArrayRef());
53 SmallVector<int64_t> laneLayout(layout.getLaneLayout().asArrayRef());
54 auto tdescShape = tdescTy.getShape();
55 auto elementType = tdescTy.getElementType();
56
57 // compute sgSize by multiply elements of laneLayout
58 // e.g. for 2D layout, sgSize = laneLayout[0] * laneLayout[1]
59 // e.g. for 1D layout, sgSize = laneLayout[0]
60 int64_t sgSize = llvm::product_of(Range&: laneLayout);
61
62 // Check if the tensor descriptor shape is distributable.
63 int64_t tensorSize = 1;
64 for (auto [tdescDim, laneDim, laneDataDim] :
65 llvm::zip_equal(t&: tdescShape, u&: laneLayout, args&: laneData)) {
66 assert((tdescDim % (laneDim * laneDataDim) == 0) &&
67 "tensor descriptor shape is not distributable");
68 tensorSize *= tdescDim;
69 }
70 // tensorSize must be adjusted for array_length.
71 tensorSize *= tdescTy.getArrayLength();
72
73 return VectorType::get(shape: {tensorSize / sgSize}, elementType);
74}
75
76FailureOr<VectorType>
77mlir::xegpu::getDistributedVectorType(VectorType originalType,
78 xegpu::LayoutAttr layout) {
79 int64_t rank = originalType.getRank();
80 if (rank < 1)
81 return failure();
82 ArrayRef<int64_t> shape = originalType.getShape();
83 // For rank > 2, leading dimensions are treated as batch/array dimensions.
84 // Drop them and use the product as arrayLength.
85 int arrayLength = 1;
86 while (shape.size() > 2) {
87 arrayLength *= shape[0];
88 shape = shape.drop_front();
89 }
90 // Drop matching leading dims from layout if the layout rank exceeds the
91 // remaining shape rank.
92 auto laneLayout = layout.getEffectiveLaneLayoutAsInt();
93 auto laneData = layout.getEffectiveLaneDataAsInt();
94 while (!laneLayout.empty() && laneLayout.size() > shape.size()) {
95 laneLayout.erase(CI: laneLayout.begin());
96 laneData.erase(CI: laneData.begin());
97 }
98 auto trimmedLayout = xegpu::LayoutAttr::get(
99 context: layout.getContext(),
100 lane_layout: SmallVector<int32_t>(laneLayout.begin(), laneLayout.end()),
101 lane_data: SmallVector<int32_t>(laneData.begin(), laneData.end()));
102 auto helperTdescTy = xegpu::TensorDescType::get(
103 shape, elementType: originalType.getElementType(), array_length: arrayLength,
104 /*boundary_check=*/true,
105 /*memory_space=*/xegpu::MemorySpace::Global, layout: trimmedLayout);
106 return xegpu::getDistributedVectorType(tdescTy: helperTdescTy);
107}
108
109FailureOr<VectorType>
110xegpu::getDistVecTypeBasedOnLaneLayout(xegpu::DistributeLayoutAttr layout,
111 VectorType originalType) {
112 if (!layout)
113 return failure();
114 assert((isa<xegpu::LayoutAttr>(layout) || isa<xegpu::SliceAttr>(layout)) &&
115 "Expecting a valid layout.");
116
117 int64_t vectorRank = originalType.getRank();
118 int64_t layoutRank = layout.getRank();
119 assert(vectorRank >= layoutRank && "Vector rank must be >= layout rank.");
120
121 // When the vector has more dimensions than the layout, only the trailing
122 // dimensions are distributed. Leading dimensions are preserved as-is.
123 int64_t offset = vectorRank - layoutRank;
124 ArrayRef<int64_t> fullShape = originalType.getShape();
125 SmallVector<int64_t> trailingShape(fullShape.begin() + offset,
126 fullShape.end());
127 auto distributedShapeOrFailure =
128 layout.computeDistributedShape(shape: trailingShape);
129 if (failed(Result: distributedShapeOrFailure))
130 return failure();
131
132 SmallVector<int64_t> resultShape(fullShape.begin(),
133 fullShape.begin() + offset);
134 resultShape.append(in_start: distributedShapeOrFailure->begin(),
135 in_end: distributedShapeOrFailure->end());
136 return VectorType::get(shape: resultShape, elementType: originalType.getElementType());
137}
138
139std::string xegpu::getTemporaryLayoutName(const OpOperand &operand) {
140 const StringRef prefix("layout_operand_");
141 unsigned idx = const_cast<OpOperand &>(operand).getOperandNumber();
142 return llvm::formatv(Fmt: "{0}{1}", Vals: prefix, Vals&: idx).str();
143}
144
145std::string xegpu::getTemporaryLayoutName(const OpResult result) {
146 const StringRef prefix = "layout_result_";
147 return llvm::formatv(Fmt: "{0}{1}", Vals: prefix, Vals: result.getResultNumber()).str();
148}
149
150xegpu::DistributeLayoutAttr xegpu::getDistributeLayoutAttr(const Value value) {
151 if (!value)
152 return nullptr;
153
154 if (auto result = dyn_cast<OpResult>(Val: value)) {
155 Operation *defOp = result.getDefiningOp();
156 assert(defOp && "result must have a defining op");
157
158 if (auto anchorOp = dyn_cast<xegpu::AnchorLayoutInterface>(Val: defOp)) {
159 auto layout = anchorOp.getAnchorLayout();
160 return layout;
161 }
162
163 std::string layoutName = getTemporaryLayoutName(result);
164 if (defOp->hasDiscardableAttr(name: layoutName)) {
165 auto layout =
166 defOp->getDiscardableAttrOfType<xegpu::DistributeLayoutAttr>(
167 name: layoutName);
168 return layout;
169 }
170 }
171
172 if (auto arg = dyn_cast<BlockArgument>(Val: value)) {
173 auto *parentOp = arg.getOwner()->getParentOp();
174 auto loop = dyn_cast_if_present<LoopLikeOpInterface>(Val: parentOp);
175 if (loop)
176 if (OpOperand *tiedInit = loop.getTiedLoopInit(bbArg: arg))
177 return getTemporaryLayout(operandOrResult: *tiedInit);
178 // An scf.while "after" argument is tied to no init operand; scf.condition
179 // feeds it. Only a pass-through is supported: the forwarded value must be
180 // the matching "before" argument, whose tied init operand carries the
181 // layout.
182 if (auto whileOp = dyn_cast_if_present<scf::WhileOp>(Val: parentOp);
183 whileOp && arg.getOwner()->getParent() == &whileOp.getAfter()) {
184 Value forwarded = whileOp.getConditionOp().getArgs()[arg.getArgNumber()];
185 if (auto beforeArg = dyn_cast<BlockArgument>(Val&: forwarded))
186 if (OpOperand *tiedInit = whileOp.getTiedLoopInit(bbArg: beforeArg))
187 return getTemporaryLayout(operandOrResult: *tiedInit);
188 }
189 }
190
191 if (auto tdescTy =
192 dyn_cast_if_present<xegpu::TensorDescType>(Val: value.getType()))
193 return tdescTy.getLayoutAttr();
194
195 return nullptr;
196}
197xegpu::DistributeLayoutAttr
198xegpu::getDistributeLayoutAttr(const OpOperand &opr) {
199 Operation *op = opr.getOwner();
200 unsigned idx = const_cast<OpOperand &>(opr).getOperandNumber();
201
202 if (auto anchorOp = dyn_cast<xegpu::AnchorLayoutInterface>(Val: op)) {
203 if (auto dpasOp = dyn_cast<xegpu::DpasOp>(Val: op)) {
204 if (idx == 0) {
205 return dpasOp.getLayoutAAttr();
206 } else if (idx == 1) {
207 return dpasOp.getLayoutBAttr();
208 } else if (idx == 2) {
209 return dpasOp.getLayoutCdAttr();
210 }
211 }
212 if (auto dpasMxOp = dyn_cast<xegpu::DpasMxOp>(Val: op)) {
213 // DpasMxOp has operands: a, b, optional acc, optional scale_a, optional
214 // scale_b
215 unsigned currentIdx = 0;
216
217 if (idx == currentIdx++)
218 return dpasMxOp.getLayoutAAttr();
219
220 if (idx == currentIdx++)
221 return dpasMxOp.getLayoutBAttr();
222
223 if (dpasMxOp.getAcc())
224 if (idx == currentIdx++)
225 return dpasMxOp.getLayoutCdAttr();
226
227 if (dpasMxOp.getScaleA())
228 if (idx == currentIdx++)
229 return dpasMxOp.getLayoutAScaleAttr();
230
231 if (dpasMxOp.getScaleB())
232 if (idx == currentIdx++)
233 return dpasMxOp.getLayoutBScaleAttr();
234
235 return nullptr;
236 }
237 if (auto convertOp = dyn_cast<xegpu::ConvertLayoutOp>(Val: op)) {
238 return convertOp.getEffectiveInputLayout();
239 }
240 auto layout = anchorOp.getAnchorLayout();
241
242 if (idx == 0)
243 return layout;
244
245 // For StoreNdOp and StoreMatrixOp,
246 // the layout is valid for the first two operands: value and memref/tdesc.
247 if (isa<xegpu::StoreNdOp, xegpu::StoreMatrixOp>(Val: op) && (idx < 2))
248 return layout;
249
250 // For gather/scatter ops the mask and offsets share the value's layout.
251 if (isa<xegpu::StoreScatterOp, xegpu::LoadGatherOp>(Val: op))
252 return layout;
253 }
254
255 std::string layoutName = xegpu::getTemporaryLayoutName(operand: opr);
256 if (op->hasDiscardableAttr(name: layoutName)) {
257 auto layout =
258 op->getDiscardableAttrOfType<xegpu::DistributeLayoutAttr>(name: layoutName);
259 return layout;
260 }
261
262 return nullptr;
263}
264
265// Returns the permanent layout attribute for the given result if it's
266// available on the defining op. Otherwise returns the provided layout.
267xegpu::DistributeLayoutAttr
268maybePickPermanentLayout(xegpu::DistributeLayoutAttr layout,
269 const OpResult &result, mlir::Operation *owner,
270 const std::string &name) {
271 xegpu::DistributeLayoutAttr candidate = layout;
272
273 if (auto loadOp = dyn_cast<xegpu::LoadGatherOp>(Val: owner)) {
274 if (auto perm = loadOp.getLayoutAttr())
275 candidate = perm;
276 }
277
278 return candidate;
279}
280
281// Returns the permanent layout attribute for the given operand if it's
282// available on the defining op. Otherwise returns the provided layout.
283xegpu::DistributeLayoutAttr
284maybePickPermanentLayout(xegpu::DistributeLayoutAttr layout,
285 const OpOperand &operand, mlir::Operation *owner,
286 const std::string &name) {
287 xegpu::DistributeLayoutAttr candidate = layout;
288 unsigned idx = const_cast<OpOperand &>(operand).getOperandNumber();
289
290 if (auto storeOp = dyn_cast<xegpu::StoreScatterOp>(Val: owner)) {
291 if (idx == 0) {
292 if (auto perm = storeOp.getLayoutAttr())
293 candidate = perm;
294 }
295 }
296
297 return candidate;
298}
299
300// TODO-LayoutRefactor: Remove this function after replacing use
301// with setTemporaryLayout or setAnchorLayout
302void xegpu::setDistributeLayoutAttr(
303 const mlir::OpResult &result,
304 const mlir::xegpu::DistributeLayoutAttr layout) {
305 Operation *owner = result.getOwner();
306
307 if (auto anchorOp = dyn_cast<xegpu::AnchorLayoutInterface>(Val: owner)) {
308 if (anchorOp.getAnchorLayout() == layout)
309 return;
310 anchorOp.setAnchorLayout(layout);
311 return;
312 }
313
314 std::string name = xegpu::getTemporaryLayoutName(result);
315 if (owner->hasDiscardableAttrOfType<DistributeLayoutAttr>(name)) {
316 return;
317 }
318 if (layout) {
319 owner->setDiscardableAttr(name, value: layout);
320 }
321}
322
323// TODO-LayoutRefactor: Remove this function after replacing use
324// with setTemporaryLayout or setAnchorLayout
325void xegpu::setDistributeLayoutAttr(const OpOperand &operand,
326 const DistributeLayoutAttr layout) {
327 Operation *owner = operand.getOwner();
328 unsigned idx = const_cast<OpOperand &>(operand).getOperandNumber();
329
330 if (!layout) {
331 return;
332 }
333 if (auto anchorOp = dyn_cast<xegpu::AnchorLayoutInterface>(Val: owner)) {
334 if (auto dpasOp = dyn_cast<xegpu::DpasOp>(Val: owner)) {
335 if (idx == 0) {
336 return dpasOp.setLayoutAAttr(layout);
337 } else if (idx == 1) {
338 return dpasOp.setLayoutBAttr(layout);
339 } else if (idx == 2) {
340 return dpasOp.setLayoutCdAttr(layout);
341 }
342 }
343 if (auto convertOp = dyn_cast<xegpu::ConvertLayoutOp>(Val: owner)) {
344 return convertOp.setInputLayoutAttr(layout);
345 }
346
347 // For store operations (StoreScatterOp, StoreNdOp, StoreMatrixOp),
348 // the layout is valid for the first two operands: value and memref/tdesc.
349 // For other operations, the layout applies to the first operand only.
350 if (isa<xegpu::StoreScatterOp, xegpu::StoreNdOp, xegpu::StoreMatrixOp>(
351 Val: owner)) {
352 if (idx < 2) {
353 anchorOp.setAnchorLayout(layout);
354 }
355 } else {
356 if (idx == 0) {
357 anchorOp.setAnchorLayout(layout);
358 }
359 }
360 }
361
362 std::string name = xegpu::getTemporaryLayoutName(operand);
363 if (owner->hasDiscardableAttrOfType<DistributeLayoutAttr>(name)) {
364 return;
365 }
366 if (layout) {
367 owner->setDiscardableAttr(name, value: layout);
368 }
369}
370
371template <typename T, typename>
372xegpu::DistributeLayoutAttr
373xegpu::getTemporaryLayout(const T &operandOrResult) {
374 Operation *op = operandOrResult.getOwner();
375
376 std::string layoutName = xegpu::getTemporaryLayoutName(operandOrResult);
377 if (op->hasDiscardableAttr(name: layoutName)) {
378 auto layout =
379 op->getDiscardableAttrOfType<xegpu::DistributeLayoutAttr>(name: layoutName);
380 return layout;
381 }
382
383 return nullptr;
384}
385
386template xegpu::DistributeLayoutAttr
387xegpu::getTemporaryLayout<mlir::OpResult>(const OpResult &result);
388template xegpu::DistributeLayoutAttr
389xegpu::getTemporaryLayout<mlir::OpOperand>(const OpOperand &operand);
390
391template <typename T, typename>
392void xegpu::setTemporaryLayout(const T &operandOrResult,
393 const xegpu::DistributeLayoutAttr layout) {
394 Operation *owner = operandOrResult.getOwner();
395 std::string name = xegpu::getTemporaryLayoutName(operandOrResult);
396 if (owner->hasDiscardableAttrOfType<xegpu::DistributeLayoutAttr>(name)) {
397 return;
398 }
399 if (layout) {
400 owner->setDiscardableAttr(name, value: layout);
401 }
402}
403
404template void xegpu::setTemporaryLayout<mlir::OpResult>(
405 const mlir::OpResult &result,
406 const mlir::xegpu::DistributeLayoutAttr layout);
407
408template void xegpu::setTemporaryLayout<mlir::OpOperand>(
409 const mlir::OpOperand &operand,
410 const mlir::xegpu::DistributeLayoutAttr layout);
411
412SmallVector<Value>
413xegpu::extractVectorsWithShapeFromValue(OpBuilder &builder, Location loc,
414 Value value, ArrayRef<int64_t> shape) {
415 auto vecTy = dyn_cast<VectorType>(Val: value.getType());
416 if (!vecTy)
417 return {value};
418
419 ArrayRef<int64_t> srcShape = vecTy.getShape();
420 if (!computeShapeRatio(shape: srcShape, subShape: shape))
421 return {value};
422
423 int64_t srcShapeRank = srcShape.size();
424 int64_t targetShapeRank = shape.size();
425
426 SmallVector<int64_t> adjustedTargetShape(srcShape.size());
427 int64_t rankDiff = srcShapeRank - targetShapeRank;
428 std::fill(first: adjustedTargetShape.begin(), last: adjustedTargetShape.begin() + rankDiff,
429 value: 1);
430 llvm::copy(Range&: shape, Out: adjustedTargetShape.begin() + rankDiff);
431
432 SmallVector<Value> result;
433 for (SmallVector<int64_t> offsets :
434 StaticTileOffsetRange(srcShape, adjustedTargetShape)) {
435 SmallVector<int64_t> staticStrides(offsets.size(), 1);
436 Value slice = vector::ExtractStridedSliceOp::create(
437 builder, location: loc, source: value, offsets, sizes: adjustedTargetShape, strides: staticStrides);
438
439 // Reshape to remove leading unit dims if needed
440 if (srcShapeRank > targetShapeRank) {
441 auto targetTy = VectorType::get(shape, elementType: vecTy.getElementType());
442 slice = vector::ShapeCastOp::create(builder, location: loc, result: targetTy, source: slice);
443 }
444 result.push_back(Elt: slice);
445 }
446
447 return result;
448}
449
450Value xegpu::createVectorWithShapeFromValues(OpBuilder &builder, Location loc,
451 ValueRange values,
452 ArrayRef<int64_t> shape) {
453 VectorType inputTy = dyn_cast<VectorType>(Val: values[0].getType());
454 assert(llvm::all_of(values.getTypes(),
455 [&](Type type) { return type == inputTy; }) &&
456 "values must be of the same VectorType");
457
458 Type elemTy = inputTy.getElementType();
459 ArrayRef<int64_t> tileShape = inputTy.getShape();
460
461 VectorType resultTy = VectorType::get(shape, elementType: elemTy);
462 auto zeroAttr = builder.getZeroAttr(type: elemTy);
463 Value result = arith::ConstantOp::create(
464 builder, location: loc, result: resultTy, value: DenseElementsAttr::get(type: resultTy, values: zeroAttr));
465
466 for (auto [src, offsets] :
467 llvm::zip_equal(t&: values, u: StaticTileOffsetRange(shape, tileShape))) {
468 SmallVector<int64_t> staticStrides(tileShape.size(), 1);
469 result = vector::InsertStridedSliceOp::create(builder, location: loc, valueToStore: src, dest: result,
470 offsets, strides: staticStrides);
471 }
472 return result;
473}
474
475std::optional<std::string> xegpu::getChipStr(Operation *op) {
476 auto gpuModuleOp = op->getParentOfType<gpu::GPUModuleOp>();
477
478 if (!gpuModuleOp)
479 return std::nullopt;
480
481 auto targetAttrs = gpuModuleOp.getTargets();
482 if (targetAttrs) {
483 for (auto &attr : *targetAttrs) {
484 auto xevmAttr = llvm::dyn_cast<xevm::XeVMTargetAttr>(Val: attr);
485 if (xevmAttr)
486 return xevmAttr.getChip().str();
487 }
488 }
489
490 return std::nullopt;
491}
492
493FailureOr<int64_t> xegpu::getNumSubgroupsFromBlockSize(Operation *op,
494 int64_t subgroupSize) {
495 auto gpuFunc = op->getParentOfType<gpu::GPUFuncOp>();
496 if (!gpuFunc)
497 return failure();
498 std::optional<ArrayRef<int32_t>> blockSize = gpuFunc.getKnownBlockSize();
499 if (!blockSize)
500 return failure();
501 if (!llvm::all_of(Range&: *blockSize, P: [](int32_t dim) {
502 return dim > 0 && llvm::isPowerOf2_32(Value: dim);
503 }))
504 return failure();
505 int64_t numSubgroups = llvm::product_of(Range&: *blockSize) / subgroupSize;
506 if (numSubgroups < 1)
507 return failure();
508 return numSubgroups;
509}
510
511/// Generates element-wise addition ops of two arrays with same length.
512SmallVector<OpFoldResult> xegpu::addElementwise(OpBuilder &builder,
513 Location loc,
514 ArrayRef<OpFoldResult> lhs,
515 ArrayRef<OpFoldResult> rhs) {
516 assert(lhs.size() == rhs.size() && "lhs and rhs must have the same size");
517 SmallVector<OpFoldResult> results;
518 for (auto [l, r] : llvm::zip_equal(t&: lhs, u&: rhs)) {
519 auto lval = getValueOrCreateConstantIndexOp(b&: builder, loc, ofr: l);
520 auto rval = getValueOrCreateConstantIndexOp(b&: builder, loc, ofr: r);
521 results.push_back(Elt: builder.createOrFold<arith::AddIOp>(location: loc, args&: lval, args&: rval));
522 }
523 return results;
524}
525
526/// Generates element-wise addition ops of two arrays with automatic alignment.
527/// When the input arrays have different sizes, the shorter array is
528/// right-aligned with the longer array, and the unmatched leading elements from
529/// the longer array are preserved unchanged. This is commonly used for offset
530/// computation where higher-dimensional offsets need to be added to
531/// lower-dimensional adjustments.
532///
533/// Example:
534/// lhs = [l1, l2, l3], rhs = [r1, r2]
535/// Result: [11, l2+r1, l3+r2]
536SmallVector<OpFoldResult>
537xegpu::addWithRightAligned(OpBuilder &builder, Location loc,
538 ArrayRef<OpFoldResult> lhs,
539 ArrayRef<OpFoldResult> rhs) {
540 // ensure a is longer than b
541 ArrayRef<OpFoldResult> a = lhs.size() >= rhs.size() ? lhs : rhs;
542 ArrayRef<OpFoldResult> b = lhs.size() >= rhs.size() ? rhs : lhs;
543 SmallVector<OpFoldResult> results(a.take_front(N: a.size() - b.size()));
544 a = a.slice(N: a.size() - b.size());
545 results.append(RHS: addElementwise(builder, loc, lhs: a, rhs: b));
546 return results;
547}
548
549template <typename T>
550int xegpu::getLargestDivisor(T dim, ArrayRef<T> candidates,
551 ArrayRef<T> candidateMultiples) {
552 static_assert(std::is_integral<T>::value, "T must be an integer type");
553 int largest = -1;
554 SmallVector<T> multiples = {1};
555 if (!candidateMultiples.empty())
556 multiples =
557 SmallVector<T>(candidateMultiples.begin(), candidateMultiples.end());
558 for (T candidate : candidates) {
559 for (T multiple : multiples) {
560 int value = static_cast<int>(candidate * multiple);
561 if (value != 0 && dim % value == 0 && value > largest)
562 largest = value;
563 }
564 }
565 return largest;
566}
567
568/// Returns the flags held by `fastmath`, or none if it is null.
569static arith::FastMathFlags getFlags(arith::FastMathFlagsAttr fastmath) {
570 return fastmath ? fastmath.getValue() : arith::FastMathFlags::none;
571}
572
573Value xegpu::subgroupReduction(Location loc, OpBuilder &builder, Value input,
574 vector::CombiningKind kind, uint32_t size,
575 arith::FastMathFlagsAttr fastmath) {
576 // First reduce on a single thread to get per lane reduction value.
577 Value laneVal = vector::ReductionOp::create(builder, location: loc, kind, vector: input,
578 fastMathFlags: getFlags(fastmath));
579 // Parallel reduction using butterfly shuffles.
580 for (uint64_t i = 1; i < size; i <<= 1) {
581 Value shuffled =
582 gpu::ShuffleOp::create(builder, location: loc, value: laneVal, offset: i, /** width = **/ width: size,
583 /** mode = **/ mode: gpu::ShuffleMode::XOR)
584 .getShuffleResult();
585 laneVal =
586 makeArithReduction(b&: builder, loc, kind, v1: laneVal, acc: shuffled, fastmath);
587 }
588 return laneVal;
589}
590
591Value xegpu::lowerToVectorReductions(TypedValue<VectorType> src,
592 TypedValue<VectorType> acc,
593 vector::CombiningKind kind,
594 int64_t reductionDim, Location loc,
595 PatternRewriter &rewriter,
596 arith::FastMathFlagsAttr fastmath) {
597 VectorType sourceType = src.getType();
598 int64_t sourceRank = sourceType.getRank();
599 // Expecting at least a 2D source vector. Leading dimensions (all except the
600 // last two) must be unit.
601 assert(sourceRank >= 2 && "expected at least a 2D source vector");
602 for (int64_t i = 0; i < sourceRank - 2; ++i)
603 assert(sourceType.getShape()[i] == 1 &&
604 "expected leading dimensions to be unit");
605 int64_t rowIdx = sourceRank - 2;
606 int64_t columnIdx = sourceRank - 1;
607 int64_t sourceH = sourceType.getShape()[rowIdx];
608 int64_t sourceW = sourceType.getShape()[columnIdx];
609 int nSlices = (reductionDim == rowIdx) ? sourceW : sourceH;
610 // Create a constant vector to hold the result of the reduction.
611 TypedAttr zeroAttr = rewriter.getZeroAttr(type: sourceType.getElementType());
612 Value reductionResult = arith::ConstantOp::create(
613 builder&: rewriter, location: loc, result: acc.getType(),
614 value: DenseElementsAttr::get(type: acc.getType(), values: zeroAttr));
615 auto srcLayout = xegpu::getTemporaryLayout(operandOrResult: dyn_cast<OpResult>(Val&: src));
616 auto accLayout = xegpu::getTemporaryLayout(operandOrResult: dyn_cast<OpResult>(Val&: acc));
617 // Reduction result should have the same layout as the accumulator.
618 xegpu::setTemporaryLayout(operandOrResult: cast<OpResult>(Val&: reductionResult), layout: accLayout);
619 // For each slice of the source, extract the slice vector, do a reduction
620 // and, insert the reduced value back to the result vector.
621 int64_t accRank = acc.getType().getRank();
622 for (int i = 0; i < nSlices; ++i) {
623 // Build nD offsets, sizes, and strides. Leading unit dims get
624 // offset=0, size=1. The last two dims are set based on reductionDim.
625 SmallVector<int64_t> sliceOffsets(sourceRank, 0);
626 SmallVector<int64_t> sliceSizes(sourceRank, 1);
627 SmallVector<int64_t> strides(sourceRank, 1);
628 if (reductionDim == columnIdx) {
629 sliceOffsets[rowIdx] = i;
630 sliceSizes[columnIdx] = sourceW;
631 } else {
632 sliceOffsets[columnIdx] = i;
633 sliceSizes[rowIdx] = sourceH;
634 }
635
636 vector::ExtractStridedSliceOp extractOp =
637 vector::ExtractStridedSliceOp::create(builder&: rewriter, location: loc, source: src, offsets: sliceOffsets,
638 sizes: sliceSizes, strides);
639 // Extract strided slice has the same layout as src.
640 xegpu::setTemporaryLayout(operandOrResult: extractOp->getOpResult(idx: 0), layout: srcLayout);
641
642 int64_t nSliceElements = extractOp.getResult().getType().getNumElements();
643
644 vector::ShapeCastOp slice = vector::ShapeCastOp::create(
645 builder&: rewriter, location: loc,
646 result: VectorType::get(shape: {nSliceElements}, elementType: sourceType.getElementType()),
647 source: extractOp.getResult());
648
649 // Shape cast output has the same layout as the accumulator. Shape cast
650 // source has the same layout as the original reduction source.
651 xegpu::setTemporaryLayout(operandOrResult: slice->getOpOperand(idx: 0), layout: srcLayout);
652 xegpu::setTemporaryLayout(operandOrResult: slice->getOpResult(idx: 0), layout: accLayout);
653 // Extract and reduction results in scalars, so no result layout is needed.
654 // Build multi-dim index into acc (sourceRank-1 dims, i.e. source shape with
655 // the reduction dim removed). Leading unit dims get index 0.
656 SmallVector<int64_t> accIdx(accRank, 0);
657 accIdx[accRank - 1] = i;
658 Value accExtract = vector::ExtractOp::create(builder&: rewriter, location: loc, source: acc, position: accIdx);
659 Value reduction = vector::ReductionOp::create(
660 builder&: rewriter, location: loc, kind, vector: slice.getResult(), acc: accExtract, fastMathFlags: getFlags(fastmath));
661 reductionResult = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: reduction,
662 dest: reductionResult, position: accIdx);
663 // Insert op should have the same layout as the accumulator.
664 xegpu::setTemporaryLayout(operandOrResult: cast<OpResult>(Val&: reductionResult), layout: accLayout);
665 }
666 return reductionResult;
667}
668
669Value xegpu::lowerCrossLaneReductionToShuffles(
670 TypedValue<VectorType> src, TypedValue<VectorType> acc,
671 vector::CombiningKind kind, int64_t reductionDim, int64_t reductionSize,
672 Location loc, PatternRewriter &rewriter,
673 arith::FastMathFlagsAttr fastmath) {
674 VectorType sourceType = src.getType();
675 int64_t sourceRank = sourceType.getRank();
676 // Expecting at least a 2D source vector. Leading dimensions (all except the
677 // last two) must be unit.
678 assert(sourceRank >= 2 && "expected at least a 2D source vector");
679 for (int64_t i = 0; i < sourceRank - 2; ++i)
680 assert(sourceType.getShape()[i] == 1 &&
681 "expected leading dimensions to be unit");
682 int64_t rowIdx = sourceRank - 2;
683 int64_t columnIdx = sourceRank - 1;
684 int64_t sourceH = sourceType.getShape()[rowIdx];
685 int64_t sourceW = sourceType.getShape()[columnIdx];
686
687 // Create a constant vector to hold the result of the reduction.
688 TypedAttr zeroAttr = rewriter.getZeroAttr(type: sourceType.getElementType());
689 Value reductionResult = arith::ConstantOp::create(
690 builder&: rewriter, location: loc, result: acc.getType(),
691 value: DenseElementsAttr::get(type: acc.getType(), values: zeroAttr));
692
693 // nSlices is the number of reduction operations needed to reduce the entire
694 // source vector. For example, if reductionDim is the row dim, we are
695 // reducing across rows, and each slice is a column. So the number of slices
696 // is the number of columns, which is sourceW.
697 int nSlices = (reductionDim == rowIdx) ? sourceW : sourceH;
698
699 // For each slice of the source, extract the slice vector, do a reduction
700 // and, insert the reduced value back to the result vector.
701 int64_t accRank = acc.getType().getRank();
702 for (int i = 0; i < nSlices; ++i) {
703 // Build nD offsets, sizes, and strides. Leading unit dims get
704 // offset=0, size=1. The last two dims are set based on reductionDim.
705 SmallVector<int64_t> sliceOffsets(sourceRank, 0);
706 SmallVector<int64_t> sliceSizes(sourceRank, 1);
707 SmallVector<int64_t> strides(sourceRank, 1);
708 if (reductionDim == columnIdx) {
709 sliceOffsets[rowIdx] = i;
710 sliceSizes[columnIdx] = sourceW;
711 } else {
712 sliceOffsets[columnIdx] = i;
713 sliceSizes[rowIdx] = sourceH;
714 }
715
716 vector::ExtractStridedSliceOp extractOp =
717 vector::ExtractStridedSliceOp::create(builder&: rewriter, location: loc, source: src, offsets: sliceOffsets,
718 sizes: sliceSizes, strides);
719 int64_t nSliceElements = extractOp.getResult().getType().getNumElements();
720 vector::ShapeCastOp slice = vector::ShapeCastOp::create(
721 builder&: rewriter, location: loc,
722 result: VectorType::get(shape: {nSliceElements}, elementType: sourceType.getElementType()),
723 source: extractOp.getResult());
724
725 SmallVector<int64_t> accIdx(accRank, 0);
726 accIdx[accRank - 1] = i;
727 Value accExtract = vector::ExtractOp::create(builder&: rewriter, location: loc, source: acc, position: accIdx);
728 Value fullReduce = xegpu::subgroupReduction(loc, builder&: rewriter, input: slice, kind,
729 size: reductionSize, fastmath);
730 fullReduce = vector::makeArithReduction(b&: rewriter, loc, kind, v1: fullReduce,
731 acc: accExtract, fastmath);
732 reductionResult = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: fullReduce,
733 dest: reductionResult, position: accIdx);
734 }
735 return reductionResult;
736}
737
738/// Returns the neutral value of the float min or max reduction `kind` over
739/// elements of `semantics`. The reduction combines this value under its
740/// fast-math `flags`, so build only a value that the format can represent and
741/// the flags allow:
742/// - A kind that ignores NaN operands uses a quiet NaN, if the format has one
743/// and `nnan` is not set.
744/// - Otherwise it uses the infinity that never wins, if the format has one and
745/// `ninf` is not set.
746/// - Otherwise it uses the lowest (for max) or highest (for min) finite value.
747/// That value never wins either: no number that the format and the flags
748/// allow is below the lowest or above the highest finite value.
749static APFloat getMinMaxNeutral(const llvm::fltSemantics &semantics,
750 vector::CombiningKind kind,
751 arith::FastMathFlags flags) {
752 bool isMax = kind == vector::CombiningKind::MAXIMUMF ||
753 kind == vector::CombiningKind::MAXNUMF ||
754 kind == vector::CombiningKind::MAXIMUMNUMF;
755 bool ignoresNaN = kind != vector::CombiningKind::MINIMUMF &&
756 kind != vector::CombiningKind::MAXIMUMF;
757 if (ignoresNaN && APFloat::semanticsHasNaN(semantics) &&
758 !arith::bitEnumContainsAny(bits: flags, bit: arith::FastMathFlags::nnan))
759 return APFloat::getQNaN(Sem: semantics);
760 bool useInf = APFloat::semanticsHasInf(semantics) &&
761 !arith::bitEnumContainsAny(bits: flags, bit: arith::FastMathFlags::ninf);
762 if (!isMax)
763 return useInf ? APFloat::getInf(Sem: semantics) : APFloat::getLargest(Sem: semantics);
764 // An unsigned format has no negative values. Its lowest value is zero, or
765 // its smallest positive value if it has no zero.
766 if (!APFloat::semanticsHasSignedRepr(semantics))
767 return APFloat::semanticsHasZero(semantics)
768 ? APFloat::getZero(Sem: semantics)
769 : APFloat::getSmallest(Sem: semantics);
770 return useInf ? APFloat::getInf(Sem: semantics, /*Negative=*/true)
771 : APFloat::getLargest(Sem: semantics, /*Negative=*/true);
772}
773
774Value xegpu::createReductionNeutralValue(OpBuilder &builder, Location loc,
775 Type type, vector::CombiningKind kind,
776 arith::FastMathFlagsAttr fastmath) {
777 auto vecTy = dyn_cast<VectorType>(Val&: type);
778 Type elemTy = vecTy ? vecTy.getElementType() : type;
779
780 // Helper to create either a splat vector or scalar constant from an attr.
781 auto makeConst = [&](Attribute scalarAttr) -> Value {
782 if (vecTy)
783 return arith::ConstantOp::create(
784 builder, location: loc, result: vecTy, value: DenseElementsAttr::get(type: vecTy, values: scalarAttr));
785 return arith::ConstantOp::create(builder, location: loc, value: cast<TypedAttr>(Val&: scalarAttr));
786 };
787
788 switch (kind) {
789 case vector::CombiningKind::ADD:
790 case vector::CombiningKind::XOR:
791 case vector::CombiningKind::OR:
792 case vector::CombiningKind::MAXUI:
793 return makeConst(builder.getZeroAttr(type: elemTy));
794
795 case vector::CombiningKind::MUL:
796 return makeConst(builder.getOneAttr(type: elemTy));
797
798 case vector::CombiningKind::AND:
799 if (auto intTy = dyn_cast<IntegerType>(Val&: elemTy))
800 return makeConst(
801 builder.getIntegerAttr(type: elemTy, value: APInt::getAllOnes(numBits: intTy.getWidth())));
802 if (elemTy.isIndex())
803 return makeConst(builder.getIndexAttr(value: -1));
804 return nullptr;
805
806 case vector::CombiningKind::MINSI:
807 if (auto intTy = dyn_cast<IntegerType>(Val&: elemTy))
808 return makeConst(builder.getIntegerAttr(
809 type: elemTy, value: APInt::getSignedMaxValue(numBits: intTy.getWidth())));
810 return nullptr;
811
812 case vector::CombiningKind::MINUI:
813 if (auto intTy = dyn_cast<IntegerType>(Val&: elemTy))
814 return makeConst(
815 builder.getIntegerAttr(type: elemTy, value: APInt::getMaxValue(numBits: intTy.getWidth())));
816 return nullptr;
817
818 case vector::CombiningKind::MAXSI:
819 if (auto intTy = dyn_cast<IntegerType>(Val&: elemTy))
820 return makeConst(builder.getIntegerAttr(
821 type: elemTy, value: APInt::getSignedMinValue(numBits: intTy.getWidth())));
822 return nullptr;
823
824 case vector::CombiningKind::MINIMUMF:
825 case vector::CombiningKind::MAXIMUMF:
826 case vector::CombiningKind::MINNUMF:
827 case vector::CombiningKind::MINIMUMNUMF:
828 case vector::CombiningKind::MAXNUMF:
829 case vector::CombiningKind::MAXIMUMNUMF:
830 if (auto floatTy = dyn_cast<FloatType>(Val&: elemTy))
831 return makeConst(builder.getFloatAttr(
832 type: elemTy, value: getMinMaxNeutral(semantics: floatTy.getFloatSemantics(), kind,
833 flags: getFlags(fastmath))));
834 return nullptr;
835 }
836 return nullptr;
837}
838
839/// Explicit instantiations
840template int xegpu::getLargestDivisor<int>(int dim, ArrayRef<int> candidates,
841 ArrayRef<int> candidateMultiples);
842template int
843xegpu::getLargestDivisor<unsigned>(unsigned dim, ArrayRef<unsigned> candidates,
844 ArrayRef<unsigned> candidateMultiples);
845
846std::optional<SmallVector<int64_t>>
847xegpu::getInner2DIfUnitLeadingDims(ArrayRef<int64_t> vals) {
848 if (vals.size() < 2)
849 return std::nullopt;
850 if (llvm::any_of(Range: vals.drop_back(N: 2), P: [](int64_t v) { return v != 1; }))
851 return std::nullopt;
852 return SmallVector<int64_t>(vals.take_back(N: 2));
853}
854
855bool xegpu::requirePacked(const xegpu::DistributeLayoutAttr layout) {
856 if (!layout)
857 return false;
858 auto laneData =
859 getInner2DIfUnitLeadingDims(vals: layout.getEffectiveLaneDataAsInt());
860 return laneData && (*laneData)[0] != 1;
861}
862
863bool xegpu::requireTranspose(const xegpu::DistributeLayoutAttr layout,
864 const xegpu::uArch::uArch *uArch) {
865 // Return false for unsupported targets.
866 // TODO: Add more support or move to target info.
867 if (!isa<xegpu::uArch::Xe2>(Val: uArch) && !isa<xegpu::uArch::Xe3p>(Val: uArch))
868 return false;
869 if (!layout)
870 return false;
871 auto laneLayout =
872 getInner2DIfUnitLeadingDims(vals: layout.getEffectiveLaneLayoutAsInt());
873 return laneLayout && (*laneLayout)[0] == uArch->getSubgroupSize() &&
874 (*laneLayout)[1] == 1;
875}
876
877bool xegpu::hasStaticShapeAndStrides(MemRefType type) {
878 if (!type.hasStaticShape())
879 return false;
880 SmallVector<int64_t> strides;
881 int64_t offset;
882 return succeeded(Result: type.getStridesAndOffset(strides, offset)) &&
883 llvm::none_of(Range&: strides, P: ShapedType::isDynamic);
884}
885
886Value xegpu::addElemOffsetInBytes(OpBuilder &builder, Location loc,
887 Value baseAddr, Value elemOffset,
888 MemRefType memrefTy) {
889 Type type = elemOffset.getType();
890 assert(baseAddr.getType() == type &&
891 "expected base address and offset of the same type");
892 auto makeConst = [&](int64_t value) -> Value {
893 return arith::ConstantOp::create(builder, location: loc, result: type,
894 value: builder.getIntegerAttr(type, value));
895 };
896
897 // finalAddr = baseAddr + elemOffset * bitWidth / 8
898 unsigned elemBits = memrefTy.getElementTypeBitWidth();
899 Value byteOffset;
900 if (elemBits % 8 == 0) {
901 // For byte-sized elements divide by 8 statically so that the division is
902 // folded into the constant:
903 // finalAddr = baseAddr + elemOffset * (bitWidth / 8)
904 byteOffset = arith::MulIOp::create(builder, location: loc, lhs: elemOffset,
905 rhs: makeConst(elemBits / 8));
906 } else {
907 // For sub-byte elements multiply first and then divide by 8, otherwise the
908 // byte size would be zero (e.g. 4 / 8 == 0 for i4):
909 // finalAddr = baseAddr + (elemOffset * bitWidth) / 8
910 Value bitSize = makeConst(elemBits);
911 Value eight = makeConst(8);
912 byteOffset = arith::DivUIOp::create(
913 builder, location: loc, lhs: arith::MulIOp::create(builder, location: loc, lhs: elemOffset, rhs: bitSize),
914 rhs: eight);
915 }
916 return arith::AddIOp::create(builder, location: loc, lhs: baseAddr, rhs: byteOffset);
917}
918
919// Check if dst shape is an expansion of src shape by inserting unit dimensions.
920// Returns true if all dimensions in src match corresponding dimensions in dst
921// (after skipping unit dimensions), and populates expandedUnitDims with the
922// indices of the unit dimensions in dst that were added (not present in src).
923// Example: src=[2,3], dst=[1,2,3,1] -> true, expandedUnitDims=[0,3]
924bool xegpu::matchUnitDimExpansion(ArrayRef<int64_t> src, ArrayRef<int64_t> dst,
925 SmallVector<int64_t> &expandedUnitDims) {
926 // All unit dimensions in dst that don't appear in src are the expanded
927 // unit dimensions
928 size_t srcIdx = 0;
929 for (size_t dstIdx = 0; dstIdx < dst.size(); ++dstIdx)
930 if (srcIdx < src.size() && src[srcIdx] == dst[dstIdx])
931 srcIdx++;
932 else if (dst[dstIdx] == 1)
933 expandedUnitDims.push_back(Elt: dstIdx);
934 else
935 return false;
936 return srcIdx == src.size();
937}
938
939// Checks if dst shape is an expansion of src shape where each dimension in src
940// is split into one or more consecutive dimensions in dst whose product equals
941// the original dimension. Populates splitDimGroups with groups of dst indices
942// that correspond to each src dimension. Example: src=[6,4], dst=[2,3,2,2] ->
943// true. Shapes that only insert unit dimensions are rejected here.
944bool xegpu::matchSplitDimExpansion(
945 ArrayRef<int64_t> src, ArrayRef<int64_t> dst,
946 SmallVector<SmallVector<int64_t>> &splitDimGroups) {
947 SmallVector<int64_t> expandedUnitDims;
948 if (matchUnitDimExpansion(src, dst, expandedUnitDims) &&
949 !expandedUnitDims.empty())
950 return false;
951
952 // each dim in src can be mapped to one or more dims in dst whose product
953 // equals to the src dim
954 size_t srcIdx = 0;
955 int64_t accumulatedSize = 1;
956 SmallVector<int64_t> currentDstDims;
957
958 splitDimGroups.clear();
959 for (size_t dstIdx = 0; dstIdx < dst.size(); ++dstIdx) {
960 if (srcIdx >= src.size())
961 return false;
962 accumulatedSize *= dst[dstIdx];
963 currentDstDims.push_back(Elt: dstIdx);
964
965 if (accumulatedSize == src[srcIdx]) {
966 // Also collect trailing unit dims in destination, if any.
967 // Leading unit dims were implicitly collected.
968 if (srcIdx == src.size() - 1) {
969 while (++dstIdx < dst.size() && dst[dstIdx] == 1)
970 currentDstDims.push_back(Elt: dstIdx);
971 }
972 // Record the mapping: srcIdx -> currentDstDims
973 splitDimGroups.push_back(Elt: currentDstDims);
974 // move to next src dim
975 srcIdx++;
976 accumulatedSize = 1;
977 currentDstDims.clear();
978 } else if (accumulatedSize > src[srcIdx]) {
979 return false;
980 }
981 }
982 return srcIdx == src.size();
983}
984
985//===----------------------------------------------------------------------===//
986// Context-aware type conversion utilities
987//===----------------------------------------------------------------------===//
988
989// Pre-computes distributed VectorType mappings for every value carried through
990// an SCF region-branch op (scf.while, scf.for, scf.if): block args (iter_args /
991// before-/after-args), op results, and the terminator operands feeding them.
992// These positions share one logical value and must convert identically, so each
993// is derived from a single source -- the layout of the feeding value (loop
994// init, `scf.condition` operand, or `scf.if` result) -- via
995// `getDistributeLayoutAttr(Value)`, and keyed by `Value`. Keying by Value is
996// required because the SCF converters
997// detach/replace the loop body mid-conversion (scf.while detaches before/after
998// blocks -> a detached-arg layout query trips an ilist assertion; scf.for
999// rebuilds the op, which loses the temporary `layout_operand_N` attrs -> the
1000// query returns null). Recording results and terminator operands lets a 1:N
1001// pass resolve them from the map after stripping the loop op's transient attrs
1002// (see XeGPUBlocking).
1003DenseMap<Value, SmallVector<Type>>
1004xegpu::precomputeLoopBlockArgTypes(Operation *topLevelOp,
1005 SubShapeAndCountFn getSubShapeAndCount) {
1006 DenseMap<Value, SmallVector<Type>> loopArgTypes;
1007 // Derive the distributed types from the feeding value's layout (the single
1008 // authoritative source) and record them for every value that shares this
1009 // loop-carried position.
1010 auto recordTypes = [&](Value layoutSrc, ArrayRef<Value> dests) {
1011 auto vecTy = dyn_cast<VectorType>(Val: layoutSrc.getType());
1012 if (!vecTy)
1013 return;
1014 auto layout = xegpu::getDistributeLayoutAttr(value: layoutSrc);
1015 if (!layout)
1016 return;
1017 auto [subShape, count] = getSubShapeAndCount(vecTy, layout);
1018 if (count <= 0)
1019 return;
1020 auto newTy = VectorType::get(shape: subShape, elementType: vecTy.getElementType());
1021 for (Value dest : dests)
1022 loopArgTypes[dest] = SmallVector<Type>(count, newTy);
1023 };
1024 topLevelOp->walk(callback: [&](Operation *op) {
1025 if (auto whileOp = dyn_cast<scf::WhileOp>(Val: op)) {
1026 // "before" args (and the after-region yield operands that feed them)
1027 // correspond to the while `inits` operands.
1028 auto yieldOp =
1029 cast<scf::YieldOp>(Val: whileOp.getAfterBody()->getTerminator());
1030 for (auto [init, beforeArg, yieldVal] :
1031 llvm::zip(t: whileOp.getInits(), u: whileOp.getBeforeArguments(),
1032 args: yieldOp.getOperands()))
1033 recordTypes(init, {beforeArg, yieldVal});
1034 // "after" args and the while results correspond to the operands of the
1035 // embedded `scf.condition` op (not the `inits`).
1036 scf::ConditionOp condOp = whileOp.getConditionOp();
1037 for (auto [condArg, afterArg, res] :
1038 llvm::zip(t: condOp.getArgs(), u: whileOp.getAfterArguments(),
1039 args: whileOp.getResults()))
1040 recordTypes(condArg, {afterArg, res});
1041 return;
1042 }
1043 if (auto forOp = dyn_cast<scf::ForOp>(Val: op)) {
1044 // Each loop-carried position pairs an init operand with its iter_arg,
1045 // its loop result, and the yield operand that feeds the next iteration.
1046 auto yieldOp = cast<scf::YieldOp>(Val: forOp.getBody()->getTerminator());
1047 for (auto [init, arg, res, yieldVal] :
1048 llvm::zip(t: forOp.getInitArgs(), u: forOp.getRegionIterArgs(),
1049 args: forOp.getResults(), args: yieldOp.getOperands()))
1050 recordTypes(init, {arg, res, yieldVal});
1051 return;
1052 }
1053 if (auto ifOp = dyn_cast<scf::IfOp>(Val: op)) {
1054 // Each result and its then/else yield operands share one position and
1055 // must convert identically; derive all from the result's layout.
1056 scf::YieldOp thenYield = ifOp.thenYield();
1057 scf::YieldOp elseYield = ifOp.elseBlock() ? ifOp.elseYield() : nullptr;
1058 for (auto [idx, res] : llvm::enumerate(First: ifOp.getResults())) {
1059 SmallVector<Value> dests{res, thenYield.getOperand(i: idx)};
1060 if (elseYield)
1061 dests.push_back(Elt: elseYield.getOperand(i: idx));
1062 recordTypes(res, dests);
1063 }
1064 return;
1065 }
1066 });
1067 return loopArgTypes;
1068}
1069
1070void xegpu::addVectorTypeConversion(
1071 TypeConverter &converter, SubShapeAndCountFn getSubShapeAndCount,
1072 DenseMap<Value, SmallVector<Type>> loopArgTypes) {
1073 // Context-aware VectorType conversion (1:1 shape-changing or 1:N). For
1074 // SCF loop block arguments (scf.while, scf.for), uses the pre-computed
1075 // map. For all other Values, retrieves the layout directly via
1076 // getDistributeLayoutAttr.
1077 auto loopArgTypeMap = std::make_shared<DenseMap<Value, SmallVector<Type>>>(
1078 args: std::move(loopArgTypes));
1079 converter.addConversion(
1080 callback: [loopArgTypeMap, getSubShapeAndCount](
1081 Value v,
1082 SmallVectorImpl<Type> &result) -> std::optional<LogicalResult> {
1083 if (!isa<VectorType>(Val: v.getType()))
1084 return std::nullopt;
1085
1086 // Check the pre-computed map first. It covers every value carried
1087 // through an SCF loop (operands, block args, results, yield
1088 // operands), all keyed by Value identity.
1089 auto it = loopArgTypeMap->find(Val: v);
1090 if (it != loopArgTypeMap->end()) {
1091 result.append(in_start: it->second.begin(), in_end: it->second.end());
1092 return success();
1093 }
1094
1095 // For all other Values, retrieve the layout directly.
1096 auto layout = xegpu::getDistributeLayoutAttr(value: v);
1097 if (!layout)
1098 return std::nullopt;
1099
1100 auto vecType = cast<VectorType>(Val: v.getType());
1101 auto [subShape, count] = getSubShapeAndCount(vecType, layout);
1102 if (count <= 0)
1103 return std::nullopt;
1104
1105 auto newTy = VectorType::get(shape: subShape, elementType: vecType.getElementType());
1106 result.append(NumInputs: count, Elt: newTy);
1107 return success();
1108 });
1109}
1110
1111void xegpu::cleanupUnrealizedConversionCasts(
1112 Operation *root,
1113 const llvm::SmallSetVector<UnrealizedConversionCastOp, 8> &existingCasts) {
1114 // Structural type conversion can generate some redundant
1115 // UnrealizedConversionCastOps to materialize the original type from the
1116 // type converted (sub-tile) type. These are redundant at this point and
1117 // can be eliminated by either folding the cancelling cast chain or, when
1118 // the original and final shapes differ but their element counts match,
1119 // inserting a vector.shape_cast instead.
1120 //
1121 // Example (shape differs but element count matches -> shape_cast):
1122 // %1 = UnrealizedConversionCastOp %0 : vector<16x1xf32>
1123 // to vector<16x16xf32>
1124 // %2 = UnrealizedConversionCastOp %1 : vector<16x16xf32>
1125 // to vector<16xf32>
1126 // becomes:
1127 // %2 = vector.shape_cast %0 : vector<16x1xf32> to vector<16xf32>
1128 //
1129 // For unpaired casts that emulate a pack (1:N) or unpack (N:1) between a
1130 // single large VectorType and N identically-typed smaller VectorTypes,
1131 // lower to vector.extract_strided_slice / vector.insert_strided_slice.
1132 auto hasIdenticalVectorTypes = [](ValueRange values) {
1133 auto types = values.getTypes();
1134 return !types.empty() && llvm::all_of(Range&: types, P: [&](Type type) {
1135 return isa<VectorType>(Val: type) && type == types.front();
1136 });
1137 };
1138 OpBuilder builder(root);
1139 root->walk(callback: [&](UnrealizedConversionCastOp op) {
1140 if (existingCasts.contains(key: op))
1141 return;
1142 // Handle N:1 cast (N >= 1) where all inputs come from a single 1:N cast.
1143 if (op.getNumResults() == 1 && op.getNumOperands() >= 1) {
1144 auto defOp =
1145 op.getInputs()[0].getDefiningOp<UnrealizedConversionCastOp>();
1146 if (defOp && !existingCasts.contains(key: defOp) &&
1147 defOp.getNumOperands() == 1 &&
1148 defOp.getNumResults() == op.getNumOperands() &&
1149 llvm::all_of(Range: op.getInputs(),
1150 P: [&](Value v) { return v.getDefiningOp() == defOp; })) {
1151 Value orig = defOp.getInputs()[0];
1152 auto origTy = dyn_cast<VectorType>(Val: orig.getType());
1153 auto resTy = dyn_cast<VectorType>(Val: op.getResult(i: 0).getType());
1154 if (origTy && resTy &&
1155 origTy.getNumElements() == resTy.getNumElements() &&
1156 origTy != resTy) {
1157 builder.setInsertionPoint(op);
1158 auto shapeCast =
1159 vector::ShapeCastOp::create(builder, location: op.getLoc(), result: resTy, source: orig);
1160 op.replaceAllUsesWith(values: ValueRange{shapeCast.getResult()});
1161 } else {
1162 op.replaceAllUsesWith(values: ValueRange{orig});
1163 }
1164 return;
1165 }
1166 // Unpaired N:1 cast emulating unpack: stitch inputs into the output
1167 // shape via vector.insert_strided_slice.
1168 auto outputTy = dyn_cast<VectorType>(Val: op.getResult(i: 0).getType());
1169 if (op.getNumOperands() > 1 && outputTy &&
1170 hasIdenticalVectorTypes(op.getInputs())) {
1171 builder.setInsertionPoint(op);
1172 Value result = xegpu::createVectorWithShapeFromValues(
1173 builder, loc: op.getLoc(), values: op.getInputs(), shape: outputTy.getShape());
1174 op->replaceAllUsesWith(values: ValueRange(result));
1175 }
1176 return;
1177 }
1178 // Handle 1:N cast where the single input comes from an N:1 cast.
1179 if (op.getNumOperands() == 1 && op.getNumResults() > 1) {
1180 auto defOp =
1181 op.getInputs()[0].getDefiningOp<UnrealizedConversionCastOp>();
1182 if (defOp && !existingCasts.contains(key: defOp) &&
1183 defOp.getNumResults() == 1 &&
1184 defOp.getNumOperands() == op.getNumResults() &&
1185 llvm::equal(LRange: ValueRange(defOp.getInputs()).getTypes(),
1186 RRange: op->getResultTypes())) {
1187 op.replaceAllUsesWith(values: defOp.getInputs());
1188 return;
1189 }
1190 // Unpaired 1:N cast emulating pack: split the input into the output
1191 // tile shape via vector.extract_strided_slice.
1192 auto tileTy = dyn_cast<VectorType>(Val: op.getResult(i: 0).getType());
1193 if (tileTy && hasIdenticalVectorTypes(op.getResults())) {
1194 builder.setInsertionPoint(op);
1195 SmallVector<Value> results = xegpu::extractVectorsWithShapeFromValue(
1196 builder, loc: op.getLoc(), value: op.getInputs()[0], shape: tileTy.getShape());
1197 op->replaceAllUsesWith(values&: results);
1198 }
1199 return;
1200 }
1201 });
1202
1203 // Erase dead casts iteratively.
1204 bool changed = true;
1205 while (changed) {
1206 changed = false;
1207 root->walk(callback: [&](UnrealizedConversionCastOp op) {
1208 if (existingCasts.contains(key: op))
1209 return;
1210 if (op.use_empty()) {
1211 op.erase();
1212 changed = true;
1213 }
1214 });
1215 }
1216}
1217
1218// Checks if dst shape is a collapse of src shape where each dim in dst is
1219// produced by one or more consecutive dims in src whose product equals the dst
1220// dim. Populates collapseDims with one group per dst dim listing the src
1221// indices collapsed into it. Unit dims in dst that have no backing src dim
1222// (leading, in-between, or trailing) get empty groups; src unit dims that
1223// fall past the last consumed dst dim are absorbed into the most-recent
1224// non-empty group.
1225// Examples:
1226// src=[8,16,32], dst=[1,4096] -> true, collapseDims=[[],[0,1,2]]
1227// src=[8,16,32], dst=[4096,1] -> true, collapseDims=[[0,1,2],[]]
1228// src=[2,3,4], dst=[6,4] -> true, collapseDims=[[0,1],[2]]
1229// src=[64], dst=[64] -> true, collapseDims=[[0]]
1230bool xegpu::matchDimCollapse(ArrayRef<int64_t> src, ArrayRef<int64_t> dst,
1231 SmallVector<SmallVector<int64_t>> &collapseDims) {
1232 collapseDims.clear();
1233 collapseDims.resize(N: dst.size());
1234
1235 // Cheap precondition: src and dst must describe the same number of
1236 // elements. Bails out early on mismatched shapes without walking the dims.
1237 int64_t srcProd = std::accumulate(first: src.begin(), last: src.end(), init: int64_t{1},
1238 binary_op: std::multiplies<int64_t>());
1239 int64_t dstProd = std::accumulate(first: dst.begin(), last: dst.end(), init: int64_t{1},
1240 binary_op: std::multiplies<int64_t>());
1241 if (srcProd != dstProd)
1242 return false;
1243
1244 // Step 1: validate the partition on the unit-dim-stripped (compact) shapes.
1245 // Unit dims play no role in the matching decision — they only need to be
1246 // placed somewhere in the final groups (handled in step 2).
1247 SmallVector<int64_t> srcCompact, dstCompact;
1248 for (int64_t s : src)
1249 if (s != 1)
1250 srcCompact.push_back(Elt: s);
1251 for (int64_t d : dst)
1252 if (d != 1)
1253 dstCompact.push_back(Elt: d);
1254
1255 size_t s = 0;
1256 for (int64_t need : dstCompact) {
1257 int64_t acc = 1;
1258 while (s < srcCompact.size() && acc < need)
1259 acc *= srcCompact[s++];
1260 if (acc != need)
1261 return false;
1262 }
1263 if (s != srcCompact.size())
1264 return false;
1265
1266 // Step 2: assign each original src index to the correct original dst group.
1267 // Walk dst in original order, advancing past unit dst dims (they keep their
1268 // pre-initialized empty group). Walk src in original order; non-unit src
1269 // dims accumulate into the current dst group, unit src dims attach to the
1270 // current group when one is open or to the most-recent non-empty group
1271 // after dst is exhausted (leading unit src dims with no group yet are
1272 // dropped).
1273 size_t dstIdx = 0;
1274 while (dstIdx < dst.size() && dst[dstIdx] == 1)
1275 dstIdx++;
1276
1277 int64_t lastNonEmpty = -1;
1278 int64_t acc = 1;
1279 for (size_t srcIdx = 0; srcIdx < src.size(); ++srcIdx) {
1280 if (dstIdx >= dst.size()) {
1281 // dst exhausted; remaining src dims are unit (validated above) and
1282 // attach to the last non-empty group, if any.
1283 if (lastNonEmpty >= 0)
1284 collapseDims[lastNonEmpty].push_back(Elt: srcIdx);
1285 continue;
1286 }
1287 acc *= src[srcIdx];
1288 collapseDims[dstIdx].push_back(Elt: srcIdx);
1289 lastNonEmpty = dstIdx;
1290 if (acc == dst[dstIdx]) {
1291 acc = 1;
1292 ++dstIdx;
1293 while (dstIdx < dst.size() && dst[dstIdx] == 1)
1294 ++dstIdx;
1295 }
1296 }
1297 return true;
1298}
1299