1//===- XeGPUSgToLaneDistribute.cpp - XeGPU SG to Lane Pass ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#include "mlir/Dialect/GPU/IR/GPUDialect.h"
9#include "mlir/Dialect/Index/IR/IndexDialect.h"
10#include "mlir/Dialect/Math/IR/Math.h"
11#include "mlir/Dialect/MemRef/IR/MemRef.h"
12#include "mlir/Dialect/SCF/Transforms/Patterns.h"
13#include "mlir/Dialect/Utils/IndexingUtils.h"
14#include "mlir/Dialect/Vector/IR/VectorOps.h"
15#include "mlir/Dialect/XeGPU/IR/XeGPU.h"
16#include "mlir/Dialect/XeGPU/Transforms/Passes.h"
17#include "mlir/Dialect/XeGPU/Transforms/Transforms.h"
18#include "mlir/Dialect/XeGPU/Transforms/XeGPULayoutImpl.h"
19#include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
20#include "mlir/Dialect/XeGPU/uArch/uArchCommon.h"
21#include "mlir/IR/Builders.h"
22#include "mlir/IR/BuiltinAttributes.h"
23#include "mlir/IR/BuiltinOps.h"
24#include "mlir/IR/BuiltinTypes.h"
25#include "mlir/IR/MLIRContext.h"
26#include "mlir/IR/Operation.h"
27#include "mlir/IR/Value.h"
28#include "mlir/IR/ValueRange.h"
29#include "mlir/Transforms/DialectConversion.h"
30#include "llvm/ADT/SetVector.h"
31#include "llvm/Support/LogicalResult.h"
32#include "llvm/Support/raw_ostream.h"
33#include <optional>
34
35namespace mlir {
36namespace xegpu {
37#define GEN_PASS_DEF_XEGPUSGTOLANEDISTRIBUTE
38#include "mlir/Dialect/XeGPU/Transforms/Passes.h.inc"
39} // namespace xegpu
40} // namespace mlir
41
42using namespace mlir;
43
44#define DEBUG_TYPE "xegpu-sg-to-lane-distribute"
45#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE "]: ")
46
47namespace {
48
49/// Casts the given vector value `v` to the expected vector type `expectedTy`.
50static Value castValueTo(ConversionPatternRewriter &rewriter,
51 TypedValue<VectorType> v, VectorType expectedTy) {
52 // If the type matches, simply return the value itself.
53 if (v.getType() == expectedTy)
54 return v;
55 // If only shape differs, use shape cast.
56 if (isa<VectorType>(Val: v.getType()) &&
57 v.getType().getNumElements() == expectedTy.getNumElements())
58 return vector::ShapeCastOp::create(builder&: rewriter, location: v.getLoc(), result: expectedTy, source: v);
59
60 // Else create an unrealized cast.
61 auto newOp = UnrealizedConversionCastOp::create(builder&: rewriter, location: v.getLoc(),
62 resultTypes: expectedTy, operands: ValueRange{v});
63 return newOp.getResult(i: 0);
64}
65
66/// A vector::MultiDimReductionOp at subgroup level in expected form if, it has
67/// exactly 1 reduction dimension, it had valid result layout attribute, and
68/// result type can be distributed to lanes using the layout.
69static bool isValidSubgroupMultiReductionOp(vector::MultiDimReductionOp op) {
70 auto resLayout = xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0));
71 // If no layout, not valid.
72 if (!resLayout || !resLayout.isForSubgroup())
73 return false;
74 // Scalar result (e.g., vector<32xf32> to f32) is valid.
75 if (op.getType().isIntOrFloat())
76 return op.getReductionDims().size() == 1;
77 VectorType resTy = dyn_cast<VectorType>(Val: op.getType());
78 if (!resTy)
79 return false;
80 // Compute the distributed result vector type based on the layout.
81 FailureOr<VectorType> resDistTypeOrFailure =
82 getDistVecTypeBasedOnLaneLayout(layout: resLayout, originalType: resTy);
83 if (failed(Result: resDistTypeOrFailure))
84 return false;
85 return op.getReductionDims().size() == 1;
86}
87
88/// A vector::MultiDimReductionOp reduces lane-locally when no data is combined
89/// across lanes, which holds exactly when the source's lane_layout along the
90/// reduction dimension is 1: every reduced element then lives in one lane.
91static bool isReductionLaneLocal(vector::MultiDimReductionOp op) {
92 // Must be valid MultiDimReductionOp.
93 assert(isValidSubgroupMultiReductionOp(op) && "Expecting a valid subgroup "
94 "MultiDimReductionOp");
95 auto srcLayout = xegpu::getTemporaryLayout(operandOrResult: op->getOpOperand(idx: 0));
96 ArrayRef<int64_t> reductionDims = op.getReductionDims();
97 assert(reductionDims.size() == 1 &&
98 "Expecting single reduction dimension for subgroup multi "
99 "reduction op");
100 int64_t reductionDim = reductionDims[0];
101 SmallVector<int64_t> srcLaneLayout = srcLayout.getEffectiveLaneLayoutAsInt();
102 assert(reductionDim < static_cast<int64_t>(srcLaneLayout.size()) &&
103 "Expecting a source lane_layout covering the reduction dimension");
104 return srcLaneLayout[reductionDim] == 1;
105}
106
107/// Given a vector type and its distributed vector type, return the list of
108/// dimensions that are distributed.
109static SmallVector<int64_t> getDistributedDims(VectorType originalType,
110 VectorType distributedType) {
111 assert(originalType.getRank() == distributedType.getRank() &&
112 "original and distributed vector types must have the same rank");
113 SmallVector<int64_t> distributedDims;
114 for (int64_t i = 0; i < originalType.getRank(); ++i) {
115 if (distributedType.getDimSize(idx: i) != originalType.getDimSize(idx: i))
116 distributedDims.push_back(Elt: i);
117 }
118 return distributedDims;
119}
120
121/// Distributes a subgroup-level CreateNdDesc op to lane-level CreateNdDesc
122/// op. This simply drops the layout attribute from the tensor descriptor type.
123struct SgToLaneCreateNdDesc
124 : public OpConversionPattern<xegpu::CreateNdDescOp> {
125 using OpConversionPattern<xegpu::CreateNdDescOp>::OpConversionPattern;
126
127 LogicalResult
128 matchAndRewrite(xegpu::CreateNdDescOp op, OpAdaptor adaptor,
129 ConversionPatternRewriter &rewriter) const override {
130 xegpu::TensorDescType resultType = op.getType();
131 // If no layout, nothing to do.
132 if (!resultType.getLayout())
133 return failure();
134
135 auto newOp = xegpu::CreateNdDescOp::create(
136 builder&: rewriter, location: op.getLoc(), resultTypes: TypeRange{resultType.dropLayouts()},
137 operands: op.getOperands(), properties: op.getProperties(),
138 discardableAttributes: op->getDiscardableAttrDictionary().getValue());
139 rewriter.replaceOp(op, newValues: newOp.getResult());
140 return success();
141 }
142};
143
144/// Distributes a subgroup-level LoadNd op to lane-level LoadNd op. Output
145/// of lane-level LoadNd op is 1D. ShapeCast is added to restore the
146/// original rank.
147struct SgToLaneLoadNd : public OpConversionPattern<xegpu::LoadNdOp> {
148 using OpConversionPattern<xegpu::LoadNdOp>::OpConversionPattern;
149
150 LogicalResult
151 matchAndRewrite(xegpu::LoadNdOp op, OpAdaptor adaptor,
152 ConversionPatternRewriter &rewriter) const override {
153 xegpu::DistributeLayoutAttr layout = op.getAnchorLayout();
154 // If no layout, nothing to do.
155 if (!layout)
156 return failure();
157 // Check if the layout attached to the tensor descriptor is same as the
158 // anchor layout. Otherwise, this is a conflict.
159 if (op.getTensorDescType().getLayout() != layout)
160 return rewriter.notifyMatchFailure(
161 arg&: op, msg: "conflicting layout attributes on tensor descriptor and anchor");
162 const auto *uArch =
163 xegpu::uArch::getUArch(archName: xegpu::getChipStr(op).value_or(u: ""));
164 if (!uArch)
165 return rewriter.notifyMatchFailure(
166 arg&: op, msg: "xegpu::LoadNdOp require target attribute attached to "
167 "determine transpose "
168 "requirement");
169 auto supportedLaneResultTyOrFailure =
170 xegpu::getDistributedVectorType(tdescTy: op.getTensorDescType());
171 auto expectedLaneResultTyOrFailure =
172 xegpu::getDistVecTypeBasedOnLaneLayout(layout, originalType: op.getType());
173 if (failed(Result: supportedLaneResultTyOrFailure))
174 return rewriter.notifyMatchFailure(
175 arg&: op, msg: "unable to compute the lane vector type for LoadNdOp");
176 if (failed(Result: expectedLaneResultTyOrFailure))
177 return rewriter.notifyMatchFailure(
178 arg&: op, msg: "unable to compute expected lane vector type from lane layout");
179 auto newOp = xegpu::LoadNdOp::create(
180 builder&: rewriter, location: op.getLoc(), value: supportedLaneResultTyOrFailure.value(),
181 TensorDesc: adaptor.getTensorDesc(), offsets: op.getMixedOffsets(), packed: op.getPackedAttr(),
182 transpose: op.getTransposeAttr(), l1_hint: op.getL1HintAttr(), l2_hint: op.getL2HintAttr(),
183 l3_hint: op.getL3HintAttr(), /**layout**/ layout: nullptr);
184 // Set the packed attribute if the layout requires it.
185 newOp.setPacked(xegpu::requirePacked(layout: cast<xegpu::LayoutAttr>(Val&: layout)));
186 // Set the transpose attribute if the layout requires it.
187 if (xegpu::requireTranspose(layout: cast<xegpu::LayoutAttr>(Val&: layout), uArch))
188 newOp.setTranspose(DenseI64ArrayAttr::get(context: rewriter.getContext(), content: {1, 0}));
189 rewriter.replaceOp(op, newValues: castValueTo(rewriter, v: newOp.getResult(),
190 expectedTy: expectedLaneResultTyOrFailure.value()));
191 return success();
192 }
193};
194
195/// Distributes a subgroup-level StoreNd op to lane-level StoreNd op. Stored
196/// value in lane-level StoreNd op is 1D. ShapeCast is added to cast the
197/// incoming value to 1D.
198struct SgToLaneStoreNd : public OpConversionPattern<xegpu::StoreNdOp> {
199 using OpConversionPattern<xegpu::StoreNdOp>::OpConversionPattern;
200
201 LogicalResult
202 matchAndRewrite(xegpu::StoreNdOp op, OpAdaptor adaptor,
203 ConversionPatternRewriter &rewriter) const override {
204 xegpu::DistributeLayoutAttr layout = op.getAnchorLayout();
205 // If no layout, nothing to do.
206 if (!layout)
207 return failure();
208 // Check if the layout attached to the tensor descriptor and value layout is
209 // same as the anchor layout. Otherwise, this is a conflict.
210 if (op.getTensorDescType().getLayout() != layout)
211 return rewriter.notifyMatchFailure(
212 arg&: op, msg: "conflicting layout attributes on tensor descriptor and anchor");
213 auto valueLayout = xegpu::getDistributeLayoutAttr(opr: op->getOpOperand(idx: 0));
214 if (valueLayout != layout)
215 return rewriter.notifyMatchFailure(
216 arg&: op, msg: "conflicting layout attributes on value and anchor");
217 auto supportedLaneValueTyOrFailure =
218 xegpu::getDistributedVectorType(tdescTy: op.getTensorDescType());
219 if (failed(Result: supportedLaneValueTyOrFailure))
220 return rewriter.notifyMatchFailure(
221 arg&: op,
222 msg: "unable to compute lane vector type for StoreNdOp value from tensor "
223 "descriptor");
224
225 xegpu::StoreNdOp::create(
226 builder&: rewriter, location: op.getLoc(),
227 value: castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getValue()),
228 expectedTy: supportedLaneValueTyOrFailure.value()),
229 TensorDesc: adaptor.getTensorDesc(), offsets: op.getMixedOffsets(), l1_hint: op.getL1HintAttr(),
230 l2_hint: op.getL2HintAttr(), l3_hint: op.getL3HintAttr(), /**layout**/ layout: nullptr);
231 rewriter.eraseOp(op);
232 return success();
233 }
234};
235
236/// Distributes a subgroup-level Dpas op to lane-level Dpas op. All inpputs
237/// and output of lane-level Dpas op are 1D. Necessary casts are added to
238/// convert the inputs and output to/from 1D.
239struct SgToLaneDpas : public OpConversionPattern<xegpu::DpasOp> {
240 using OpConversionPattern<xegpu::DpasOp>::OpConversionPattern;
241
242 LogicalResult
243 matchAndRewrite(xegpu::DpasOp op, OpAdaptor adaptor,
244 ConversionPatternRewriter &rewriter) const override {
245 // Check if the op has A, B and CD layouts attached.
246 auto layoutA = cast<xegpu::LayoutAttr>(Val: op.getLayoutAAttr());
247 auto layoutB = cast<xegpu::LayoutAttr>(Val: op.getLayoutBAttr());
248 auto layoutCd = cast<xegpu::LayoutAttr>(Val: op.getLayoutCdAttr());
249 if (!layoutA || !layoutB || !layoutCd)
250 return failure();
251 auto laneResultTyOrFailure =
252 xegpu::getDistributedVectorType(originalType: op.getType(), layout: layoutCd);
253 auto laneATypeOrFailure =
254 xegpu::getDistributedVectorType(originalType: op.getLhs().getType(), layout: layoutA);
255 auto laneBTypeOrFailure =
256 xegpu::getDistributedVectorType(originalType: op.getRhs().getType(), layout: layoutB);
257 auto expectedLaneResultTyOrFailure =
258 xegpu::getDistVecTypeBasedOnLaneLayout(layout: layoutCd, originalType: op.getType());
259 if (failed(Result: laneResultTyOrFailure) || failed(Result: laneATypeOrFailure) ||
260 failed(Result: laneBTypeOrFailure))
261 return rewriter.notifyMatchFailure(
262 arg&: op, msg: "failed to calculate supported lane vector types for DpasOp "
263 "from layouts");
264 if (failed(Result: expectedLaneResultTyOrFailure))
265 return rewriter.notifyMatchFailure(
266 arg&: op, msg: "unable to compute expected lane vector type for DpasOp from "
267 "lane layout");
268
269 // Validate bit widths match uArch packed format requirements
270 const auto *uArch =
271 xegpu::uArch::getUArch(archName: xegpu::getChipStr(op).value_or(u: ""));
272 if (uArch) {
273 const auto *uArchInstruction =
274 dyn_cast<xegpu::uArch::SubgroupMatrixMultiplyAcc>(
275 Val: uArch->getInstruction(
276 instKind: xegpu::uArch::InstructionKind::SubgroupMatrixMultiplyAcc));
277 if (uArchInstruction) {
278 auto laneAType = laneATypeOrFailure.value();
279 auto laneBType = laneBTypeOrFailure.value();
280 // Calculate total packed bit width = element bit width * vector size
281 unsigned aPackedBitWidth =
282 laneAType.getElementTypeBitWidth() * laneAType.getNumElements();
283 unsigned bPackedBitWidth =
284 laneBType.getElementTypeBitWidth() * laneBType.getNumElements();
285 unsigned expectedABitSize = uArchInstruction->getPackedFormatBitSizeA();
286 unsigned expectedBBitSize = uArchInstruction->getPackedFormatBitSizeB();
287
288 if (aPackedBitWidth % expectedABitSize != 0)
289 return rewriter.notifyMatchFailure(
290 arg&: op,
291 msg: "A operand packed bit width must be a multiple of uArch packed "
292 "format requirement");
293 if (bPackedBitWidth % expectedBBitSize != 0)
294 return rewriter.notifyMatchFailure(
295 arg&: op,
296 msg: "B operand packed bit width must be a multiple of uArch packed "
297 "format requirement");
298 }
299 }
300
301 auto newOp = xegpu::DpasOp::create(
302 builder&: rewriter, location: op->getLoc(), result: laneResultTyOrFailure.value(),
303 lhs: castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getLhs()),
304 expectedTy: laneATypeOrFailure.value()),
305 rhs: castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getRhs()),
306 expectedTy: laneBTypeOrFailure.value()),
307 acc: castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getAcc()),
308 expectedTy: laneResultTyOrFailure.value()),
309 /** layoutA**/ layout_a: nullptr,
310 /** layoutB**/ layout_b: nullptr, /** layoutCd**/ layout_cd: nullptr);
311 // Explicitly set the new types to enable correct type materializations.
312 rewriter.replaceOp(op, newValues: castValueTo(rewriter, v: newOp.getResult(),
313 expectedTy: expectedLaneResultTyOrFailure.value()));
314 return success();
315 }
316};
317
318/// Distributes elementwise ops to lane-level elementwise ops. This
319/// currently handles elementwise ops with single result only.
320struct SgToLaneElementWise : public ConversionPattern {
321 SgToLaneElementWise(TypeConverter &typeConverter, MLIRContext *ctx)
322 : ConversionPattern(MatchAnyOpTypeTag(), /*benefit=*/1, ctx) {}
323
324 LogicalResult
325 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
326 ConversionPatternRewriter &rewriter) const override {
327 // Only match ops with elementwise trait and single result.
328 if (!OpTrait::hasElementwiseMappableTraits(op) || op->getNumResults() != 1)
329 return failure();
330
331 auto resultType = dyn_cast<VectorType>(Val: op->getResult(idx: 0).getType());
332 if (!resultType)
333 return rewriter.notifyMatchFailure(
334 arg&: op, msg: "operation result is not a vector type");
335
336 xegpu::DistributeLayoutAttr layout =
337 xegpu::getTemporaryLayout(operandOrResult: llvm::cast<OpResult>(Val: op->getResult(idx: 0)));
338 if (!layout || !layout.isForSubgroup())
339 return rewriter.notifyMatchFailure(
340 arg&: op, msg: "operation result does not have subgroup distribute layout");
341
342 auto laneShapeOrFailure =
343 xegpu::getDistVecTypeBasedOnLaneLayout(layout, originalType: resultType);
344
345 if (failed(Result: laneShapeOrFailure))
346 return rewriter.notifyMatchFailure(
347 arg&: op, msg: "unable to compute lane vector type from the layout");
348
349 VectorType newResultType = laneShapeOrFailure.value();
350 OperationState state(op->getLoc(), op->getName());
351 state.addOperands(newOperands: operands);
352 state.addTypes(newTypes: newResultType);
353 // Copy all attributes except for DistributeLayoutAttr.
354 for (auto attr : op->getDiscardableAttrDictionary().getValue()) {
355 if (!isa<xegpu::DistributeLayoutAttr>(Val: attr.getValue()))
356 state.addAttribute(name: attr.getName(), attr: attr.getValue());
357 }
358 state.propertiesAttr = op->getPropertiesAsAttribute();
359 Operation *newOp = rewriter.create(state);
360
361 rewriter.replaceOp(op, newValues: newOp->getResult(idx: 0));
362 return success();
363 }
364};
365
366/// Distributes a subgroup-level arith ConstantOp to lane-level arith
367/// ConstantOp.
368///
369/// Splat constants are rebuilt with the lane-local vector type. Non-splat
370/// constants are distributed by extracting each lane_data-sized block from
371/// the full constant and inserting it at the correct position in the
372/// distributed vector using insert_strided_slice.
373struct SgToLaneArithConstant : public OpConversionPattern<arith::ConstantOp> {
374 using OpConversionPattern<arith::ConstantOp>::OpConversionPattern;
375
376 LogicalResult
377 matchAndRewrite(arith::ConstantOp op, OpAdaptor adaptor,
378 ConversionPatternRewriter &rewriter) const override {
379 auto resultType = dyn_cast<VectorType>(Val: op.getType());
380 if (!resultType)
381 return failure();
382
383 // Only handle dense vector constants.
384 auto denseAttr = dyn_cast<DenseElementsAttr>(Val: op.getValue());
385 if (!denseAttr)
386 return rewriter.notifyMatchFailure(
387 arg&: op, msg: "only dense vector constants are supported");
388
389 xegpu::DistributeLayoutAttr layout =
390 xegpu::getTemporaryLayout(operandOrResult: llvm::cast<OpResult>(Val: op.getResult()));
391 if (!layout || !layout.isForSubgroup())
392 return rewriter.notifyMatchFailure(
393 arg&: op, msg: "operation result does not have subgroup distribute layout");
394
395 auto laneShapeOrFailure =
396 xegpu::getDistVecTypeBasedOnLaneLayout(layout, originalType: resultType);
397
398 if (failed(Result: laneShapeOrFailure))
399 return rewriter.notifyMatchFailure(
400 arg&: op, msg: "unable to compute lane vector type from the layout");
401
402 VectorType newResultType = laneShapeOrFailure.value();
403 Location loc = op.getLoc();
404
405 // Splat constants: every lane gets the same value, so just rebuild the
406 // splat with the distributed type.
407 if (denseAttr.isSplat()) {
408 auto scalarValue = denseAttr.getSplatValue<Attribute>();
409 auto newDenseAttr = DenseElementsAttr::get(type: newResultType, values: scalarValue);
410 auto newOp =
411 arith::ConstantOp::create(builder&: rewriter, location: loc, result: newResultType, value: newDenseAttr);
412 rewriter.replaceOp(op, newValues: newOp.getResult());
413 return success();
414 }
415
416 // Non-splat constants: each lane extracts the elements it owns from the
417 // full constant using the distributed coordinates from the layout.
418 auto fullConst =
419 arith::ConstantOp::create(builder&: rewriter, location: loc, result: resultType, value: denseAttr);
420
421 Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(),
422 /*upperBound=*/upper_bound: mlir::IntegerAttr());
423 auto maybeCoordsVec = layout.computeDistributedCoords(
424 builder&: rewriter, loc, linearId: laneId, shape: resultType.getShape());
425 if (failed(Result: maybeCoordsVec))
426 return rewriter.notifyMatchFailure(
427 arg&: op, msg: "failed to compute distributed coordinates from layout");
428
429 SmallVector<SmallVector<Value>> coordsVec = maybeCoordsVec.value();
430 SmallVector<int64_t> laneData = layout.getEffectiveLaneDataAsInt();
431 ArrayRef<int64_t> distShape = newResultType.getShape();
432 int64_t rank = newResultType.getRank();
433
434 // Each lane owns one lane_data-sized block per distribution unit.
435 // computeDistributedCoords returns those block starts in row-major order
436 // over the block grid (distShape / laneData).
437 SmallVector<int64_t> blockGridShape(rank);
438 for (int64_t d = 0; d < rank; d++)
439 blockGridShape[d] = distShape[d] / laneData[d];
440 SmallVector<int64_t> blockGridStrides = computeStrides(sizes: blockGridShape);
441
442 auto blockType = VectorType::get(shape: laneData, elementType: newResultType.getElementType());
443 SmallVector<int64_t> unitTile(rank, 1);
444 SmallVector<int64_t> strides(rank, 1);
445
446 Value result = arith::ConstantOp::create(
447 builder&: rewriter, location: loc, result: newResultType, value: rewriter.getZeroAttr(type: newResultType));
448
449 for (auto [blockIdx, blockStart] : llvm::enumerate(First&: coordsVec)) {
450 // Gather the block's elements from the full constant. The block start is
451 // lane-dynamic, so extract element-by-element (row-major over lane_data)
452 // instead.
453 SmallVector<Value> blockElems;
454 for (SmallVector<int64_t> off :
455 StaticTileOffsetRange(laneData, unitTile)) {
456 SmallVector<OpFoldResult> pos(rank);
457 for (int64_t d = 0; d < rank; d++)
458 pos[d] = getAsOpFoldResult(val: arith::AddIOp::create(
459 builder&: rewriter, location: loc, lhs: blockStart[d],
460 rhs: arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: off[d])));
461 blockElems.push_back(Elt: vector::ExtractOp::create(
462 builder&: rewriter, location: loc, source: fullConst.getResult(), position: pos));
463 }
464
465 // Rebuild the block keeping its lane_data shape, then place it with
466 // insert_strided_slice so the block keeps its orientation in the
467 // distributed vector (e.g. a [2, 1] block stays a vertical 2x1 slice).
468 Value block =
469 vector::FromElementsOp::create(builder&: rewriter, location: loc, result: blockType, elements: blockElems);
470 SmallVector<int64_t> blockGridPos =
471 delinearize(linearIndex: blockIdx, strides: blockGridStrides);
472 SmallVector<int64_t> offsets(rank);
473 for (int64_t d = 0; d < rank; d++)
474 offsets[d] = blockGridPos[d] * laneData[d];
475 result = vector::InsertStridedSliceOp::create(builder&: rewriter, location: loc, valueToStore: block,
476 dest: result, offsets, strides);
477 }
478
479 rewriter.replaceOp(op, newValues: result);
480 return success();
481 }
482};
483
484/// Distributes a subgroup-level PrefetchNd op to lane-level PrefetchNd op.
485struct SgToLanePrefetchNd : public OpConversionPattern<xegpu::PrefetchNdOp> {
486 using OpConversionPattern<xegpu::PrefetchNdOp>::OpConversionPattern;
487
488 LogicalResult
489 matchAndRewrite(xegpu::PrefetchNdOp op, OpAdaptor adaptor,
490 ConversionPatternRewriter &rewriter) const override {
491 xegpu::DistributeLayoutAttr layout = op.getAnchorLayout();
492 // If no layout, nothing to do.
493 if (!layout)
494 return failure();
495
496 xegpu::PrefetchNdOp::create(builder&: rewriter, location: op.getLoc(), TensorDesc: adaptor.getTensorDesc(),
497 offsets: op.getMixedOffsets(), l1_hint: op.getL1HintAttr(),
498 l2_hint: op.getL2HintAttr(), l3_hint: op.getL3HintAttr(),
499 /**layout**/ layout: nullptr);
500 rewriter.eraseOp(op);
501 return success();
502 }
503};
504
505/// Distributes a subgroup-level LoadGather (xegpu.load) op to lane-level.
506///
507/// Example 1 (1D):
508/// layout = #xegpu.layout<lane_layout = [16], lane_data = [1]>
509/// %mask = producer_op : vector<16xi1>
510/// %offset = producer_op : vector<16xindex>
511/// %0 = xegpu.load %src[%offset], %mask : memref<256xf16>,
512/// vector<16xindex>, vector<16xi1> -> vector<16xf16>
513/// Distributed to:
514/// %mask = producer_op : vector<1xi1>
515/// %offset = producer_op : vector<1xindex>
516/// %0 = xegpu.load %src[%offset], %mask : memref<256xf16>,
517/// vector<1xindex>, vector<1xi1> -> vector<1xf16>
518///
519/// Example 2 (3D with leading unit dims):
520/// layout = #xegpu.layout<lane_layout = [1, 1, 16], lane_data = [1, 1, 1]>
521/// %mask = producer_op : vector<1x1x16xi1>
522/// %offset = producer_op : vector<1x1x16xindex>
523/// %0 = xegpu.load %src[%offset], %mask : memref<256xf16>,
524/// vector<1x1x16xindex>, vector<1x1x16xi1> -> vector<1x1x16xf16>
525/// Distributed to:
526/// %mask = producer_op : vector<1x1x1xi1>
527/// %offset = producer_op : vector<1x1x1xindex>
528/// %0 = xegpu.load %src[%offset], %mask : memref<256xf16>,
529/// vector<1xindex>, vector<1xi1> -> vector<1xf16>
530struct SgToLaneLoadGather : public OpConversionPattern<xegpu::LoadGatherOp> {
531 using OpConversionPattern<xegpu::LoadGatherOp>::OpConversionPattern;
532
533 LogicalResult
534 matchAndRewrite(xegpu::LoadGatherOp op, OpAdaptor adaptor,
535 ConversionPatternRewriter &rewriter) const override {
536 xegpu::DistributeLayoutAttr layout = op.getAnchorLayout();
537 if (!layout)
538 return failure();
539
540 VectorType origResultTy = op.getValueType();
541 if (!origResultTy)
542 return failure();
543
544 // Check that leading dimensions are unit.
545 int effectiveVecRank = 1;
546 ArrayRef<int64_t> shape = origResultTy.getShape();
547 if (llvm::any_of(
548 Range: shape.take_front(N: origResultTy.getRank() - effectiveVecRank),
549 P: [](int64_t d) { return d != 1; }))
550 return rewriter.notifyMatchFailure(
551 arg&: op, msg: "Only unit dimensions allowed for the leading "
552 "dimensions of the load vector!");
553
554 auto distResultTyOrFailure =
555 xegpu::getDistVecTypeBasedOnLaneLayout(layout, originalType: origResultTy);
556 if (failed(Result: distResultTyOrFailure))
557 return rewriter.notifyMatchFailure(
558 arg&: op, msg: "unable to compute expected lane vector type from lane layout");
559
560 VectorType distResultTy = distResultTyOrFailure.value();
561 VectorType distResultTy1D = VectorType::get(shape: {distResultTy.getNumElements()},
562 elementType: distResultTy.getElementType());
563
564 // Flatten offsets and mask to 1D to match the 1D result type.
565 Value distOffsets = adaptor.getOffsets();
566 auto distOffsetsTy = cast<VectorType>(Val: distOffsets.getType());
567 VectorType offsetsTy1D = VectorType::get(shape: {distOffsetsTy.getNumElements()},
568 elementType: distOffsetsTy.getElementType());
569 distOffsets = castValueTo(
570 rewriter, v: cast<TypedValue<VectorType>>(Val&: distOffsets), expectedTy: offsetsTy1D);
571
572 Value distMask = adaptor.getMask();
573 auto distMaskTy = cast<VectorType>(Val: distMask.getType());
574 VectorType maskTy1D = VectorType::get(shape: {distMaskTy.getNumElements()},
575 elementType: distMaskTy.getElementType());
576 distMask =
577 castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val&: distMask), expectedTy: maskTy1D);
578
579 Value distSource = adaptor.getSource();
580 auto newOp = xegpu::LoadGatherOp::create(
581 builder&: rewriter, location: op.getLoc(), value: distResultTy1D, source: distSource, offsets: distOffsets,
582 mask: distMask, l1_hint: op.getL1HintAttr(), l2_hint: op.getL2HintAttr(), l3_hint: op.getL3HintAttr(),
583 /*layout=*/nullptr, /*contiguity=*/nullptr);
584
585 Value result = newOp->getResult(idx: 0);
586 if (distResultTy1D != distResultTy)
587 result = castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val&: result),
588 expectedTy: distResultTy);
589 rewriter.replaceOp(op, newValues: result);
590 return success();
591 }
592};
593
594/// This pattern distributes a subgroup-level vector.reduction op to
595/// lane-level. This require shuffling the data across the lanes (using
596/// gpu::ShuffleOp) and reducing in stages until all lanes have the final
597/// result.
598struct SgToLaneVectorReduction
599 : public OpConversionPattern<vector::ReductionOp> {
600 using OpConversionPattern<vector::ReductionOp>::OpConversionPattern;
601
602 LogicalResult
603 matchAndRewrite(vector::ReductionOp op, OpAdaptor adaptor,
604 ConversionPatternRewriter &rewriter) const override {
605 auto layout = xegpu::getDistributeLayoutAttr(value: op.getVector());
606
607 // If no layout, nothing to do.
608 if (!layout || !layout.isForSubgroup())
609 return failure();
610
611 VectorType srcVecType = op.getSourceVectorType();
612 // Only rank 1 vectors supported.
613 if (srcVecType.getRank() != 1)
614 return rewriter.notifyMatchFailure(
615 arg&: op, msg: "Only rank 1 reductions can be distributed.");
616 // Lane layout must have the same rank as the vector.
617 if (layout.getRank() != srcVecType.getRank())
618 return rewriter.notifyMatchFailure(
619 arg&: op, msg: "Layout rank does not match vector rank.");
620
621 // Get the subgroup size from the layout.
622 int64_t sgSize = layout.getEffectiveLaneLayoutAsInt()[0];
623 const auto *uArch =
624 xegpu::uArch::getUArch(archName: xegpu::getChipStr(op).value_or(u: ""));
625 if (!uArch)
626 return rewriter.notifyMatchFailure(
627 arg&: op, msg: "xegpu::ReductionOp require target attribute attached to "
628 "determine subgroup size");
629
630 // Only subgroup-sized vectors supported.
631 if (sgSize != uArch->getSubgroupSize() ||
632 srcVecType.getShape()[0] % sgSize != 0)
633 return rewriter.notifyMatchFailure(arg&: op,
634 msg: "Invalid layout or reduction vector "
635 "dimension must match subgroup size.");
636
637 if (!op.getType().isIntOrFloat())
638 return rewriter.notifyMatchFailure(
639 arg&: op, msg: "Reduction distribution currently only supports floats and "
640 "integer types.");
641
642 // Get the distributed vector (per lane portion).
643 Value laneValVec = adaptor.getVector();
644
645 // Distribute and reduce across lanes in the subgroup.
646 Value fullReduce =
647 xegpu::subgroupReduction(loc: op.getLoc(), builder&: rewriter, input: laneValVec,
648 kind: op.getKind(), size: sgSize, fastmath: op.getFastmathAttr());
649
650 // If there's an accumulator, combine it with the reduced value.
651 if (adaptor.getAcc())
652 fullReduce = vector::makeArithReduction(
653 b&: rewriter, loc: op.getLoc(), kind: op.getKind(), v1: fullReduce, acc: adaptor.getAcc(),
654 fastmath: op.getFastmathAttr());
655
656 rewriter.replaceOp(op, newValues: fullReduce);
657 return success();
658 }
659};
660
661/// This pattern distributes a subgroup-level vector.multi_reduction op to
662/// lane-level only if the reduction is lane-local. This means that
663/// reduction dimension is not distributed to lanes and each lane does its own
664/// local reduction.
665struct SgToLaneMultiDimReduction
666 : public OpConversionPattern<vector::MultiDimReductionOp> {
667 using OpConversionPattern<vector::MultiDimReductionOp>::OpConversionPattern;
668
669 LogicalResult
670 matchAndRewrite(vector::MultiDimReductionOp op, OpAdaptor adaptor,
671 ConversionPatternRewriter &rewriter) const override {
672 Value result;
673 ArrayRef<int64_t> reductionDims = op.getReductionDims();
674 assert(reductionDims.size() == 1 &&
675 "Expecting single reduction dimension for subgroup multi "
676 "reduction op");
677 // For rank > 2, ensure leading dimensions are unit.
678 VectorType sourceType = op.getSourceVectorType();
679 int64_t rank = sourceType.getRank();
680 if (rank > 2) {
681 ArrayRef<int64_t> shape = sourceType.getShape();
682 if (llvm::any_of(Range: shape.take_front(N: rank - 2),
683 P: [](int64_t d) { return d != 1; }))
684 return rewriter.notifyMatchFailure(
685 arg&: op, msg: "only unit leading dimensions are supported for "
686 "multi_reduction with rank > 2");
687 }
688 // Handle scalar result: full reduction of a distributed vector to a
689 // scalar. First do a local vector reduction, then cross-lane shuffles.
690 if (op.getType().isIntOrFloat()) {
691 auto reductionDim = reductionDims[0];
692 VectorType origSourceType = op.getSourceVectorType();
693 int64_t reductionDimSize = origSourceType.getShape()[reductionDim];
694 // Local reduction to scalar, then cross-lane butterfly shuffles.
695 result = xegpu::subgroupReduction(loc: op.getLoc(), builder&: rewriter,
696 input: adaptor.getSource(), kind: op.getKind(),
697 size: reductionDimSize, fastmath: op.getFastmathAttr());
698 // Combine with accumulator if present.
699 if (adaptor.getAcc())
700 result = vector::makeArithReduction(b&: rewriter, loc: op.getLoc(), kind: op.getKind(),
701 v1: result, acc: adaptor.getAcc(),
702 fastmath: op.getFastmathAttr());
703 } else if (isReductionLaneLocal(op)) {
704 // For lane-local reduction, lower to a sequence of vector.reduction ops
705 // over 1D slices extracted from the distributed source vector. This is
706 // required so we dont have 2D source vectors at xegpu-linearize.
707 auto reductionDim = reductionDims[0];
708 result = xegpu::lowerToVectorReductions(
709 src: cast<TypedValue<VectorType>>(Val: adaptor.getSource()),
710 acc: cast<TypedValue<VectorType>>(Val: adaptor.getAcc()), kind: op.getKind(),
711 reductionDim, loc: op.getLoc(), rewriter, fastmath: op.getFastmathAttr());
712 } else {
713 auto reductionDim = reductionDims[0];
714 VectorType sourceType = op.getSourceVectorType();
715 int64_t reductionDimSize = sourceType.getShape()[reductionDim];
716 result = xegpu::lowerCrossLaneReductionToShuffles(
717 src: cast<TypedValue<VectorType>>(Val: adaptor.getSource()),
718 acc: cast<TypedValue<VectorType>>(Val: adaptor.getAcc()), kind: op.getKind(),
719 reductionDim, reductionSize: reductionDimSize, loc: op.getLoc(), rewriter,
720 fastmath: op.getFastmathAttr());
721 }
722 rewriter.replaceOp(op, newValues: result);
723 return success();
724 }
725};
726
727/// Helper to compute distributed coordinates for matrix ops.
728/// When not using subgroup_block_io, each lane computes its own
729/// coordinates based on the layout and lane ID.
730static SmallVector<Value> computeDistributedCoordsForMatrixOp(
731 ConversionPatternRewriter &rewriter, Location loc,
732 xegpu::DistributeLayoutAttr layout, ArrayRef<int64_t> payloadShape,
733 ValueRange origOffsets) {
734 Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(),
735 /*upperBound=*/upper_bound: mlir::IntegerAttr());
736 auto maybeCoords =
737 layout.computeDistributedCoords(builder&: rewriter, loc, linearId: laneId, shape: payloadShape);
738 if (failed(Result: maybeCoords))
739 return {};
740 assert(maybeCoords.value().size() == 1 &&
741 "Expected one set of distributed offsets");
742 SmallVector<OpFoldResult> ofrVec = xegpu::addWithRightAligned(
743 builder&: rewriter, loc, lhs: getAsOpFoldResult(values: maybeCoords.value()[0]),
744 rhs: getAsOpFoldResult(values: origOffsets));
745 return llvm::map_to_vector(C&: ofrVec, F: llvm::CastTo<Value>);
746}
747
748/// This pattern distributes a subgroup-level LoadMatrix op to lane-level.
749struct SgToLaneLoadMatrix : public OpConversionPattern<xegpu::LoadMatrixOp> {
750 using OpConversionPattern<xegpu::LoadMatrixOp>::OpConversionPattern;
751
752 LogicalResult
753 matchAndRewrite(xegpu::LoadMatrixOp op, OpAdaptor adaptor,
754 ConversionPatternRewriter &rewriter) const override {
755 auto layout = op.getLayoutAttr();
756 // If no layout, nothing to do.
757 if (!layout)
758 return failure();
759
760 VectorType sgPayloadTy = dyn_cast<VectorType>(Val: op.getResult().getType());
761 if (!sgPayloadTy)
762 return rewriter.notifyMatchFailure(
763 arg&: op, msg: "the matrix op payload must be a vector type");
764
765 auto loc = op.getLoc();
766 auto offsets = op.getMixedOffsets();
767 if (offsets.empty())
768 return rewriter.notifyMatchFailure(arg&: op, msg: "the load op must have offsets");
769
770 FailureOr<VectorType> distPayloadTyOrFailure =
771 getDistVecTypeBasedOnLaneLayout(layout, originalType: sgPayloadTy);
772 if (failed(Result: distPayloadTyOrFailure))
773 return rewriter.notifyMatchFailure(
774 arg&: op, msg: "Failed to distribute matrix op payload based on layout.");
775
776 SmallVector<Value> offsetsAsValues =
777 vector::getAsValues(builder&: rewriter, loc, foldResults: offsets);
778
779 SmallVector<Value> newCoords = offsetsAsValues;
780 if (!op.getSubgroupBlockIoAttr()) {
781 newCoords = computeDistributedCoordsForMatrixOp(
782 rewriter, loc, layout, payloadShape: sgPayloadTy.getShape(), origOffsets: offsetsAsValues);
783 if (newCoords.empty())
784 return rewriter.notifyMatchFailure(
785 arg&: op, msg: "Failed to compute distributed coordinates.");
786 }
787
788 SmallVector<int64_t> newConstOffsets(op.getConstOffsets().size(),
789 ShapedType::kDynamic);
790 DenseI64ArrayAttr newConstOffsetsAttr =
791 rewriter.getDenseI64ArrayAttr(values: newConstOffsets);
792
793 auto newOp = xegpu::LoadMatrixOp::create(
794 builder&: rewriter, location: loc, res: *distPayloadTyOrFailure, mem_desc: adaptor.getMemDesc(),
795 offsets: ValueRange(newCoords), const_offsets: newConstOffsetsAttr, subgroup_block_io: op.getSubgroupBlockIoAttr(),
796 layout: xegpu::DistributeLayoutAttr{});
797 rewriter.replaceOp(op, newValues: newOp.getResult());
798 return success();
799 }
800};
801
802/// Distributes a subgroup-level vector.transpose op to lane-level.
803struct SgToLaneVectorTranspose
804 : public OpConversionPattern<vector::TransposeOp> {
805 using OpConversionPattern<vector::TransposeOp>::OpConversionPattern;
806
807 LogicalResult
808 matchAndRewrite(vector::TransposeOp op, OpAdaptor adaptor,
809 ConversionPatternRewriter &rewriter) const override {
810 xegpu::DistributeLayoutAttr sourceLayout =
811 xegpu::getTemporaryLayout(operandOrResult: op->getOpOperand(idx: 0));
812 xegpu::DistributeLayoutAttr resultLayout =
813 xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0));
814 if (!sourceLayout || !resultLayout)
815 return rewriter.notifyMatchFailure(
816 arg&: op, msg: "the source or result vector of the transpose op lacks layout "
817 "attribute");
818 ArrayRef<int64_t> perm = op.getPermutation();
819 // Result layout must be a transpose of source layout.
820 if (!resultLayout.isTransposeOf(other: sourceLayout, perm,
821 kind: xegpu::LayoutKind::Lane))
822 return rewriter.notifyMatchFailure(
823 arg&: op, msg: "the source or result vector layouts must be transposes of "
824 "each other");
825 FailureOr<VectorType> distributedResultTypeOrFailure =
826 getDistVecTypeBasedOnLaneLayout(layout: resultLayout, originalType: op.getResultVectorType());
827 if (failed(Result: distributedResultTypeOrFailure))
828 return rewriter.notifyMatchFailure(
829 arg&: op, msg: "Failed to distribute the result vector type in "
830 "vector::Transpose op");
831 auto newOp = vector::TransposeOp::create(builder&: rewriter, location: op.getLoc(),
832 vector: adaptor.getVector(), permutation: perm);
833 rewriter.replaceOp(op, newValues: castValueTo(rewriter, v: newOp.getResult(),
834 expectedTy: distributedResultTypeOrFailure.value()));
835 return success();
836 }
837};
838
839/// Distributes a subgroup-level vector.bitcast op to lane-level.
840/// Bitcast only impacts the innermost dimension of the source/result vectors.
841struct SgToLaneVectorBitcast : public OpConversionPattern<vector::BitCastOp> {
842 using OpConversionPattern<vector::BitCastOp>::OpConversionPattern;
843
844 LogicalResult
845 matchAndRewrite(vector::BitCastOp op, OpAdaptor adaptor,
846 ConversionPatternRewriter &rewriter) const override {
847 xegpu::DistributeLayoutAttr resultLayout =
848 xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0));
849 if (!resultLayout)
850 return rewriter.notifyMatchFailure(
851 arg&: op, msg: "result vector of the bitcast op lacks layout attribute");
852 FailureOr<VectorType> distributedResultTypeOrFailure =
853 getDistVecTypeBasedOnLaneLayout(layout: resultLayout, originalType: op.getResultVectorType());
854 if (failed(Result: distributedResultTypeOrFailure))
855 return rewriter.notifyMatchFailure(
856 arg&: op, msg: "Failed to distribute the result vector type in "
857 "vector::BitCast op");
858 auto newOp = vector::BitCastOp::create(
859 builder&: rewriter, location: op.getLoc(), result: distributedResultTypeOrFailure.value(),
860 source: adaptor.getSource());
861 rewriter.replaceOp(op, newValues: newOp.getResult());
862 return success();
863 }
864};
865
866/// Distributes a subgroup-level vector.create_mask or vector.constant_mask op
867/// to lane-level.
868/// The pattern constructs a mask based on the following bounds check:
869/// ```
870/// for d in [0, ..., maskRank):
871/// mask &= (staticOffset[d] < (bound[d] - base[d]))
872/// ```
873/// where
874/// - `base` is the coordinate vector of the first distributed unit.
875/// - `staticOffset` is the offsets vector per element.
876/// - `bound` is the original mask bound for the corresponding dimension.
877/// The mask vector contains *all* elements (i.e., non-unit `lane_data` is
878/// flattened). For example,
879/// ```
880/// %mask = vector.create_mask %bound_0, %bound_1 {
881/// lane_layout = [1, 16], lane_data = [2, 1]
882/// }: vector<8x32xi1>
883/// ```
884/// Has 8 dist units (of shape [2, 1]) with offsets for lane 0:
885/// {
886/// [0, 0], [0, 16],
887/// [2, 0], [2, 16],
888/// [4, 0], [4, 16],
889/// [6, 0], [6, 16]
890/// }
891/// We check the distance to the bound from the first dist. unit:
892/// %base_0 = genCoords(gpu.lane_id, layout)[0][0]
893/// %base_1 = genCoords(gpu.lane_id, layout)[0][1]
894/// %baseDistanceToBound_0 = %bound_0 - %base_0
895/// %baseDistanceToBound_1 = %bound_1 - %base_1
896/// The pattern flattens the dimension d coordinates of each element:
897/// %staticOffset_0 = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7]
898/// %staticOffset_1 = [0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16]
899/// and compares each coordinate against the corresponding mask bound dim:
900/// %mask_0 = arith.cmpi slt, %staticOffset_0, bcast(%baseDistanceToBound_0)
901/// %mask_1 = arith.cmpi slt, %staticOffset_1, bcast(%baseDistanceToBound_1)
902/// %mask = shape_cast(%mask_0 & %mask_1) : vector<16xi1> to vector<8x2xi1>
903///
904template <typename OpType,
905 typename = std::enable_if_t<llvm::is_one_of<
906 OpType, vector::CreateMaskOp, vector::ConstantMaskOp>::value>>
907struct SgToLaneCreateMask : public OpConversionPattern<OpType> {
908 using OpConversionPattern<OpType>::OpConversionPattern;
909
910 LogicalResult
911 matchAndRewrite(OpType op, typename OpType::Adaptor adaptor,
912 ConversionPatternRewriter &rewriter) const override {
913 xegpu::DistributeLayoutAttr layout =
914 xegpu::getTemporaryLayout(op->getOpResult(0));
915 if (!layout || !layout.isForSubgroup())
916 return rewriter.notifyMatchFailure(
917 op, "operation result does not have subgroup distribute layout");
918
919 VectorType origType = op.getType();
920 FailureOr<VectorType> distTypeOrFailure =
921 getDistVecTypeBasedOnLaneLayout(layout, originalType: origType);
922 if (failed(Result: distTypeOrFailure))
923 return rewriter.notifyMatchFailure(
924 op, "unable to compute lane vector type from the layout");
925
926 VectorType distType = distTypeOrFailure.value();
927 Location loc = op.getLoc();
928
929 // Materialize the original mask bounds as Values.
930 SmallVector<Value> origBounds;
931 if constexpr (std::is_same_v<OpType, vector::CreateMaskOp>) {
932 origBounds.append(op.getOperands().begin(), op.getOperands().end());
933 } else {
934 auto dimSizes = op.getMaskDimSizesAttr().asArrayRef();
935 for (auto dimSize : dimSizes)
936 origBounds.push_back(
937 Elt: arith::ConstantIndexOp::create(rewriter, loc, dimSize).getResult());
938 }
939
940 ArrayRef<int64_t> origShape = origType.getShape();
941
942 // Use computeDistributedCoords to get the coordinates each WI owns.
943 Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(),
944 /*upperBound=*/upper_bound: mlir::IntegerAttr());
945 auto maybeCoordsVec =
946 layout.computeDistributedCoords(builder&: rewriter, loc, linearId: laneId, shape: origShape);
947 if (failed(Result: maybeCoordsVec))
948 return rewriter.notifyMatchFailure(
949 op, "failed to compute distributed coordinates from layout");
950
951 SmallVector<SmallVector<Value>> laneDataCoords = maybeCoordsVec.value();
952 SmallVector<int64_t> laneData = layout.getEffectiveLaneDataAsInt();
953 ArrayRef<int64_t> distShape = distType.getShape();
954 int64_t rank = distType.getRank();
955 int64_t numElements = distType.getNumElements();
956
957 if (static_cast<int64_t>(laneData.size()) != rank ||
958 !computeShapeRatio(shape: distShape, subShape: laneData))
959 return rewriter.notifyMatchFailure(
960 op, "lane_data does not tile the distributed vector");
961
962 SmallVector<int64_t> distStrides = computeStrides(sizes: distShape);
963 assert(static_cast<int64_t>(laneDataCoords.size()) *
964 computeProduct(laneData) ==
965 numElements &&
966 "number of coordinate sets must match number of lane_data blocks");
967
968 // Static offsets to be applied to a dynamic lane's dist units coordinates.
969 SmallVector<SmallVector<int64_t>> staticLaneDataOffsetsOrig =
970 layout.computeStaticDistributedCoords(/*linearId=*/0, shape: origShape);
971 if (staticLaneDataOffsetsOrig.empty() ||
972 staticLaneDataOffsetsOrig.size() != laneDataCoords.size())
973 return rewriter.notifyMatchFailure(
974 op, "static and dynamic coordinates disagree on the distribution "
975 "unit count");
976
977 SmallVector<int64_t> unitTile(rank, 1);
978 int64_t linearLaneDataIdx = 0;
979 // Layout and shape information is static, compute static offset of lane's
980 // elements in the source dimensions. Each dim is a flat vec of element
981 // offsets. We store dim-major to have an easy materialization as one const
982 // vector.
983 SmallVector<SmallVector<int64_t>> staticElemOffset(
984 rank, SmallVector<int64_t>(numElements));
985 // For each lane_data block in the lane's dist shape
986 for (SmallVector<int64_t> laneDataOffsetDist :
987 StaticTileOffsetRange(distShape, laneData)) {
988 ArrayRef<int64_t> staticLaneDataOffsetOrig =
989 staticLaneDataOffsetsOrig[linearLaneDataIdx++];
990 // For each element in the lane_data block
991 for (SmallVector<int64_t> elemOffsetInLaneData :
992 StaticTileOffsetRange(laneData, unitTile)) {
993 // For each dim of indexing space
994 SmallVector<int64_t> elementOffsetDist(rank);
995 for (int64_t d = 0; d < rank; d++)
996 elementOffsetDist[d] =
997 laneDataOffsetDist[d] + elemOffsetInLaneData[d];
998 int64_t elemLinearizedIdxDist =
999 linearize(offsets: elementOffsetDist, basis: distStrides);
1000 for (int64_t d = 0; d < rank; d++)
1001 staticElemOffset[d][elemLinearizedIdxDist] =
1002 staticLaneDataOffsetOrig[d] + elemOffsetInLaneData[d];
1003 }
1004 }
1005
1006 // Check, whether ALL elements of the distributed mask are within the valid
1007 // extent of EACH source dimension.
1008 // Expressed as dyn_offset[d] + static_offset[d] < dyn_mask_bound[d],
1009 // or equivalently,
1010 // static_offset[d] < dyn_mask_bound[d] - dyn_offset[d]
1011 auto flatIndexType = VectorType::get(shape: numElements, elementType: rewriter.getIndexType());
1012 Value inBounds;
1013 for (int64_t d = 0; d < rank; d++) {
1014 std::optional<int64_t> constBound = getConstantIntValue(ofr: origBounds[d]);
1015 if (constBound && *constBound >= origShape[d])
1016 continue;
1017
1018 Value materializedStaticOffset = arith::ConstantOp::create(
1019 builder&: rewriter, location: loc, value: rewriter.getIndexVectorAttr(values: staticElemOffset[d]));
1020 // Consider only the first unit, all others are compile-time multiple
1021 // offsets of the first one and are already encoded in staticOffsets.
1022 Value validDimExtent = arith::SubIOp::create(builder&: rewriter, location: loc, lhs: origBounds[d],
1023 rhs: laneDataCoords[0][d]);
1024 // Dim extent applies to all elements.
1025 Value validDimExtentPerElement = vector::BroadcastOp::create(
1026 builder&: rewriter, location: loc, result: flatIndexType, source: validDimExtent);
1027 Value elementMaskInDim = arith::CmpIOp::create(
1028 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::slt, lhs: materializedStaticOffset,
1029 rhs: validDimExtentPerElement);
1030 // An element is valid iff it is within the valid extent of ALL
1031 // dimensions.
1032 inBounds = inBounds ? arith::AndIOp::create(builder&: rewriter, location: loc, lhs: inBounds,
1033 rhs: elementMaskInDim)
1034 .getResult()
1035 : elementMaskInDim;
1036 }
1037 // Every dim's bound saturates the mask extent, so no comparison was
1038 // emitted at all: every element of every lane is in bounds.
1039 if (!inBounds) {
1040 rewriter.replaceOp(op, arith::ConstantOp::create(
1041 builder&: rewriter, location: loc, result: distType,
1042 value: DenseElementsAttr::get(type: distType, value: true)));
1043 return success();
1044 }
1045 auto resMask =
1046 rewriter.createOrFold<vector::ShapeCastOp>(location: loc, args&: distType, args&: inBounds);
1047 rewriter.replaceOp(op, resMask);
1048 return success();
1049 }
1050};
1051
1052/// This pattern distributes a subgroup-level StoreMatrix op to lane-level.
1053struct SgToLaneStoreMatrix : public OpConversionPattern<xegpu::StoreMatrixOp> {
1054 using OpConversionPattern<xegpu::StoreMatrixOp>::OpConversionPattern;
1055
1056 LogicalResult
1057 matchAndRewrite(xegpu::StoreMatrixOp op, OpAdaptor adaptor,
1058 ConversionPatternRewriter &rewriter) const override {
1059 auto layout = op.getLayoutAttr();
1060 // If no layout, nothing to do.
1061 if (!layout)
1062 return failure();
1063
1064 VectorType sgPayloadTy = dyn_cast<VectorType>(Val: op.getData().getType());
1065 if (!sgPayloadTy)
1066 return rewriter.notifyMatchFailure(
1067 arg&: op, msg: "the matrix op payload must be a vector type");
1068
1069 auto loc = op.getLoc();
1070 auto offsets = op.getMixedOffsets();
1071 if (offsets.empty())
1072 return rewriter.notifyMatchFailure(arg&: op, msg: "the store op must have offsets");
1073
1074 FailureOr<VectorType> distPayloadTyOrFailure =
1075 getDistVecTypeBasedOnLaneLayout(layout, originalType: sgPayloadTy);
1076 if (failed(Result: distPayloadTyOrFailure))
1077 return rewriter.notifyMatchFailure(
1078 arg&: op, msg: "Failed to distribute matrix op payload based on layout.");
1079
1080 SmallVector<Value> offsetsAsValues =
1081 vector::getAsValues(builder&: rewriter, loc, foldResults: offsets);
1082
1083 SmallVector<Value> newCoords = offsetsAsValues;
1084 if (!op.getSubgroupBlockIoAttr()) {
1085 newCoords = computeDistributedCoordsForMatrixOp(
1086 rewriter, loc, layout, payloadShape: sgPayloadTy.getShape(), origOffsets: offsetsAsValues);
1087 if (newCoords.empty())
1088 return rewriter.notifyMatchFailure(
1089 arg&: op, msg: "Failed to compute distributed coordinates.");
1090 }
1091
1092 SmallVector<int64_t> newConstOffsets(op.getConstOffsets().size(),
1093 ShapedType::kDynamic);
1094 DenseI64ArrayAttr newConstOffsetsAttr =
1095 rewriter.getDenseI64ArrayAttr(values: newConstOffsets);
1096
1097 xegpu::StoreMatrixOp::create(
1098 builder&: rewriter, location: loc, resultTypes: TypeRange{},
1099 data: castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getData()),
1100 expectedTy: distPayloadTyOrFailure.value()),
1101 mem_desc: adaptor.getMemDesc(), offsets: ValueRange(newCoords), const_offsets: newConstOffsetsAttr,
1102 subgroup_block_io: op.getSubgroupBlockIoAttr(), layout: xegpu::DistributeLayoutAttr{});
1103 rewriter.eraseOp(op);
1104 return success();
1105 }
1106};
1107
1108/// Distributes a subgroup-level StoreScatter (xegpu.store) op to
1109/// lane-level.
1110///
1111/// Example 1 (1D):
1112/// layout = #xegpu.layout<lane_layout = [16], lane_data = [1]>
1113/// %mask = producer_op : vector<16xi1>
1114/// %offset = producer_op : vector<16xindex>
1115/// xegpu.store %payload, %src[%offset], %mask : vector<16xf16>,
1116/// memref<256xf16>, vector<16xindex>, vector<16xi1>
1117/// Distributed to:
1118/// %mask = producer_op : vector<1xi1>
1119/// %offset = producer_op : vector<1xindex>
1120/// xegpu.store %payload, %src[%offset], %mask : vector<1xf16>,
1121/// memref<256xf16>, vector<1xindex>, vector<1xi1>
1122///
1123/// Example 2 (3D with leading unit dims):
1124/// layout = #xegpu.layout<lane_layout = [1, 1, 16], lane_data = [1, 1, 1]>
1125/// %mask = producer_op : vector<1x1x16xi1>
1126/// %offset = producer_op : vector<1x1x16xindex>
1127/// xegpu.store %payload, %src[%offset], %mask : vector<1x1x16xf16>,
1128/// memref<256xf16>, vector<1x1x16xindex>, vector<1x1x16xi1>
1129/// Distributed to:
1130/// %mask = producer_op : vector<1x1x1xi1>
1131/// %offset = producer_op : vector<1x1x1xindex>
1132/// xegpu.store %payload, %src[%offset], %mask : vector<1xf16>,
1133/// memref<256xf16>, vector<1xindex>, vector<1xi1>
1134struct SgToLaneStoreScatter
1135 : public OpConversionPattern<xegpu::StoreScatterOp> {
1136 using OpConversionPattern<xegpu::StoreScatterOp>::OpConversionPattern;
1137
1138 LogicalResult
1139 matchAndRewrite(xegpu::StoreScatterOp op, OpAdaptor adaptor,
1140 ConversionPatternRewriter &rewriter) const override {
1141 xegpu::DistributeLayoutAttr layout = op.getAnchorLayout();
1142 if (!layout)
1143 return failure();
1144
1145 VectorType origValueTy = op.getValueType();
1146 if (!origValueTy)
1147 return failure();
1148
1149 // Check that all leading dimensions are unit dimensions.
1150 int effectiveVecRank = 1;
1151 ArrayRef<int64_t> shape = origValueTy.getShape();
1152 if (llvm::any_of(Range: shape.take_front(N: origValueTy.getRank() - effectiveVecRank),
1153 P: [](int64_t d) { return d != 1; }))
1154 return rewriter.notifyMatchFailure(
1155 arg&: op, msg: "Only unit dimensions allowed for the leading "
1156 "dimensions of the store vector!");
1157
1158 auto distValueTyOrFailure =
1159 xegpu::getDistVecTypeBasedOnLaneLayout(layout, originalType: origValueTy);
1160 if (failed(Result: distValueTyOrFailure))
1161 return rewriter.notifyMatchFailure(
1162 arg&: op, msg: "unable to compute expected lane vector type from lane layout");
1163
1164 VectorType distValueTy = distValueTyOrFailure.value();
1165 VectorType distValueTy1D = VectorType::get(shape: {distValueTy.getNumElements()},
1166 elementType: distValueTy.getElementType());
1167
1168 Value distValue = adaptor.getValue();
1169 if (distValue.getType() != distValueTy1D)
1170 distValue = castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val&: distValue),
1171 expectedTy: distValueTy1D);
1172
1173 // Flatten offsets and mask to 1D to match the 1D value type.
1174 Value distOffsets = adaptor.getOffsets();
1175 auto distOffsetsTy = cast<VectorType>(Val: distOffsets.getType());
1176 VectorType offsetsTy1D = VectorType::get(shape: {distOffsetsTy.getNumElements()},
1177 elementType: distOffsetsTy.getElementType());
1178 distOffsets = castValueTo(
1179 rewriter, v: cast<TypedValue<VectorType>>(Val&: distOffsets), expectedTy: offsetsTy1D);
1180
1181 Value distMask = adaptor.getMask();
1182 auto distMaskTy = cast<VectorType>(Val: distMask.getType());
1183 VectorType maskTy1D = VectorType::get(shape: {distMaskTy.getNumElements()},
1184 elementType: distMaskTy.getElementType());
1185 distMask =
1186 castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val&: distMask), expectedTy: maskTy1D);
1187
1188 Value distDest = adaptor.getDest();
1189 xegpu::StoreScatterOp::create(builder&: rewriter, location: op.getLoc(), value: distValue, dest: distDest,
1190 offsets: distOffsets, mask: distMask, l1_hint: op.getL1HintAttr(),
1191 l2_hint: op.getL2HintAttr(), l3_hint: op.getL3HintAttr(),
1192 /*layout=*/nullptr,
1193 /*contiguity=*/nullptr);
1194 rewriter.eraseOp(op);
1195 return success();
1196 }
1197};
1198
1199/// Distribute a vector::StepOp to lane-level.
1200/// The layout must have exactly 1 effective lane dimension.
1201/// We completely resolve the vector::StepOp by computing the lane_data-sized
1202/// subranges.
1203struct SgToLaneVectorStep : public OpConversionPattern<vector::StepOp> {
1204 using OpConversionPattern<vector::StepOp>::OpConversionPattern;
1205
1206 LogicalResult
1207 matchAndRewrite(vector::StepOp op, OpAdaptor adaptor,
1208 ConversionPatternRewriter &rewriter) const override {
1209 xegpu::DistributeLayoutAttr resultLayout =
1210 xegpu::getTemporaryLayout(operandOrResult: op->getResult(idx: 0));
1211 if (!resultLayout || !resultLayout.isForSubgroup())
1212 return rewriter.notifyMatchFailure(
1213 arg&: op, msg: "the result vector of the step op lacks subgroup layout");
1214
1215 auto loc = op.getLoc();
1216 auto stepResultVecTy = op.getResult().getType();
1217 auto laneShapeOrFailure =
1218 xegpu::getDistVecTypeBasedOnLaneLayout(layout: resultLayout, originalType: stepResultVecTy);
1219 if (failed(Result: laneShapeOrFailure))
1220 return rewriter.notifyMatchFailure(
1221 arg&: op, msg: "unable to compute lane vector type from the layout");
1222 VectorType newVecTy = laneShapeOrFailure.value();
1223
1224 Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(),
1225 /*upperBound=*/upper_bound: mlir::IntegerAttr());
1226 auto laneDataBlockCoords = resultLayout.computeDistributedCoords(
1227 builder&: rewriter, loc, linearId: laneId, shape: stepResultVecTy.getShape());
1228 if (failed(Result: laneDataBlockCoords))
1229 return rewriter.notifyMatchFailure(
1230 arg&: op, msg: "failed to compute lane data block coordinates");
1231
1232 auto laneDataBlockCoordsVec = laneDataBlockCoords.value();
1233 auto laneDataBlockLength = resultLayout.getEffectiveLaneDataAsInt()[0];
1234 assert(static_cast<int64_t>(laneDataBlockCoordsVec.size()) ==
1235 newVecTy.getNumElements() / laneDataBlockLength);
1236 SmallVector<Value> stepVals;
1237 // For each lane_data block, reconstruct its sub-range
1238 // from the range of SG-level vector.step.Example: vector.step
1239 // {slice<layout<lane_layout=[2,4,2], lane_data=[1,2,1]>, dims=[0,2]>} :
1240 // vector<16xindex>
1241 // Each logical lane holds 4 elements as 2 blocks of 2 elements each.
1242 // The blocks are round-robin distributed, so logical lane id 0
1243 // holds values [0,1, 8,9].
1244 for (auto &laneDataBlockCoords : laneDataBlockCoordsVec) {
1245 auto laneDataBlockStartCoord = laneDataBlockCoords[0];
1246 stepVals.push_back(Elt: laneDataBlockStartCoord);
1247 for (int i = 1; i < laneDataBlockLength; ++i) {
1248 auto offset = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: i);
1249 stepVals.push_back(Elt: arith::AddIOp::create(
1250 builder&: rewriter, location: loc, lhs: laneDataBlockStartCoord, rhs: offset));
1251 }
1252 }
1253 assert(static_cast<int64_t>(stepVals.size()) == newVecTy.getNumElements() &&
1254 "Expecting the number of step values to match the number of "
1255 "elements in the vector");
1256 auto stepOpVal =
1257 vector::FromElementsOp::create(builder&: rewriter, location: loc, result: newVecTy, elements: stepVals);
1258 rewriter.replaceOp(op, newOp: stepOpVal);
1259 return success();
1260 }
1261};
1262
1263/// Distributes a subgroup-level vector.extract op to lane-level. Only
1264/// handles sub-vector extraction (result is VectorType, not scalar).
1265struct SgToLaneVectorExtract : public OpConversionPattern<vector::ExtractOp> {
1266 using OpConversionPattern<vector::ExtractOp>::OpConversionPattern;
1267
1268 LogicalResult
1269 matchAndRewrite(vector::ExtractOp op, OpAdaptor adaptor,
1270 ConversionPatternRewriter &rewriter) const override {
1271 // Only handle vector results (not scalar extraction).
1272 auto resultType = dyn_cast<VectorType>(Val: op.getType());
1273 if (!resultType)
1274 return rewriter.notifyMatchFailure(arg&: op, msg: "scalar extract not supported");
1275
1276 xegpu::DistributeLayoutAttr layout =
1277 xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0));
1278 if (!layout || !layout.isForSubgroup())
1279 return failure();
1280
1281 // This implementation assumes distribution only happens on the innermost
1282 // dimension. Verify that lane_layout[0...n-2] are all unit.
1283 auto laneLayout = layout.getEffectiveLaneLayoutAsInt();
1284 if (llvm::any_of(Range: ArrayRef<int64_t>(laneLayout).drop_back(N: 1),
1285 P: [](int64_t v) { return v != 1; }))
1286 return rewriter.notifyMatchFailure(
1287 arg&: op, msg: "only innermost dimension distribution is supported for "
1288 "vector.extract");
1289
1290 auto newOp = vector::ExtractOp::create(
1291 builder&: rewriter, location: op.getLoc(), source: adaptor.getSource(), position: op.getMixedPosition());
1292 rewriter.replaceOp(op, newValues: newOp.getResult());
1293 return success();
1294 }
1295};
1296
1297/// This pattern distributes a subgroup-level ShapeCast op to lane-level.
1298struct SgToLaneVectorShapeCast
1299 : public OpConversionPattern<vector::ShapeCastOp> {
1300 using OpConversionPattern<vector::ShapeCastOp>::OpConversionPattern;
1301
1302 LogicalResult
1303 matchAndRewrite(vector::ShapeCastOp op, OpAdaptor adaptor,
1304 ConversionPatternRewriter &rewriter) const override {
1305 xegpu::DistributeLayoutAttr resultLayout =
1306 xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0));
1307 if (!resultLayout || !resultLayout.isForSubgroup())
1308 return rewriter.notifyMatchFailure(
1309 arg&: op, msg: "the result vector of the shape_cast op lacks subgroup layout");
1310
1311 auto resultDistTypeOrFailure = xegpu::getDistVecTypeBasedOnLaneLayout(
1312 layout: resultLayout, originalType: op.getResultVectorType());
1313 if (failed(Result: resultDistTypeOrFailure))
1314 return rewriter.notifyMatchFailure(
1315 arg&: op, msg: "failed to get distributed vector type for result");
1316
1317 Value source = adaptor.getSource();
1318 auto newShapeCast = vector::ShapeCastOp::create(
1319 builder&: rewriter, location: op.getLoc(), result: resultDistTypeOrFailure.value(), source);
1320 rewriter.replaceOp(op, newOp: newShapeCast);
1321 return success();
1322 }
1323};
1324
1325/// Distributes a subgroup-level vector.extract_strided_slice op to
1326/// lane-level. If the result is distributed, the offsets and sizes are
1327/// adjusted to match the distributed types.
1328struct SgToLaneVectorExtractStridedSlice
1329 : public OpConversionPattern<vector::ExtractStridedSliceOp> {
1330 using OpConversionPattern<vector::ExtractStridedSliceOp>::OpConversionPattern;
1331
1332 LogicalResult
1333 matchAndRewrite(vector::ExtractStridedSliceOp op, OpAdaptor adaptor,
1334 ConversionPatternRewriter &rewriter) const override {
1335 xegpu::DistributeLayoutAttr resultLayout =
1336 xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0));
1337 if (!resultLayout || !resultLayout.isForSubgroup())
1338 return failure();
1339
1340 VectorType resultType = op.getType();
1341 auto distResultTyOrFailure =
1342 xegpu::getDistVecTypeBasedOnLaneLayout(layout: resultLayout, originalType: resultType);
1343 if (failed(Result: distResultTyOrFailure))
1344 return rewriter.notifyMatchFailure(
1345 arg&: op, msg: "unable to compute distributed vector type from lane layout");
1346 VectorType distResultTy = *distResultTyOrFailure;
1347
1348 SmallVector<int64_t> distributedDims =
1349 getDistributedDims(originalType: resultType, distributedType: distResultTy);
1350
1351 // Collect updated sizes, offsets, strides. Pad to full source rank.
1352 int64_t sourceRank = op.getSourceVectorType().getRank();
1353 SmallVector<Attribute> updatedSizes =
1354 llvm::map_to_vector(C: op.getSizes(), F: [](Attribute attr) { return attr; });
1355 SmallVector<Attribute> updatedOffsets = llvm::map_to_vector(
1356 C: op.getOffsets(), F: [](Attribute attr) { return attr; });
1357 SmallVector<Attribute> updatedStrides = llvm::map_to_vector(
1358 C: op.getStrides(), F: [](Attribute attr) { return attr; });
1359 for (int64_t i = op.getSizes().size(); i < sourceRank; ++i) {
1360 updatedSizes.push_back(
1361 Elt: rewriter.getI64IntegerAttr(value: op.getSourceVectorType().getDimSize(idx: i)));
1362 updatedOffsets.push_back(Elt: rewriter.getI64IntegerAttr(value: 0));
1363 updatedStrides.push_back(Elt: rewriter.getI64IntegerAttr(value: 1));
1364 }
1365
1366 // Each distributed dim shrinks by its own lane count, so its size and
1367 // offset are rescaled by that count.
1368 if (!distributedDims.empty()) {
1369 auto sourceLayout = xegpu::getTemporaryLayout(operandOrResult: op->getOpOperand(idx: 0));
1370 if (!sourceLayout || sourceLayout.getEffectiveLaneLayoutAsInt().empty())
1371 return rewriter.notifyMatchFailure(
1372 arg&: op, msg: "source of extract_strided_slice lacks distribution layout");
1373 SmallVector<int64_t> laneLayout =
1374 sourceLayout.getEffectiveLaneLayoutAsInt();
1375 SmallVector<int64_t> laneData = sourceLayout.getEffectiveLaneDataAsInt();
1376 ArrayRef<int64_t> sourceShape = op.getSourceVectorType().getShape();
1377 for (int64_t distDim : distributedDims) {
1378 int64_t lanes = laneLayout[distDim];
1379 if (lanes == 0 || sourceShape[distDim] % lanes != 0)
1380 return rewriter.notifyMatchFailure(
1381 arg&: op, msg: "source size along a distributed dim is not a multiple of "
1382 "its lane count");
1383 int64_t distrDimOffset =
1384 cast<IntegerAttr>(Val&: updatedOffsets[distDim]).getInt();
1385 if (distrDimOffset % (lanes * laneData[distDim]) != 0)
1386 return rewriter.notifyMatchFailure(
1387 arg&: op, msg: "offset along a distributed dim is not a multiple of its "
1388 "lane tile");
1389 updatedSizes[distDim] =
1390 rewriter.getI64IntegerAttr(value: distResultTy.getDimSize(idx: distDim));
1391 updatedOffsets[distDim] =
1392 rewriter.getI64IntegerAttr(value: distrDimOffset / lanes);
1393 }
1394 }
1395
1396 auto newOp = vector::ExtractStridedSliceOp::create(
1397 builder&: rewriter, location: op.getLoc(), resultType0: distResultTy, source: adaptor.getSource(),
1398 offsets: ArrayAttr::get(context: rewriter.getContext(), value: updatedOffsets),
1399 sizes: ArrayAttr::get(context: rewriter.getContext(), value: updatedSizes),
1400 strides: ArrayAttr::get(context: rewriter.getContext(), value: updatedStrides));
1401 rewriter.replaceOp(op, newValues: newOp.getResult());
1402 return success();
1403 }
1404};
1405
1406/// This pattern distributes a subgroup-level `vector.broadcast` op to
1407/// lane-level. The pattern supports three cases:
1408///
1409/// 1) Broadcast a low-rank vector to high-rank vector: The low-rank input
1410/// vector must have a slice layout of the result. If the distributed source
1411/// and target vector types are identical, this lowers to a no-op; otherwise,
1412/// it remains a broadcast but operates on distributed vectors.
1413///
1414/// 2) Broadcast a same-rank vector with identical layouts for source and
1415/// target: The source vector must have unit dimensions, and lane_data must
1416/// be unit size for those unit dims. This always lowers to a no-op.
1417///
1418/// 3) Broadcast a scalar with no layout: This always lowers to a broadcast
1419/// from scalar to distributed result type.
1420///
1421/// Example 1 (low-rank to high-rank broadcast):
1422/// ```
1423/// %0 = "some_op"() {layout_result_0 =
1424/// #xegpu.slice<#xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>,
1425/// dims = [0]>} : () -> vector<16xf16>
1426/// %1 = vector.broadcast %0 {layout_result_0 =
1427/// #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>}
1428/// : vector<16xf16> to vector<16x16xf16>
1429/// ```
1430/// is distributed to:
1431/// ```
1432/// %0 = "some_op"() : () -> vector<1xf16>
1433/// %1 = vector.broadcast %0 : vector<1xf16> to vector<16x1xf16>
1434/// ```
1435///
1436/// Example 2 (same-rank broadcast, no-op):
1437/// ```
1438/// %0 = "some_op"() {layout_result_0 =
1439/// #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>}
1440/// : () -> vector<16x1xf16>
1441/// %1 = vector.broadcast %0 {layout_result_0 =
1442/// #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>}
1443/// : vector<16x1xf16> to vector<16x16xf16>
1444/// ```
1445/// is distributed to (no-op, source already matches distributed result type):
1446/// ```
1447/// %0 = "some_op"() : () -> vector<16x1xf16>
1448/// // broadcast is eliminated, %0 is used directly
1449/// ```
1450///
1451/// Example 3 (scalar to vector broadcast):
1452/// ```
1453/// %0 = "some_op"() : () -> f16
1454/// %1 = vector.broadcast %0 {layout_result_0 =
1455/// #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>}
1456/// : f16 to vector<16x16xf16>
1457/// ```
1458/// is distributed to:
1459/// ```
1460/// %0 = "some_op"() : f16
1461/// %1 = vector.broadcast %0 : f16 to vector<16x1xf16>
1462/// ```
1463struct SgToLaneBroadcast : public OpConversionPattern<vector::BroadcastOp> {
1464 using OpConversionPattern<vector::BroadcastOp>::OpConversionPattern;
1465
1466 LogicalResult
1467 matchAndRewrite(vector::BroadcastOp op, OpAdaptor adaptor,
1468 ConversionPatternRewriter &rewriter) const override {
1469 xegpu::DistributeLayoutAttr resultLayout =
1470 xegpu::getTemporaryLayout(operandOrResult: cast<OpResult>(Val: op.getResult()));
1471 if (!resultLayout || !resultLayout.isForSubgroup())
1472 return rewriter.notifyMatchFailure(
1473 arg&: op, msg: "result does not have subgroup distribute layout");
1474
1475 VectorType destType = op.getResultVectorType();
1476 VectorType sourceType = dyn_cast<VectorType>(Val: op.getSourceType());
1477
1478 xegpu::DistributeLayoutAttr sourceLayout =
1479 xegpu::getTemporaryLayout(operandOrResult: op->getOpOperand(idx: 0));
1480
1481 if (sourceType) {
1482 int64_t rankDiff = destType.getRank() - sourceType.getRank();
1483 if (rankDiff > 0) {
1484 // Case 1: Low-rank to high-rank broadcast.
1485 if (!sourceLayout || !sourceLayout.isSliceOf(other: resultLayout))
1486 op.emitWarning(
1487 message: "broadcast source layout must be a slice of result layout");
1488 } else if (rankDiff == 0) {
1489 // Case 2: Same-rank broadcast.
1490 auto broadcastUnitDimsSet = op.computeBroadcastedUnitDims();
1491 SmallVector<int64_t> broadcastUnitDims(broadcastUnitDimsSet.begin(),
1492 broadcastUnitDimsSet.end());
1493 assert(sourceLayout.isEqualTo(
1494 sourceLayout.setUnitDimData(broadcastUnitDims)) &&
1495 "The sg_data for unit dimensions should be set as 1");
1496 sourceLayout = sourceLayout.setUnitDimLayout(broadcastUnitDims);
1497 }
1498 } else {
1499 // Case 3: Scalar to vector broadcast.
1500 if (sourceLayout)
1501 return rewriter.notifyMatchFailure(
1502 arg&: op, msg: "broadcast from scalar must not have a layout attribute");
1503 }
1504
1505 auto destDistType =
1506 xegpu::getDistVecTypeBasedOnLaneLayout(layout: resultLayout, originalType: destType);
1507 if (failed(Result: destDistType))
1508 return rewriter.notifyMatchFailure(
1509 arg&: op, msg: "failed to distribute the result vector type");
1510
1511 Value source = adaptor.getSource();
1512 // If the adapted source already matches the dest dist type, it's a no-op.
1513 if (source.getType() == destDistType.value()) {
1514 rewriter.replaceOp(op, newValues: source);
1515 return success();
1516 }
1517
1518 auto newOp = vector::BroadcastOp::create(builder&: rewriter, location: op.getLoc(),
1519 result: destDistType.value(), source);
1520 rewriter.replaceOp(op, newOp);
1521 return success();
1522 }
1523};
1524
1525/// Distributes a subgroup-level vector.insert_strided_slice op to
1526/// lane-level. If the dest is distributed, the offsets are adjusted to
1527/// match the distributed types.
1528struct SgToLaneVectorInsertStridedSlice
1529 : public OpConversionPattern<vector::InsertStridedSliceOp> {
1530 using OpConversionPattern<vector::InsertStridedSliceOp>::OpConversionPattern;
1531
1532 LogicalResult
1533 matchAndRewrite(vector::InsertStridedSliceOp op, OpAdaptor adaptor,
1534 ConversionPatternRewriter &rewriter) const override {
1535 xegpu::DistributeLayoutAttr resultLayout =
1536 xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0));
1537 if (!resultLayout || !resultLayout.isForSubgroup())
1538 return failure();
1539
1540 VectorType destType = op.getDestVectorType();
1541 auto distDestTyOrFailure =
1542 xegpu::getDistVecTypeBasedOnLaneLayout(layout: resultLayout, originalType: destType);
1543 if (failed(Result: distDestTyOrFailure))
1544 return rewriter.notifyMatchFailure(
1545 arg&: op, msg: "unable to compute distributed vector type from lane layout");
1546 VectorType distDestTy = *distDestTyOrFailure;
1547
1548 SmallVector<int64_t> destDistributedDims =
1549 getDistributedDims(originalType: destType, distributedType: distDestTy);
1550
1551 SmallVector<Attribute> updatedOffsets = llvm::map_to_vector(
1552 C: op.getOffsets(), F: [](Attribute attr) { return attr; });
1553
1554 if (!destDistributedDims.empty()) {
1555 if (destDistributedDims.size() != 1)
1556 return rewriter.notifyMatchFailure(
1557 arg&: op, msg: "only single dimension distribution is supported");
1558 int64_t destDistDim = destDistributedDims[0];
1559
1560 VectorType srcType = op.getSourceVectorType();
1561 // The distributed dim must be in the last k (source rank) dims of dest.
1562 int64_t sourceDistDim =
1563 destDistDim - (destType.getRank() - srcType.getRank());
1564 if (sourceDistDim < 0)
1565 return rewriter.notifyMatchFailure(
1566 arg&: op, msg: "distributed dimension must be in the last k dims of dest");
1567
1568 auto destLayout = xegpu::getTemporaryLayout(operandOrResult: op->getOpOperand(idx: 1));
1569 auto sourceLayout = xegpu::getTemporaryLayout(operandOrResult: op->getOpOperand(idx: 0));
1570 if (!destLayout || !sourceLayout ||
1571 destLayout.getEffectiveLaneLayoutAsInt().empty() ||
1572 sourceLayout.getEffectiveLaneLayoutAsInt().empty())
1573 return rewriter.notifyMatchFailure(
1574 arg&: op, msg: "source or dest of insert_strided_slice lacks distribution "
1575 "layout");
1576
1577 auto destLaneData = destLayout.getEffectiveLaneDataAsInt();
1578 auto sourceLaneData = sourceLayout.getEffectiveLaneDataAsInt();
1579 // Only check lane_data for the distributed dimension. Non-distributed
1580 // dimensions may have non-unit lane_data (e.g., packed layouts).
1581 if ((destDistDim < static_cast<int64_t>(destLaneData.size()) &&
1582 destLaneData[destDistDim] != 1) ||
1583 (sourceDistDim < static_cast<int64_t>(sourceLaneData.size()) &&
1584 sourceLaneData[sourceDistDim] != 1))
1585 return rewriter.notifyMatchFailure(
1586 arg&: op, msg: "expecting unit lane data along the distributed dimension");
1587
1588 // The distributed dimension may span only a subset of the subgroup, so
1589 // divide its size and offset by the lanes that actually cover it.
1590 auto destLaneLayout = destLayout.getEffectiveLaneLayoutAsInt();
1591 int64_t numLanesAlongDim =
1592 destDistDim < static_cast<int64_t>(destLaneLayout.size())
1593 ? destLaneLayout[destDistDim]
1594 : 1;
1595
1596 int64_t srcDistrDimSize = srcType.getDimSize(idx: sourceDistDim);
1597 if (srcDistrDimSize % numLanesAlongDim != 0)
1598 return rewriter.notifyMatchFailure(
1599 arg&: op, msg: "source distributed dim size is not a multiple of "
1600 "the number of lanes along that dimension");
1601
1602 int64_t destDistrDimOffset =
1603 cast<IntegerAttr>(Val: op.getOffsets()[destDistDim]).getInt();
1604 if (destDistrDimOffset % numLanesAlongDim != 0)
1605 return rewriter.notifyMatchFailure(
1606 arg&: op, msg: "offset along distributed dim is not a multiple of "
1607 "the number of lanes along that dimension");
1608 // Adjust offset for the distributed dimension.
1609 updatedOffsets[destDistDim] =
1610 rewriter.getI64IntegerAttr(value: destDistrDimOffset / numLanesAlongDim);
1611 }
1612
1613 auto newOp = vector::InsertStridedSliceOp::create(
1614 builder&: rewriter, location: op.getLoc(), result: distDestTy, valueToStore: adaptor.getValueToStore(),
1615 dest: adaptor.getDest(),
1616 offsets: ArrayAttr::get(context: rewriter.getContext(), value: updatedOffsets), strides: op.getStrides());
1617 rewriter.replaceOp(op, newValues: newOp.getResult());
1618 return success();
1619 }
1620};
1621
1622/// Distributes a subgroup-level vector.insert op to lane-level. Only
1623/// handles sub-vector insertion (value to store is VectorType, not scalar).
1624struct SgToLaneVectorInsert : public OpConversionPattern<vector::InsertOp> {
1625 using OpConversionPattern<vector::InsertOp>::OpConversionPattern;
1626
1627 LogicalResult
1628 matchAndRewrite(vector::InsertOp op, OpAdaptor adaptor,
1629 ConversionPatternRewriter &rewriter) const override {
1630 // Only handle vector value-to-store (not scalar insertion).
1631 auto valueType = dyn_cast<VectorType>(Val: op.getValueToStoreType());
1632 if (!valueType)
1633 return rewriter.notifyMatchFailure(arg&: op, msg: "scalar insert not supported");
1634
1635 xegpu::DistributeLayoutAttr layout =
1636 xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0));
1637 if (!layout || !layout.isForSubgroup())
1638 return failure();
1639
1640 // verify that the outer k dimensions (for offsets)
1641 // don't have non-unit lane_layout.
1642 auto laneLayout = layout.getEffectiveLaneLayoutAsInt();
1643 if (llvm::any_of(Range: ArrayRef<int64_t>(laneLayout).drop_back(N: 1),
1644 P: [](int64_t v) { return v != 1; }))
1645 return rewriter.notifyMatchFailure(
1646 arg&: op, msg: "only innermost dimension distribution is supported for "
1647 "vector.insert");
1648
1649 auto newOp = vector::InsertOp::create(
1650 builder&: rewriter, location: op.getLoc(), valueToStore: adaptor.getValueToStore(), dest: adaptor.getDest(),
1651 position: op.getMixedPosition());
1652 rewriter.replaceOp(op, newValues: newOp.getResult());
1653 return success();
1654 }
1655};
1656
1657/// Redistributes `src` for a `convert_layout` that changes only the
1658/// `lane_layout` along the outer (distributed) dimension, shrinking it from
1659/// `currentLaneNum` to `targetLaneNum` lanes (a partial-subgroup
1660/// distribution). Because the data is no longer replicated across all lanes,
1661/// each surviving lane must gather the values that previously lived in the
1662/// lanes that are dropped. The values are gathered with `gpu.shuffle` and
1663/// concatenated with the lane-local data using `vector.shuffle`, which doubles
1664/// the distributed outer dimension when the lane count is halved.
1665///
1666/// Only halving the lane count (a factor of two) is currently supported.
1667/// Returns the redistributed value on success, or failure if `src` cannot be
1668/// shuffled (e.g. it is not a rank-2 vector or its bit width is not a multiple
1669/// of 32).
1670static FailureOr<Value>
1671shuffleDataAsLaneLayoutChange(ConversionPatternRewriter &rewriter, Location loc,
1672 Value src, int64_t currentLaneNum,
1673 int64_t targetLaneNum) {
1674 VectorType srcTy = dyn_cast<VectorType>(Val: src.getType());
1675 if (!srcTy || srcTy.getRank() != 2)
1676 return failure();
1677 // Only halving the lane count (factor of two) is supported for now.
1678 if (targetLaneNum <= 0 || currentLaneNum != targetLaneNum * 2)
1679 return failure();
1680 // gpu.shuffle operates on i32, so the data must be a multiple of 32 bits.
1681 int64_t vectorBitWidth =
1682 srcTy.getNumElements() * srcTy.getElementTypeBitWidth();
1683 if (vectorBitWidth % 32 != 0)
1684 return failure();
1685
1686 // A vector cannot be shuffled across lanes directly:
1687 // -- cast the source to a 1D vector of i32
1688 // -- create a temp 1D vector of i32 initialized to zero
1689 // -- for each i32 element:
1690 // ---- extract it from the source bundle
1691 // ---- gpu.shuffle to gather the value from the partner lane
1692 // ---- insert it into the temp bundle
1693 // -- cast the temp back to the source vector type
1694 // -- vector.shuffle the source and temp to concatenate along the outer dim
1695 Type shuffleElemTy = rewriter.getI32Type();
1696 int64_t numShuffles = vectorBitWidth / 32;
1697 VectorType shuffleBundleTy = VectorType::get(shape: {numShuffles}, elementType: shuffleElemTy);
1698 // Initialize temp to zero.
1699 Value temp = arith::ConstantOp::create(
1700 builder&: rewriter, location: loc,
1701 value: DenseElementsAttr::get(type: shuffleBundleTy,
1702 values: IntegerAttr::get(type: shuffleElemTy, value: 0)));
1703 VectorType flatSrcTy =
1704 VectorType::get(shape: {srcTy.getNumElements()}, elementType: srcTy.getElementType());
1705 Value flatSrc = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: flatSrcTy, source: src);
1706 Value shuffleBundle =
1707 vector::BitCastOp::create(builder&: rewriter, location: loc, result: shuffleBundleTy, source: flatSrc);
1708 for (int64_t i = 0; i < numShuffles; i++) {
1709 Value shuffleElem =
1710 vector::ExtractOp::create(builder&: rewriter, location: loc, source: shuffleBundle, position: i);
1711 shuffleElem = gpu::ShuffleOp::create(builder&: rewriter, location: loc, value: shuffleElem, offset: 0,
1712 width: targetLaneNum, mode: gpu::ShuffleMode::UP)
1713 .getResult(i: 0);
1714 temp = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: shuffleElem, dest: temp, position: i);
1715 }
1716 temp = vector::BitCastOp::create(builder&: rewriter, location: loc, result: flatSrcTy, source: temp);
1717 temp = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: srcTy, source: temp);
1718
1719 // Concatenate the lane-local and gathered data along the outer dimension.
1720 SmallVector<int64_t> indices(srcTy.getShape()[0] * 2);
1721 std::iota(first: indices.begin(), last: indices.end(), value: 0);
1722 Value res = vector::ShuffleOp::create(builder&: rewriter, location: loc, v1: src, v2: temp, mask: indices);
1723 return res;
1724}
1725
1726/// Repacks `src`'s `lane_data` along `repackDim` between round-robin and
1727/// contiguous form with an `xegpu.lane_shuffle`, which moves each lane's run of
1728/// `k` elements across lanes while preserving the element type.
1729///
1730/// `inputData`/`targetData` are the `repackDim` `lane_data` of the input and
1731/// target layouts; exactly one must be 1 (round-robin) and the other `k`
1732/// (contiguous). Returns failure if that does not hold.
1733static FailureOr<Value> repackLaneData(ConversionPatternRewriter &rewriter,
1734 Location loc, Value src,
1735 int64_t repackDim, int64_t inputData,
1736 int64_t targetData) {
1737 auto srcTy = dyn_cast<VectorType>(Val: src.getType());
1738 if (!srcTy)
1739 return failure();
1740 int64_t rank = srcTy.getRank();
1741 Type elemTy = srcTy.getElementType();
1742 int64_t k = srcTy.getShape()[repackDim];
1743
1744 bool roundRobinToContig = inputData == 1 && targetData == k;
1745 bool contigToRoundRobin = inputData == k && targetData == 1;
1746 if (!roundRobinToContig && !contigToRoundRobin)
1747 return failure();
1748
1749 // Round-robin -> contiguous gathers a lane's strided elements into
1750 // consecutive positions (pack); the reverse scatters them back (unpack).
1751 xegpu::LaneShuffleMode mode = roundRobinToContig
1752 ? xegpu::LaneShuffleMode::Pack
1753 : xegpu::LaneShuffleMode::Unpack;
1754 VectorType runTy = VectorType::get(shape: {k}, elementType: elemTy);
1755
1756 // Common case: the lane fragment is a single run (every dimension other than
1757 // `repackDim` is unit), so collapse it to 1D, shuffle once, and restore it.
1758 if (srcTy.getNumElements() == k) {
1759 if (rank == 1)
1760 return Value(
1761 xegpu::LaneShuffleOp::create(builder&: rewriter, location: loc, result: runTy, source: src, mode));
1762 Value flat = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: runTy, source: src);
1763 Value shuffled =
1764 xegpu::LaneShuffleOp::create(builder&: rewriter, location: loc, result: runTy, source: flat, mode);
1765 return Value(vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: srcTy, source: shuffled));
1766 }
1767
1768 // When `repackDim` is innermost each run is a contiguous sub-vector, so it is
1769 // extracted and re-inserted as a whole.
1770 if (repackDim == rank - 1) {
1771 SmallVector<int64_t> outerShape(srcTy.getShape().drop_back());
1772 int64_t numRuns = computeProduct(basis: outerShape);
1773 SmallVector<int64_t> outerStrides = computeStrides(sizes: outerShape);
1774 Value result = arith::ConstantOp::create(builder&: rewriter, location: loc, result: srcTy,
1775 value: rewriter.getZeroAttr(type: srcTy));
1776 for (int64_t i = 0; i < numRuns; ++i) {
1777 SmallVector<int64_t> pos = delinearize(linearIndex: i, strides: outerStrides);
1778 Value run = vector::ExtractOp::create(builder&: rewriter, location: loc, source: src, position: pos);
1779 Value shuffled =
1780 xegpu::LaneShuffleOp::create(builder&: rewriter, location: loc, result: runTy, source: run, mode);
1781 result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: shuffled, dest: result, position: pos);
1782 }
1783 return result;
1784 }
1785
1786 // Otherwise each run is strided along `repackDim`: extract the `k`-long slice
1787 // (a sub-vector that is unit along every other dim), flatten it to 1D,
1788 // shuffle, and insert it back.
1789 SmallVector<int64_t> keptShape;
1790 SmallVector<int64_t> keptDims;
1791 for (int64_t d = 0; d < rank; ++d)
1792 if (d != repackDim) {
1793 keptShape.push_back(Elt: srcTy.getShape()[d]);
1794 keptDims.push_back(Elt: d);
1795 }
1796 int64_t numRuns = computeProduct(basis: keptShape);
1797 SmallVector<int64_t> keptStrides = computeStrides(sizes: keptShape);
1798 SmallVector<int64_t> sliceSizes(rank, 1);
1799 sliceSizes[repackDim] = k;
1800 SmallVector<int64_t> sliceStrides(rank, 1);
1801 VectorType sliceTy = VectorType::get(shape: sliceSizes, elementType: elemTy);
1802 Value result = arith::ConstantOp::create(builder&: rewriter, location: loc, result: srcTy,
1803 value: rewriter.getZeroAttr(type: srcTy));
1804 for (int64_t i = 0; i < numRuns; ++i) {
1805 SmallVector<int64_t> keptPos = delinearize(linearIndex: i, strides: keptStrides);
1806 SmallVector<int64_t> offsets(rank, 0);
1807 for (auto [dim, coord] : llvm::zip_equal(t&: keptDims, u&: keptPos))
1808 offsets[dim] = coord;
1809 Value slice = vector::ExtractStridedSliceOp::create(
1810 builder&: rewriter, location: loc, source: src, offsets, sizes: sliceSizes, strides: sliceStrides);
1811 Value run = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: runTy, source: slice);
1812 Value repacked =
1813 xegpu::LaneShuffleOp::create(builder&: rewriter, location: loc, result: runTy, source: run, mode);
1814 Value repackedSlice =
1815 vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: sliceTy, source: repacked);
1816 result = vector::InsertStridedSliceOp::create(
1817 builder&: rewriter, location: loc, valueToStore: repackedSlice, dest: result, offsets, strides: sliceStrides);
1818 }
1819 return result;
1820}
1821
1822/// Folds a subgroup-level ConvertLayout op with compatible lane layouts.
1823struct SgToLaneConvertLayout
1824 : public OpConversionPattern<xegpu::ConvertLayoutOp> {
1825 using OpConversionPattern<xegpu::ConvertLayoutOp>::OpConversionPattern;
1826
1827 LogicalResult
1828 matchAndRewrite(xegpu::ConvertLayoutOp op, OpAdaptor adaptor,
1829 ConversionPatternRewriter &rewriter) const override {
1830 auto inputLayout = op.getEffectiveInputLayout();
1831 auto targetLayout = op.getTargetLayoutAttr();
1832 Type valType = op.getResult().getType();
1833
1834 if (valType.isIntOrFloat()) {
1835 rewriter.replaceOp(op, newValues: op.getSource());
1836 return success();
1837 }
1838
1839 auto resShape = cast<VectorType>(Val&: valType).getShape();
1840 SmallVector<int64_t> resShapeVec(resShape.begin(), resShape.end());
1841
1842 // Equivalent layouts: the convert_layout is a no-op and folds to its
1843 // source.
1844 if (inputLayout.isCompatibleWith(other: targetLayout, shape: resShapeVec,
1845 level: xegpu::LayoutKind::Lane)) {
1846 rewriter.replaceOp(op, newValues: adaptor.getSource());
1847 return success();
1848 }
1849
1850 // Handle the special case where the conversion redistributes a value
1851 // across a fraction of the subgroup: the lane_layout shrinks along the
1852 // outer (distributed) dimension while lane_data stays the same. Only a
1853 // pure outer-dimension lane_layout change is supported, so the inner
1854 // lane_layout must be unit (making the outer dim the only distributed one)
1855 // and the outer lane_layout must be genuinely distributed (> 1), which
1856 // also rules out the degenerate [1, 1] layout.
1857 if (inputLayout.getEffectiveOrderAsInt() ==
1858 targetLayout.getEffectiveOrderAsInt() &&
1859 inputLayout.getRank() == 2 && targetLayout.getRank() == 2) {
1860 auto laneLayout = inputLayout.getEffectiveLaneLayoutAsInt();
1861 auto targetLaneLayout = targetLayout.getEffectiveLaneLayoutAsInt();
1862 auto laneData = inputLayout.getEffectiveLaneDataAsInt();
1863 auto targetLaneData = targetLayout.getEffectiveLaneDataAsInt();
1864 if (laneLayout.size() == 2 && targetLaneLayout.size() == 2 &&
1865 laneData == targetLaneData && laneLayout[1] == 1 &&
1866 targetLaneLayout[1] == 1 && laneLayout[0] > 1 &&
1867 laneLayout[0] != targetLaneLayout[0]) {
1868 FailureOr<Value> res = shuffleDataAsLaneLayoutChange(
1869 rewriter, loc: op.getLoc(), src: adaptor.getSource(), currentLaneNum: laneLayout[0],
1870 targetLaneNum: targetLaneLayout[0]);
1871 if (succeeded(Result: res)) {
1872 rewriter.replaceOp(op, newValues: *res);
1873 return success();
1874 }
1875 }
1876 }
1877
1878 // Handle a pure `lane_data` repack: `lane_layout` and `order` are unchanged
1879 // and exactly one dimension's `lane_data` switches between round-robin
1880 // (lane_data 1) and contiguous (lane_data == run length). The elements per
1881 // lane are unchanged, but their assignment to lanes is not, so the data is
1882 // moved across lanes with `xegpu.lane_shuffle`. The changed dimension must
1883 // be one of the two innermost ones, since sg-to-lane distribution is 2D.
1884 if (inputLayout.getEffectiveOrderAsInt() ==
1885 targetLayout.getEffectiveOrderAsInt() &&
1886 inputLayout.getEffectiveLaneLayoutAsInt() ==
1887 targetLayout.getEffectiveLaneLayoutAsInt()) {
1888 auto laneLayout = inputLayout.getEffectiveLaneLayoutAsInt();
1889 auto laneData = inputLayout.getEffectiveLaneDataAsInt();
1890 auto targetLaneData = targetLayout.getEffectiveLaneDataAsInt();
1891 // Find the single dimension whose lane_data changed; bail out if more
1892 // than one differs.
1893 int64_t rank = laneData.size();
1894 int64_t repackDim = -1;
1895 bool multipleChanged = false;
1896 for (int64_t d = 0; d < rank; ++d)
1897 if (laneData[d] != targetLaneData[d]) {
1898 if (repackDim != -1)
1899 multipleChanged = true;
1900 repackDim = d;
1901 }
1902
1903 // `repackDim` must be the distributed dim (lane_layout != 1) and the
1904 // other innermost dim non-distributed (lane_layout == 1).
1905 int64_t otherDim = repackDim == rank - 1 ? rank - 2 : rank - 1;
1906 bool laneLayoutOk = repackDim != -1 && laneLayout[repackDim] != 1 &&
1907 (rank < 2 || laneLayout[otherDim] == 1);
1908
1909 // Exactly one dimension must change, and it must be one of the two
1910 // innermost (>= rank - 2).
1911 if (repackDim != -1 && repackDim >= rank - 2 && !multipleChanged &&
1912 laneLayoutOk) {
1913 FailureOr<Value> res = repackLaneData(
1914 rewriter, loc: op.getLoc(), src: adaptor.getSource(), repackDim,
1915 inputData: laneData[repackDim], targetData: targetLaneData[repackDim]);
1916 if (succeeded(Result: res)) {
1917 rewriter.replaceOp(op, newValues: *res);
1918 return success();
1919 }
1920 }
1921 }
1922
1923 return rewriter.notifyMatchFailure(
1924 arg&: op, msg: "lowering incompatible convert_layout not yet supported");
1925 }
1926};
1927
1928/// The offsets and shape of one `lane_data`-sized piece of a lane's fragment.
1929/// `computeDistributedCoords` returns one coordinate per distribution unit,
1930/// enumerated row major over `shape / (lane_layout * lane_data)`, and unit `u`
1931/// owns the piece of the fragment at `delinearize(u) * lane_data`.
1932struct LaneFragmentPiece {
1933 SmallVector<int64_t> offsets;
1934 SmallVector<int64_t> sizes;
1935};
1936
1937static LaneFragmentPiece getLaneFragmentPiece(int64_t unit,
1938 ArrayRef<int64_t> shape,
1939 ArrayRef<int64_t> laneLayout,
1940 ArrayRef<int64_t> laneData) {
1941 int64_t rank = shape.size();
1942 SmallVector<int64_t> units(rank);
1943 for (int64_t d = 0; d < rank; ++d)
1944 units[d] = shape[d] / std::min(a: shape[d], b: laneLayout[d] * laneData[d]);
1945 SmallVector<int64_t> unitCoords = delinearize(linearIndex: unit, strides: computeStrides(sizes: units));
1946 LaneFragmentPiece piece;
1947 for (int64_t d = 0; d < rank; ++d) {
1948 piece.offsets.push_back(Elt: unitCoords[d] * laneData[d]);
1949 piece.sizes.push_back(Elt: laneData[d]);
1950 }
1951 return piece;
1952}
1953
1954/// Last-resort lowering for a `convert_layout` that none of the patterns above
1955/// handle: round-trip the value through shared local memory, writing each
1956/// lane's fragment to the coordinates `input_layout` gives it and reading it
1957/// back from the ones `target_layout` gives it. Any pair of layouts that can
1958/// both distribute the value works, at the cost of an SLM write and read.
1959///
1960/// The op is subgroup level, but every subgroup of the workgroup executes it on
1961/// its own data, so the scratch holds one tile per subgroup and each subgroup
1962/// addresses its own through a `gpu.subgroup_id` offset on the outermost
1963/// dimension. The subgroup count comes from the kernel's `known_block_size`.
1964///
1965/// A lane's fragment is one `lane_data`-sized piece per distribution unit, so
1966/// the transfer is one `store_matrix`/`load_matrix` per unit; layouts with
1967/// a single unit, which is the common case, give a single pair.
1968struct SgToLaneConvertLayoutViaSLM
1969 : public OpConversionPattern<xegpu::ConvertLayoutOp> {
1970 using OpConversionPattern<xegpu::ConvertLayoutOp>::OpConversionPattern;
1971
1972 LogicalResult
1973 matchAndRewrite(xegpu::ConvertLayoutOp op, OpAdaptor adaptor,
1974 ConversionPatternRewriter &rewriter) const override {
1975 xegpu::DistributeLayoutAttr inputLayout = op.getEffectiveInputLayout();
1976 xegpu::DistributeLayoutAttr targetLayout = op.getTargetLayoutAttr();
1977 if (!inputLayout || !targetLayout || !inputLayout.isForSubgroup() ||
1978 !targetLayout.isForSubgroup())
1979 return rewriter.notifyMatchFailure(arg&: op, msg: "both layouts must be lane level");
1980
1981 auto valueTy = dyn_cast<VectorType>(Val: op.getResult().getType());
1982 if (!valueTy)
1983 return rewriter.notifyMatchFailure(arg&: op, msg: "value type must be a vector");
1984 Type elemTy = valueTy.getElementType();
1985 // The scratch is sized in bytes, so sub-byte elements would not get a
1986 // well-defined address.
1987 if (!elemTy.isIntOrFloat() || elemTy.getIntOrFloatBitWidth() % 8 != 0)
1988 return rewriter.notifyMatchFailure(
1989 arg&: op, msg: "element type must be a whole number of bytes");
1990
1991 ArrayRef<int64_t> sgShape = valueTy.getShape();
1992 int64_t rank = sgShape.size();
1993 if (rank != inputLayout.getRank() || rank != targetLayout.getRank())
1994 return rewriter.notifyMatchFailure(
1995 arg&: op, msg: "both layouts must have the rank of the value");
1996
1997 FailureOr<VectorType> distInputTy =
1998 xegpu::getDistVecTypeBasedOnLaneLayout(layout: inputLayout, originalType: valueTy);
1999 FailureOr<VectorType> distTargetTy =
2000 xegpu::getDistVecTypeBasedOnLaneLayout(layout: targetLayout, originalType: valueTy);
2001 if (failed(Result: distInputTy) || failed(Result: distTargetTy))
2002 return rewriter.notifyMatchFailure(
2003 arg&: op, msg: "value type must be distributable by both layouts");
2004
2005 const auto *uArch =
2006 xegpu::uArch::getUArch(archName: xegpu::getChipStr(op).value_or(u: ""));
2007 if (!uArch)
2008 return rewriter.notifyMatchFailure(
2009 arg&: op, msg: "target attribute is required to determine the subgroup size");
2010 FailureOr<int64_t> numSubgroups =
2011 xegpu::getNumSubgroupsFromBlockSize(op, subgroupSize: uArch->getSubgroupSize());
2012 if (failed(Result: numSubgroups))
2013 return rewriter.notifyMatchFailure(
2014 arg&: op, msg: "the scratch holds one tile per subgroup, so @known_block_size "
2015 "must be attached to the kernel, with power-of-two dimensions "
2016 "covering at least one subgroup");
2017
2018 Location loc = op.getLoc();
2019
2020 // One tile per subgroup, stacked along the outermost dimension.
2021 SmallVector<int64_t> slmShape(sgShape);
2022 slmShape[0] *= *numSubgroups;
2023 int64_t slmBytes =
2024 computeProduct(basis: slmShape) * elemTy.getIntOrFloatBitWidth() / 8;
2025 auto slmTy = MemRefType::get(shape: {slmBytes}, elementType: rewriter.getI8Type(), map: {}, memorySpaceInd: 3);
2026 Value slm = memref::AllocaOp::create(builder&: rewriter, location: loc, memrefType: slmTy);
2027 Value memDesc = xegpu::CreateMemDescOp::create(
2028 builder&: rewriter, location: loc,
2029 mem_desc: xegpu::MemDescType::get(context: rewriter.getContext(), shape: slmShape, elementType: elemTy,
2030 mem_layout: nullptr),
2031 source: slm);
2032
2033 // This subgroup's tile starts at `subgroup_id` tiles into the scratch.
2034 Value sgId = gpu::SubgroupIdOp::create(builder&: rewriter, location: loc,
2035 result: rewriter.getIndexType(), upper_bound: nullptr);
2036 SmallVector<Value> base(rank);
2037 base[0] = arith::MulIOp::create(
2038 builder&: rewriter, location: loc, lhs: sgId,
2039 rhs: arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: sgShape[0]));
2040 for (int64_t d = 1; d < rank; ++d)
2041 base[d] = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
2042
2043 Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(),
2044 /*upperBound=*/upper_bound: mlir::IntegerAttr());
2045 auto dynamicOffsets = rewriter.getDenseI64ArrayAttr(
2046 values: SmallVector<int64_t>(rank, ShapedType::kDynamic));
2047
2048 // Adds this subgroup's base to one distribution unit's coordinates.
2049 auto withBase = [&](ArrayRef<Value> coords) {
2050 SmallVector<Value> offsets;
2051 for (auto [coord, b] : llvm::zip_equal(t&: coords, u&: base))
2052 offsets.push_back(Elt: arith::AddIOp::create(builder&: rewriter, location: loc, lhs: coord, rhs: b));
2053 return offsets;
2054 };
2055
2056 // Write phase: every lane stores the pieces `input_layout` gives it.
2057 auto storeCoords =
2058 inputLayout.computeDistributedCoords(builder&: rewriter, loc, linearId: laneId, shape: sgShape);
2059 if (failed(Result: storeCoords))
2060 return rewriter.notifyMatchFailure(
2061 arg&: op, msg: "failed to compute the input_layout coordinates");
2062 SmallVector<int64_t> inputLaneLayout =
2063 inputLayout.getEffectiveLaneLayoutAsInt();
2064 SmallVector<int64_t> inputLaneData =
2065 inputLayout.getEffectiveLaneDataAsInt();
2066 Value fragment =
2067 castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getSource()),
2068 expectedTy: *distInputTy);
2069 for (auto [unit, coords] : llvm::enumerate(First&: *storeCoords)) {
2070 LaneFragmentPiece piece =
2071 getLaneFragmentPiece(unit, shape: sgShape, laneLayout: inputLaneLayout, laneData: inputLaneData);
2072 Value data = fragment;
2073 if (piece.sizes != SmallVector<int64_t>(distInputTy->getShape()))
2074 data = vector::ExtractStridedSliceOp::create(
2075 builder&: rewriter, location: loc, source: fragment, offsets: piece.offsets, sizes: piece.sizes,
2076 strides: SmallVector<int64_t>(rank, 1));
2077 xegpu::StoreMatrixOp::create(builder&: rewriter, location: loc, resultTypes: TypeRange{}, data, mem_desc: memDesc,
2078 offsets: ValueRange(withBase(coords)), const_offsets: dynamicOffsets,
2079 subgroup_block_io: nullptr, layout: xegpu::DistributeLayoutAttr{});
2080 }
2081
2082 // The scratch is only exchanged between the lanes of one subgroup, which is
2083 // a single hardware thread, so ordering the write before the read is enough
2084 // and no workgroup barrier is needed.
2085 xegpu::FenceOp::create(builder&: rewriter, location: loc, memory_kind: xegpu::MemorySpace::SLM,
2086 fence_scope: xegpu::FenceScope::Workgroup);
2087
2088 // Read phase: every lane loads the pieces `target_layout` gives it.
2089 auto loadCoords =
2090 targetLayout.computeDistributedCoords(builder&: rewriter, loc, linearId: laneId, shape: sgShape);
2091 if (failed(Result: loadCoords))
2092 return rewriter.notifyMatchFailure(
2093 arg&: op, msg: "failed to compute the target_layout coordinates");
2094 SmallVector<int64_t> targetLaneLayout =
2095 targetLayout.getEffectiveLaneLayoutAsInt();
2096 SmallVector<int64_t> targetLaneData =
2097 targetLayout.getEffectiveLaneDataAsInt();
2098 Value result;
2099 if (loadCoords->size() > 1)
2100 result = arith::ConstantOp::create(builder&: rewriter, location: loc, result: *distTargetTy,
2101 value: rewriter.getZeroAttr(type: *distTargetTy));
2102 for (auto [unit, coords] : llvm::enumerate(First&: *loadCoords)) {
2103 LaneFragmentPiece piece =
2104 getLaneFragmentPiece(unit, shape: sgShape, laneLayout: targetLaneLayout, laneData: targetLaneData);
2105 auto pieceTy = VectorType::get(shape: piece.sizes, elementType: elemTy);
2106 Value loaded = xegpu::LoadMatrixOp::create(
2107 builder&: rewriter, location: loc, res: pieceTy, mem_desc: memDesc, offsets: ValueRange(withBase(coords)),
2108 const_offsets: dynamicOffsets, subgroup_block_io: nullptr, layout: xegpu::DistributeLayoutAttr{});
2109 if (!result) {
2110 result = loaded;
2111 break;
2112 }
2113 result = vector::InsertStridedSliceOp::create(
2114 builder&: rewriter, location: loc, valueToStore: loaded, dest: result, offsets: piece.offsets,
2115 strides: SmallVector<int64_t>(rank, 1));
2116 }
2117
2118 rewriter.replaceOp(op, newValues: castValueTo(rewriter,
2119 v: cast<TypedValue<VectorType>>(Val&: result),
2120 expectedTy: *distTargetTy));
2121 return success();
2122 }
2123};
2124
2125/// `getEffectiveLaneDataAsInt` is empty when `lane_data` is unset, so the unit
2126/// check also rejects layouts that are not lane level.
2127static bool hasDefaultOrderAndUnitLaneData(xegpu::DistributeLayoutAttr layout) {
2128 if (layout.getRank() != 2)
2129 return false;
2130 return layout.getEffectiveLaneDataAsInt() == SmallVector<int64_t>{1, 1} &&
2131 layout.getEffectiveOrderAsInt() == SmallVector<int64_t>{1, 0};
2132}
2133
2134/// The quantities every element-to-lane redistribution needs, see
2135/// `matchElementLaneRedistribution`.
2136struct ElementLaneRedistribution {
2137 /// Subgroup-level type of the converted value.
2138 VectorType valueType;
2139 /// `valueType` as distributed by `input_layout` and by `target_layout`.
2140 VectorType distributedInput;
2141 VectorType distributedTarget;
2142 /// Effective `lane_layout` of `input_layout` and of `target_layout`. These
2143 /// always differ, otherwise the conversion would already have folded.
2144 SmallVector<int64_t> inputLaneLayout;
2145 SmallVector<int64_t> targetLaneLayout;
2146 int64_t subgroupSize;
2147};
2148
2149/// Recognizes an `xegpu.convert_layout` that redistributes individual elements
2150/// between the lanes of a subgroup, which is the common condition the three
2151/// slice-attributed lowerings below have.
2152/// Both input and target layouts must satisfy `hasDefaultOrderAndUnitLaneData`
2153/// and must be able to distribute the value, the two can only differ in which
2154/// lane owns which element.
2155static FailureOr<ElementLaneRedistribution>
2156matchElementLaneRedistribution(xegpu::ConvertLayoutOp op,
2157 ConversionPatternRewriter &rewriter) {
2158 xegpu::DistributeLayoutAttr inputLayout = op.getEffectiveInputLayout();
2159 xegpu::DistributeLayoutAttr targetLayout = op.getTargetLayoutAttr();
2160 if (!hasDefaultOrderAndUnitLaneData(layout: inputLayout) ||
2161 !hasDefaultOrderAndUnitLaneData(layout: targetLayout))
2162 return rewriter.notifyMatchFailure(
2163 arg&: op, msg: "both layouts must be rank 2 with effective lane_data [1, 1] and "
2164 "effective order [1, 0]");
2165
2166 auto valueType = dyn_cast<VectorType>(Val: op.getResult().getType());
2167 if (!valueType)
2168 return rewriter.notifyMatchFailure(arg&: op, msg: "value type must be a vector");
2169
2170 FailureOr<VectorType> distributedInput =
2171 xegpu::getDistVecTypeBasedOnLaneLayout(layout: inputLayout, originalType: valueType);
2172 FailureOr<VectorType> distributedTarget =
2173 xegpu::getDistVecTypeBasedOnLaneLayout(layout: targetLayout, originalType: valueType);
2174 if (failed(Result: distributedInput) || failed(Result: distributedTarget))
2175 return rewriter.notifyMatchFailure(
2176 arg&: op, msg: "value type must be distributable by both layouts");
2177
2178 const auto *uArch =
2179 xegpu::uArch::getUArch(archName: xegpu::getChipStr(op).value_or(u: ""));
2180 if (!uArch)
2181 return rewriter.notifyMatchFailure(
2182 arg&: op, msg: "target attribute is required to determine the subgroup size");
2183
2184 ElementLaneRedistribution redistribution;
2185 redistribution.valueType = valueType;
2186 redistribution.distributedInput = *distributedInput;
2187 redistribution.distributedTarget = *distributedTarget;
2188 redistribution.inputLaneLayout = inputLayout.getEffectiveLaneLayoutAsInt();
2189 redistribution.targetLaneLayout = targetLayout.getEffectiveLaneLayoutAsInt();
2190 redistribution.subgroupSize = uArch->getSubgroupSize();
2191 return redistribution;
2192}
2193
2194/// Returns the lane stride of the single distributed dimension of `slice`, i.e.
2195/// the distance in lane ids between two adjacent coordinates along that
2196/// dimension. `slice` must have exactly one non-sliced dimension whose parent
2197/// `lane_layout` extent is greater than one; the stride is the product of the
2198/// parent `lane_layout` extents that precede it in the parent `order`.
2199///
2200/// Example: for
2201/// #xegpu.slice<#xegpu.layout<lane_layout = [8, 1, 2], lane_data = [4, 1, 1],
2202/// order = [0, 2, 1]>, dims = [0]>
2203/// the only such dimension is parent dim 2, of extent 2. `order` makes dim 0
2204/// the fastest varying, with extent 8, so lanes 0..7 hold coordinate 0 and
2205/// lanes 8..15 hold coordinate 1: the stride is 8.
2206static FailureOr<int64_t> getDistributedDimLaneStride(xegpu::SliceAttr slice) {
2207 xegpu::SliceAttr flattened = slice.flatten();
2208 auto parent = dyn_cast<xegpu::LayoutAttr>(Val: flattened.getParent());
2209 if (!parent)
2210 return failure();
2211
2212 SmallVector<int64_t> parentLaneLayout = parent.getEffectiveLaneLayoutAsInt();
2213 SmallVector<int64_t> parentOrder = parent.getEffectiveOrderAsInt();
2214 if (parentLaneLayout.size() != parentOrder.size())
2215 return failure();
2216
2217 ArrayRef<int64_t> slicedDims = flattened.getDims().asArrayRef();
2218 std::optional<int64_t> distributedDim;
2219 for (int64_t dim = 0, rank = static_cast<int64_t>(parentLaneLayout.size());
2220 dim < rank; ++dim) {
2221 if (llvm::is_contained(Range&: slicedDims, Element: dim) || parentLaneLayout[dim] == 1)
2222 continue;
2223 if (distributedDim)
2224 return failure();
2225 distributedDim = dim;
2226 }
2227 if (!distributedDim)
2228 return failure();
2229
2230 int64_t stride = 1;
2231 for (int64_t dim : parentOrder) {
2232 if (dim == *distributedDim)
2233 return stride;
2234 stride *= parentLaneLayout[dim];
2235 }
2236 return failure();
2237}
2238
2239/// Distributes the slice-attributed `xegpu.convert_layout` whose source is
2240/// fully broadcast, i.e. every lane holds the whole value, onto the rows of a
2241/// subset of the lanes. Each lane keeps a single element, so no data crosses
2242/// lanes and one `vector.extract` suffices.
2243///
2244/// The input layout has effective `lane_layout` [1, 1], so the distributed
2245/// source is the whole value; the target has [n, 1], so lane `l` keeps row
2246/// `l % n` and the distributed result is `vector<1x1>`.
2247///
2248/// The source is flattened first because `xegpu-vector-linearize` cannot
2249/// linearize a `vector.extract` with a dynamic position out of a rank-2 value.
2250struct SgToLaneConvertLayoutBroadcastExtract
2251 : public OpConversionPattern<xegpu::ConvertLayoutOp> {
2252 using OpConversionPattern<xegpu::ConvertLayoutOp>::OpConversionPattern;
2253
2254 LogicalResult
2255 matchAndRewrite(xegpu::ConvertLayoutOp op, OpAdaptor adaptor,
2256 ConversionPatternRewriter &rewriter) const override {
2257 xegpu::DistributeLayoutAttr inputLayout = op.getEffectiveInputLayout();
2258 xegpu::DistributeLayoutAttr targetLayout = op.getTargetLayoutAttr();
2259
2260 if (!isa<xegpu::SliceAttr>(Val: inputLayout))
2261 return rewriter.notifyMatchFailure(arg&: op,
2262 msg: "input_layout must be #xegpu.slice");
2263 if (!isa<xegpu::LayoutAttr>(Val: targetLayout))
2264 return rewriter.notifyMatchFailure(arg&: op,
2265 msg: "target_layout must be #xegpu.layout");
2266
2267 FailureOr<ElementLaneRedistribution> redistribution =
2268 matchElementLaneRedistribution(op, rewriter);
2269 if (failed(Result: redistribution))
2270 return failure();
2271
2272 if (redistribution->inputLaneLayout != SmallVector<int64_t>{1, 1})
2273 return rewriter.notifyMatchFailure(
2274 arg&: op, msg: "input_layout effective lane_layout must be [1, 1]");
2275 SmallVector<int64_t> targetLaneLayout = redistribution->targetLaneLayout;
2276 if (targetLaneLayout[0] <= 1 || targetLaneLayout[1] != 1)
2277 return rewriter.notifyMatchFailure(
2278 arg&: op, msg: "target_layout effective lane_layout must be [n, 1] with n > 1");
2279 if (redistribution->distributedInput != redistribution->valueType)
2280 return rewriter.notifyMatchFailure(
2281 arg&: op, msg: "distributed input_layout type must equal the value type");
2282 if (redistribution->distributedTarget.getShape() != ArrayRef<int64_t>{1, 1})
2283 return rewriter.notifyMatchFailure(
2284 arg&: op, msg: "distributed target_layout type must be vector<1x1>");
2285 if (targetLaneLayout[0] > redistribution->subgroupSize)
2286 return rewriter.notifyMatchFailure(
2287 arg&: op, msg: "target_layout effective lane_layout[0] must not exceed the "
2288 "subgroup size");
2289
2290 VectorType valueType = redistribution->valueType;
2291 Location loc = op.getLoc();
2292 Value src =
2293 castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getSource()),
2294 expectedTy: redistribution->distributedInput);
2295 auto flatType = VectorType::get(shape: {valueType.getNumElements()},
2296 elementType: valueType.getElementType());
2297 Value flat = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: flatType, source: src);
2298 Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(),
2299 /*upperBound=*/upper_bound: mlir::IntegerAttr());
2300 Value rowCount =
2301 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: targetLaneLayout[0]);
2302 Value row = arith::RemUIOp::create(builder&: rewriter, location: loc, lhs: laneId, rhs: rowCount);
2303 Value element = vector::ExtractOp::create(builder&: rewriter, location: loc, source: flat,
2304 position: ArrayRef<OpFoldResult>{row});
2305 rewriter.replaceOpWithNewOp<vector::FromElementsOp>(
2306 op, args&: redistribution->distributedTarget, args&: element);
2307 return success();
2308 }
2309};
2310
2311/// Distributes the slice-attributed `xegpu.convert_layout` whose source is
2312/// broadcast over two lane groups onto the rows of a subset of the lanes.
2313/// Column `c` of row `r` is owned by lane `r + c * stride`, where `stride` is
2314/// the lane stride of the input's distributed dimension (see
2315/// `getDistributedDimLaneStride`), so each lane extracts its own element and
2316/// gathers the columns of its row with one `gpu.shuffle idx` per column.
2317///
2318/// The input layout has effective `lane_layout` [1, 2], so the distributed
2319/// source holds one column per lane group; the target has [n, 1], so lane `l`
2320/// keeps row `l % n` of both columns and the distributed result is
2321/// `vector<1x2>`.
2322///
2323/// xegpu.convert_layout %src
2324/// <{input_layout = #xegpu.slice<#xegpu.layout<lane_layout = [8, 1, 2],
2325/// lane_data = [4, 1, 1],
2326/// order = [0, 2, 1]>,
2327/// dims = [0]>,
2328/// target_layout = #xegpu.layout<lane_layout = [8, 1],
2329/// lane_data = [1, 1]>}>
2330/// : vector<8x2xf8E8M0FNU>
2331///
2332/// becomes, with lane `l` holding all 8 rows of column `l / 8`:
2333///
2334/// %flat = vector.shape_cast %src : vector<8x1xf8E8M0FNU> to
2335/// vector<8xf8E8M0FNU>
2336/// %lane = gpu.lane_id
2337/// %row = arith.remui %lane, %c8 : index
2338/// %own = vector.extract %flat[%row] : f8E8M0FNU from
2339/// vector<8xf8E8M0FNU>
2340/// %rowI32 = arith.index_cast %row : index to i32
2341/// %owner1 = arith.addi %rowI32, %c8_i32 : i32
2342/// %col0, %v0 = gpu.shuffle idx %own, %rowI32, %c16_i32 : f8E8M0FNU
2343/// %col1, %v1 = gpu.shuffle idx %own, %owner1, %c16_i32 : f8E8M0FNU
2344/// %res = vector.from_elements %col0, %col1 : vector<1x2xf8E8M0FNU>
2345///
2346/// The source is flattened first because `xegpu-vector-linearize` cannot
2347/// linearize a `vector.extract` with a dynamic position out of a rank-2 value.
2348///
2349/// All lanes gather every column, including the one they already hold, so that
2350/// lanes outside the target `lane_layout` hold replicas as partial lane layouts
2351/// require.
2352struct SgToLaneConvertLayoutPartialBroadcastExtractShuffle
2353 : public OpConversionPattern<xegpu::ConvertLayoutOp> {
2354 using OpConversionPattern<xegpu::ConvertLayoutOp>::OpConversionPattern;
2355
2356 LogicalResult
2357 matchAndRewrite(xegpu::ConvertLayoutOp op, OpAdaptor adaptor,
2358 ConversionPatternRewriter &rewriter) const override {
2359 xegpu::DistributeLayoutAttr inputLayout = op.getEffectiveInputLayout();
2360 xegpu::DistributeLayoutAttr targetLayout = op.getTargetLayoutAttr();
2361
2362 auto inputSlice = dyn_cast<xegpu::SliceAttr>(Val&: inputLayout);
2363 if (!inputSlice)
2364 return rewriter.notifyMatchFailure(arg&: op,
2365 msg: "input_layout must be #xegpu.slice");
2366 if (!isa<xegpu::LayoutAttr>(Val: targetLayout))
2367 return rewriter.notifyMatchFailure(arg&: op,
2368 msg: "target_layout must be #xegpu.layout");
2369
2370 FailureOr<ElementLaneRedistribution> redistribution =
2371 matchElementLaneRedistribution(op, rewriter);
2372 if (failed(Result: redistribution))
2373 return failure();
2374
2375 VectorType valueType = redistribution->valueType;
2376 Type elementType = valueType.getElementType();
2377 if (!elementType.isIntOrFloat() ||
2378 !llvm::is_contained(Set: {8u, 16u, 32u, 64u},
2379 Element: elementType.getIntOrFloatBitWidth()))
2380 return rewriter.notifyMatchFailure(
2381 arg&: op, msg: "element type must be an int or float of bit width 8, 16, 32 or "
2382 "64 to be carried by gpu.shuffle");
2383
2384 if (redistribution->inputLaneLayout != SmallVector<int64_t>{1, 2})
2385 return rewriter.notifyMatchFailure(
2386 arg&: op, msg: "input_layout effective lane_layout must be [1, 2]");
2387 SmallVector<int64_t> targetLaneLayout = redistribution->targetLaneLayout;
2388 if (targetLaneLayout[0] <= 1 || targetLaneLayout[1] != 1)
2389 return rewriter.notifyMatchFailure(
2390 arg&: op, msg: "target_layout effective lane_layout must be [n, 1] with n > 1");
2391 if (redistribution->distributedInput.getShape() !=
2392 ArrayRef<int64_t>{valueType.getDimSize(idx: 0), 1})
2393 return rewriter.notifyMatchFailure(
2394 arg&: op, msg: "distributed input_layout type must be vector<shape[0]x1>");
2395 if (redistribution->distributedTarget.getShape() != ArrayRef<int64_t>{1, 2})
2396 return rewriter.notifyMatchFailure(
2397 arg&: op, msg: "distributed target_layout type must be vector<1x2>");
2398
2399 FailureOr<int64_t> stride = getDistributedDimLaneStride(slice: inputSlice);
2400 if (failed(Result: stride))
2401 return rewriter.notifyMatchFailure(
2402 arg&: op, msg: "input_layout parent must have exactly one non-sliced dimension "
2403 "with lane_layout extent greater than one");
2404 // The two broadcast groups must together cover the whole subgroup, so that
2405 // lane `r + c * stride` is the lane holding column `c` of row `r`.
2406 if (*stride * redistribution->inputLaneLayout[1] !=
2407 redistribution->subgroupSize)
2408 return rewriter.notifyMatchFailure(
2409 arg&: op, msg: "input_layout distributed dimension lane stride times its extent "
2410 "must equal the subgroup size");
2411 // Lane `r + c * stride` must hold row `r`, i.e. (r + stride) %
2412 // lane_layout[0] must be r, which requires the row count to divide the
2413 // stride.
2414 if (*stride % targetLaneLayout[0] != 0)
2415 return rewriter.notifyMatchFailure(
2416 arg&: op, msg: "target_layout effective lane_layout[0] must divide the "
2417 "input_layout distributed dimension lane stride");
2418
2419 Location loc = op.getLoc();
2420 Value src =
2421 castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getSource()),
2422 expectedTy: redistribution->distributedInput);
2423 auto flatType =
2424 VectorType::get(shape: {redistribution->distributedInput.getNumElements()},
2425 elementType: valueType.getElementType());
2426 Value flat = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: flatType, source: src);
2427 Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(),
2428 /*upperBound=*/upper_bound: mlir::IntegerAttr());
2429 Value rowCount =
2430 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: targetLaneLayout[0]);
2431 Value row = arith::RemUIOp::create(builder&: rewriter, location: loc, lhs: laneId, rhs: rowCount);
2432 Value own = vector::ExtractOp::create(builder&: rewriter, location: loc, source: flat,
2433 position: ArrayRef<OpFoldResult>{row});
2434 // Gather column `c` of `row` from lane `row + c * stride`, which owns it.
2435 Type i32Type = rewriter.getI32Type();
2436 Value width = arith::ConstantIntOp::create(builder&: rewriter, location: loc, type: i32Type,
2437 value: redistribution->subgroupSize);
2438 Value rowI32 = arith::IndexCastOp::create(builder&: rewriter, location: loc, out: i32Type, in: row);
2439 Value strideI32 =
2440 arith::ConstantIntOp::create(builder&: rewriter, location: loc, type: i32Type, value: *stride);
2441 Value otherOwner = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: rowI32, rhs: strideI32);
2442 Value column0 = gpu::ShuffleOp::create(builder&: rewriter, location: loc, value: own, offset: rowI32, width,
2443 mode: gpu::ShuffleMode::IDX)
2444 .getShuffleResult();
2445 Value column1 = gpu::ShuffleOp::create(builder&: rewriter, location: loc, value: own, offset: otherOwner,
2446 width, mode: gpu::ShuffleMode::IDX)
2447 .getShuffleResult();
2448 rewriter.replaceOpWithNewOp<vector::FromElementsOp>(
2449 op, args&: redistribution->distributedTarget, args: ValueRange{column0, column1});
2450 return success();
2451 }
2452};
2453
2454/// Distributes the slice-attributed `xegpu.convert_layout` whose source is
2455/// fully broadcast and whose target splits the two columns of the value over
2456/// two lane groups. Each lane keeps one whole column, which is a stride-2
2457/// subset of the row-major source, so the two columns are separated with one
2458/// `vector.deinterleave` and the lane's group selects between them.
2459///
2460/// The input layout has effective `lane_layout` [1, 1], so the distributed
2461/// source is the whole value; the target has [1, 2] and its two groups tile the
2462/// subgroup, so the first half of the lanes keeps column 0 and the second half
2463/// column 1, one column per lane.
2464struct SgToLaneConvertLayoutDeinterleaveSelect
2465 : public OpConversionPattern<xegpu::ConvertLayoutOp> {
2466 using OpConversionPattern<xegpu::ConvertLayoutOp>::OpConversionPattern;
2467
2468 LogicalResult
2469 matchAndRewrite(xegpu::ConvertLayoutOp op, OpAdaptor adaptor,
2470 ConversionPatternRewriter &rewriter) const override {
2471 xegpu::DistributeLayoutAttr inputLayout = op.getEffectiveInputLayout();
2472 xegpu::DistributeLayoutAttr targetLayout = op.getTargetLayoutAttr();
2473
2474 if (!isa<xegpu::SliceAttr>(Val: inputLayout))
2475 return rewriter.notifyMatchFailure(arg&: op,
2476 msg: "input_layout must be #xegpu.slice");
2477 auto targetSlice = dyn_cast<xegpu::SliceAttr>(Val&: targetLayout);
2478 if (!targetSlice)
2479 return rewriter.notifyMatchFailure(arg&: op,
2480 msg: "target_layout must be #xegpu.slice");
2481
2482 FailureOr<ElementLaneRedistribution> redistribution =
2483 matchElementLaneRedistribution(op, rewriter);
2484 if (failed(Result: redistribution))
2485 return failure();
2486
2487 VectorType valueType = redistribution->valueType;
2488 if (redistribution->inputLaneLayout != SmallVector<int64_t>{1, 1})
2489 return rewriter.notifyMatchFailure(
2490 arg&: op, msg: "input_layout effective lane_layout must be [1, 1]");
2491 SmallVector<int64_t> targetLaneLayout = redistribution->targetLaneLayout;
2492 if (targetLaneLayout != SmallVector<int64_t>{1, 2})
2493 return rewriter.notifyMatchFailure(
2494 arg&: op, msg: "target_layout effective lane_layout must be [1, 2]");
2495 if (redistribution->distributedInput != valueType)
2496 return rewriter.notifyMatchFailure(
2497 arg&: op, msg: "distributed input_layout type must equal the value type");
2498 if (redistribution->distributedTarget.getShape() !=
2499 ArrayRef<int64_t>{valueType.getDimSize(idx: 0), 1})
2500 return rewriter.notifyMatchFailure(
2501 arg&: op, msg: "distributed target_layout type must be vector<shape[0]x1>");
2502
2503 FailureOr<int64_t> stride = getDistributedDimLaneStride(slice: targetSlice);
2504 if (failed(Result: stride))
2505 return rewriter.notifyMatchFailure(
2506 arg&: op, msg: "target_layout parent must have exactly one non-sliced dimension "
2507 "with lane_layout extent greater than one");
2508 // The two lane groups must together cover the whole subgroup, so that
2509 // `lane_id / stride` is the index of the column the lane keeps.
2510 if (*stride * targetLaneLayout[1] != redistribution->subgroupSize)
2511 return rewriter.notifyMatchFailure(
2512 arg&: op, msg: "target_layout distributed dimension lane stride times its "
2513 "extent must equal the subgroup size");
2514
2515 Location loc = op.getLoc();
2516 Value src =
2517 castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getSource()),
2518 expectedTy: redistribution->distributedInput);
2519 auto flatType = VectorType::get(shape: {valueType.getNumElements()},
2520 elementType: valueType.getElementType());
2521 Value flat = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: flatType, source: src);
2522 auto deinterleaved = vector::DeinterleaveOp::create(builder&: rewriter, location: loc, source: flat);
2523 Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(),
2524 /*upperBound=*/upper_bound: mlir::IntegerAttr());
2525 Value strideVal = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: *stride);
2526 Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
2527 Value group = arith::DivUIOp::create(builder&: rewriter, location: loc, lhs: laneId, rhs: strideVal);
2528 Value isFirstGroup = arith::CmpIOp::create(
2529 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq, lhs: group, rhs: zero);
2530 Value selected = arith::SelectOp::create(builder&: rewriter, location: loc, condition: isFirstGroup,
2531 true_value: deinterleaved.getResultEven(),
2532 false_value: deinterleaved.getResultOdd());
2533 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(
2534 op, args&: redistribution->distributedTarget, args&: selected);
2535 return success();
2536 }
2537};
2538
2539// Trivially distribute `vector.interleave`
2540struct SgToLaneVectorInterleave
2541 : public OpConversionPattern<vector::InterleaveOp> {
2542 using OpConversionPattern<vector::InterleaveOp>::OpConversionPattern;
2543
2544 LogicalResult
2545 matchAndRewrite(vector::InterleaveOp op, OpAdaptor adaptor,
2546 ConversionPatternRewriter &rewriter) const override {
2547
2548 auto newOp = vector::InterleaveOp::create(
2549 builder&: rewriter, location: op.getLoc(), lhs: adaptor.getLhs(), rhs: adaptor.getRhs());
2550 rewriter.replaceOp(op, newValues: newOp.getResult());
2551 return success();
2552 }
2553};
2554
2555// Trivially distribute `vector.deinterleave`
2556struct SgToLaneVectorDeinterleave
2557 : public OpConversionPattern<vector::DeinterleaveOp> {
2558 using OpConversionPattern<vector::DeinterleaveOp>::OpConversionPattern;
2559
2560 LogicalResult
2561 matchAndRewrite(vector::DeinterleaveOp op, OpAdaptor adaptor,
2562 ConversionPatternRewriter &rewriter) const override {
2563
2564 auto newOp = vector::DeinterleaveOp::create(builder&: rewriter, location: op.getLoc(),
2565 source: adaptor.getSource());
2566 rewriter.replaceOp(op, newValues: newOp.getResults());
2567 return success();
2568 }
2569};
2570
2571struct SgToLaneDpasMx : public OpConversionPattern<xegpu::DpasMxOp> {
2572 using OpConversionPattern<xegpu::DpasMxOp>::OpConversionPattern;
2573
2574 LogicalResult
2575 matchAndRewrite(xegpu::DpasMxOp op, OpAdaptor adaptor,
2576 ConversionPatternRewriter &rewriter) const override {
2577 const auto *uArch =
2578 xegpu::uArch::getUArch(archName: xegpu::getChipStr(op).value_or(u: ""));
2579 if (!uArch)
2580 return failure();
2581 if (!uArch->isSupportedInstruction(
2582 instr: xegpu::uArch::InstructionKind::SubgroupScaledMatrixMultiplyAcc))
2583 return rewriter.notifyMatchFailure(
2584 arg&: op, msg: "target uArch does not support scaled subgroup mma");
2585 // Check if the op has A, B and CD layouts attached.
2586 auto layoutA = cast<xegpu::LayoutAttr>(Val: op.getLayoutAAttr());
2587 auto layoutB = cast<xegpu::LayoutAttr>(Val: op.getLayoutBAttr());
2588 auto layoutCd = cast<xegpu::LayoutAttr>(Val: op.getLayoutCdAttr());
2589 if (!layoutA || !layoutB || !layoutCd)
2590 return rewriter.notifyMatchFailure(
2591 arg&: op, msg: "missing required layout attributes for DpasMxOp distribution");
2592
2593 // Retrieve expected types, according to anchor layouts.
2594 auto expected1DTypeResult =
2595 xegpu::getDistributedVectorType(originalType: op.getType(), layout: layoutCd);
2596 auto expected1DTypeA =
2597 xegpu::getDistributedVectorType(originalType: op.getA().getType(), layout: layoutA);
2598 auto expected1DTypeB =
2599 xegpu::getDistributedVectorType(originalType: op.getB().getType(), layout: layoutB);
2600
2601 VectorType expected1DTypeScaleA, expected1DTypeScaleB;
2602 if (op.getScaleA()) {
2603 auto layoutScaleA = cast<xegpu::LayoutAttr>(Val: op.getLayoutAScaleAttr());
2604 auto expected1DTypeScaleAOrFailure = xegpu::getDistributedVectorType(
2605 originalType: cast<VectorType>(Val: op.getScaleA().getType()), layout: layoutScaleA);
2606 if (failed(Result: expected1DTypeScaleAOrFailure))
2607 return rewriter.notifyMatchFailure(
2608 arg&: op, msg: "failed to calculate expected 1D vector type for scale A");
2609 expected1DTypeScaleA = expected1DTypeScaleAOrFailure.value();
2610 }
2611 if (op.getScaleB()) {
2612 auto layoutScaleB = cast<xegpu::LayoutAttr>(Val: op.getLayoutBScaleAttr());
2613 auto expected1DTypeScaleBOrFailure = xegpu::getDistributedVectorType(
2614 originalType: cast<VectorType>(Val: op.getScaleB().getType()), layout: layoutScaleB);
2615 if (failed(Result: expected1DTypeScaleBOrFailure))
2616 return rewriter.notifyMatchFailure(
2617 arg&: op, msg: "failed to calculate expected 1D vector type for scale B");
2618 expected1DTypeScaleB = expected1DTypeScaleBOrFailure.value();
2619 }
2620
2621 auto expectedNDTypeResult =
2622 xegpu::getDistVecTypeBasedOnLaneLayout(layout: layoutCd, originalType: op.getType());
2623 if (failed(Result: expected1DTypeResult) || failed(Result: expected1DTypeA) ||
2624 failed(Result: expected1DTypeB))
2625 return rewriter.notifyMatchFailure(
2626 arg&: op,
2627 msg: "failed to calculate supported workitem 1D vector types for DpasOp "
2628 "from layouts");
2629 if (failed(Result: expectedNDTypeResult))
2630 return rewriter.notifyMatchFailure(
2631 arg&: op, msg: "unable to compute expected workitem vector type for DpasOp from "
2632 "lane layout");
2633
2634 // Validate bit widths match uArch packed format requirements
2635 const auto *uArchInstruction = dyn_cast<
2636 xegpu::uArch::SubgroupScaledMatrixMultiplyAcc>(Val: uArch->getInstruction(
2637 instKind: xegpu::uArch::InstructionKind::SubgroupScaledMatrixMultiplyAcc));
2638 assert(uArchInstruction);
2639 auto wiAType = expected1DTypeA.value();
2640 auto wiBType = expected1DTypeB.value();
2641 // Calculate total packed bit width = element bit width * vector size
2642 unsigned aPackedBitWidth =
2643 wiAType.getElementTypeBitWidth() * wiAType.getNumElements();
2644 unsigned bPackedBitWidth =
2645 wiBType.getElementTypeBitWidth() * wiBType.getNumElements();
2646 if (aPackedBitWidth % uArchInstruction->getPackedFormatBitSizeA())
2647 return rewriter.notifyMatchFailure(
2648 arg&: op, msg: "A operand packed bit width must be a multiple of uArch packed "
2649 "format requirement");
2650 if (bPackedBitWidth % uArchInstruction->getPackedFormatBitSizeB())
2651 return rewriter.notifyMatchFailure(
2652 arg&: op, msg: "B operand packed bit width must be a multiple of uArch packed "
2653 "format requirement");
2654
2655 auto newOp = xegpu::DpasMxOp::create(
2656 builder&: rewriter, location: op->getLoc(), result: expected1DTypeResult.value(),
2657 a: castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getA()),
2658 expectedTy: expected1DTypeA.value()),
2659 b: castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getB()),
2660 expectedTy: expected1DTypeB.value()),
2661 acc: op.getAcc()
2662 ? castValueTo(rewriter,
2663 v: cast<TypedValue<VectorType>>(Val: adaptor.getAcc()),
2664 expectedTy: expected1DTypeResult.value())
2665 : nullptr,
2666
2667 scale_a: op.getScaleA()
2668 ? castValueTo(rewriter,
2669 v: cast<TypedValue<VectorType>>(Val: adaptor.getScaleA()),
2670 expectedTy: expected1DTypeScaleA)
2671 : nullptr,
2672 scale_b: op.getScaleB()
2673 ? castValueTo(rewriter,
2674 v: cast<TypedValue<VectorType>>(Val: adaptor.getScaleB()),
2675 expectedTy: expected1DTypeScaleB)
2676 : nullptr,
2677 /** layoutA**/ layout_a: nullptr,
2678 /** layoutB**/ layout_b: nullptr, /** layoutCd**/ layout_cd: nullptr,
2679 /** layoutAScale**/ layout_a_scale: nullptr, /** layoutBScale**/ layout_b_scale: nullptr);
2680 // Explicitly set the new types to enable correct type materializations.
2681 rewriter.replaceOp(op, newValues: castValueTo(rewriter, v: newOp.getResult(),
2682 expectedTy: expectedNDTypeResult.value()));
2683 return success();
2684 }
2685};
2686
2687struct XeGPUSgToLaneDistributePass
2688 : public xegpu::impl::XeGPUSgToLaneDistributeBase<
2689 XeGPUSgToLaneDistributePass> {
2690 void runOnOperation() override;
2691};
2692
2693} // namespace
2694
2695void XeGPUSgToLaneDistributePass::runOnOperation() {
2696
2697 // Recover temporary operand layouts for usage in patterns.
2698 Operation *root = getOperation();
2699 if (!xegpu::recoverTemporaryLayouts(rootOp: root)) {
2700 signalPassFailure();
2701 return;
2702 }
2703
2704 // Collect existing UnrealizedConversionCastOps. These must be preserved.
2705 llvm::SmallSetVector<UnrealizedConversionCastOp, 8> existingCasts;
2706 root->walk(
2707 callback: [&](UnrealizedConversionCastOp castOp) { existingCasts.insert(X: castOp); });
2708 // Perform a structural type conversion to convert structural ops to have WI
2709 // types. This will insert UnrealizedConversionCastOps to make the IR
2710 // valid.
2711 {
2712 ConversionTarget target(getContext());
2713 TypeConverter typeConverter;
2714 RewritePatternSet patterns(&getContext());
2715 // Source (N:1) and target (1:1) materializations using
2716 // UnrealizedConversionCastOp.
2717 auto materializeCast = [](OpBuilder &builder, Type type, ValueRange inputs,
2718 Location loc) -> Value {
2719 return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: type, operands: inputs)
2720 .getResult(i: 0);
2721 };
2722 typeConverter.addSourceMaterialization(callback&: materializeCast);
2723 typeConverter.addTargetMaterialization(callback&: materializeCast);
2724 xegpu::populateXeGPUSgToLaneDistributeTypeConversions(typeConverter, topLevelOp: root);
2725 scf::populateSCFStructuralTypeConversionsAndLegality(typeConverter,
2726 patterns, target);
2727 xegpu::populateXeGPUSgToLaneDistributeTypeConversionAndLegality(
2728 typeConverter, patterns, target, topLevelOp: root);
2729 target.addLegalOp<UnrealizedConversionCastOp>();
2730 // Fold only after the patterns. A fold builds new ops without the
2731 // temporary layout attribute, so they are never distributed. For example,
2732 // an all-ones vector.constant_mask folds to an arith.constant that stays
2733 // at subgroup size behind an unrealized cast.
2734 ConversionConfig config;
2735 config.foldingMode = DialectConversionFoldingMode::AfterPatterns;
2736 if (failed(
2737 Result: applyPartialConversion(op: root, target, patterns: std::move(patterns), config)))
2738 return signalPassFailure();
2739 }
2740 // Fold cancelling cast chains and erase dead casts.
2741 xegpu::cleanupUnrealizedConversionCasts(root, existingCasts);
2742 xegpu::removeTemporaryLayoutAttrs(op: getOperation());
2743}
2744
2745void xegpu::populateXeGPUSgToLaneDistributeTypeConversions(
2746 TypeConverter &typeConverter, Operation *topLevelOp) {
2747 // Pass through any type by default; more specific conversions registered
2748 // below override this for TensorDescType and (distributing) VectorType.
2749 typeConverter.addConversion(callback: [](Type type) -> Type { return type; });
2750 // For TensorDescType, drop the layout attribute if any.
2751 typeConverter.addConversion(callback: [](TensorDescType type) -> Type {
2752 if (type.getLayoutAttr()) {
2753 return type.dropLayouts();
2754 }
2755 return type;
2756 });
2757 // For VectorType, distribute based on the lane layout (1:1 shape-changing
2758 // conversion). Uses xegpu::addVectorTypeConversion with a pre-computed
2759 // map for SCF loop block args (see precomputeLoopBlockArgTypes for the
2760 // rationale).
2761 auto getSubShapeAndCount = [](VectorType vecTy,
2762 xegpu::DistributeLayoutAttr layout)
2763 -> std::pair<SmallVector<int64_t>, int> {
2764 auto distTyOrFailure = getDistVecTypeBasedOnLaneLayout(layout, originalType: vecTy);
2765 if (failed(Result: distTyOrFailure))
2766 return {{}, 0};
2767 return {SmallVector<int64_t>(distTyOrFailure->getShape()), 1};
2768 };
2769 auto loopArgTypes =
2770 xegpu::precomputeLoopBlockArgTypes(topLevelOp, getSubShapeAndCount);
2771 xegpu::addVectorTypeConversion(converter&: typeConverter, getSubShapeAndCount,
2772 loopArgTypes: std::move(loopArgTypes));
2773}
2774
2775void xegpu::populateXeGPUSgToLaneDistributeTypeConversionAndLegality(
2776 TypeConverter &typeConverter, RewritePatternSet &patterns,
2777 ConversionTarget &target, Operation *topLevelOp) {
2778 populateXeGPUSgToLaneDistributeTypeConversions(typeConverter, topLevelOp);
2779 // CreateNdDescOp is legal only if its result type has no layout attribute.
2780 target.addDynamicallyLegalOp<xegpu::CreateNdDescOp>(
2781 callback: [&](xegpu::CreateNdDescOp op) { return !op.getType().getLayoutAttr(); });
2782 // Any anchor XeGPU op is legal only if it has no anchor layout.
2783 target.addDynamicallyLegalDialect<xegpu::XeGPUDialect>(callback: [](Operation *op) {
2784 if (isa<xegpu::ConvertLayoutOp>(Val: op))
2785 return false;
2786 auto anchorOp = dyn_cast<AnchorLayoutInterface>(Val: op);
2787 if (!anchorOp)
2788 return true;
2789 return !anchorOp.getAnchorLayout();
2790 });
2791 // Arith constants are legal only if they have no temporary layout attribute.
2792 target.addDynamicallyLegalOp<arith::ConstantOp>(
2793 callback: [=](arith::ConstantOp op) -> bool {
2794 // If the result type is not a vector, it's legal.
2795 if (!isa<VectorType>(Val: op.getResult().getType()))
2796 return true;
2797 return !xegpu::getTemporaryLayout(operandOrResult: dyn_cast<OpResult>(Val: op.getResult()));
2798 });
2799 // In math and arith dialects, only handle elementwise ops with a single
2800 // result and with a result layout attribute.
2801 target.addDynamicallyLegalDialect<math::MathDialect, arith::ArithDialect>(
2802 callback: [=](Operation *op) -> std::optional<bool> {
2803 // Only handle elementwise mappable ops
2804 if (!OpTrait::hasElementwiseMappableTraits(op))
2805 return true;
2806 // Only handle ops with single vector result
2807 if (op->getNumResults() != 1)
2808 return true;
2809
2810 VectorType resultType =
2811 dyn_cast<VectorType>(Val: op->getResult(idx: 0).getType());
2812 if (!resultType)
2813 return true;
2814
2815 // Check if all operands are vectors of the same shape
2816 for (Value operand : op->getOperands()) {
2817 VectorType operandType = dyn_cast<VectorType>(Val: operand.getType());
2818 if (!operandType || operandType.getShape() != resultType.getShape()) {
2819 return true;
2820 }
2821 }
2822 return !xegpu::getTemporaryLayout(operandOrResult: dyn_cast<OpResult>(Val: op->getResult(idx: 0)));
2823 });
2824 // vector::ReductionOp is legal only if its source has no distribute layout
2825 // attribute.
2826 target.addDynamicallyLegalOp<vector::ReductionOp>(
2827 callback: [=](vector::ReductionOp op) -> bool {
2828 auto layout = xegpu::getDistributeLayoutAttr(value: op.getVector());
2829 return !layout;
2830 });
2831 // vector::MultiDimReductionOp op legality.
2832 target.addDynamicallyLegalOp<vector::MultiDimReductionOp>(
2833 callback: [=](vector::MultiDimReductionOp op) -> bool {
2834 return !isValidSubgroupMultiReductionOp(op);
2835 });
2836 target.addDynamicallyLegalOp<vector::CreateMaskOp, vector::ConstantMaskOp,
2837 vector::TransposeOp, vector::BitCastOp,
2838 vector::ShapeCastOp, vector::StepOp,
2839 vector::BroadcastOp>(callback: [=](Operation *op) -> bool {
2840 return !xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0));
2841 });
2842 target.addDynamicallyLegalOp<vector::ExtractOp>(
2843 callback: [=](vector::ExtractOp op) -> bool {
2844 if (!isa<VectorType>(Val: op.getType()))
2845 return true;
2846 return !xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0));
2847 });
2848 target.addDynamicallyLegalOp<vector::InsertOp>(
2849 callback: [=](vector::InsertOp op) -> bool {
2850 return !xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0));
2851 });
2852 target.addDynamicallyLegalOp<vector::ExtractStridedSliceOp>(
2853 callback: [=](vector::ExtractStridedSliceOp op) -> bool {
2854 return !xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0));
2855 });
2856 target.addDynamicallyLegalOp<vector::InsertStridedSliceOp>(
2857 callback: [=](vector::InsertStridedSliceOp op) -> bool {
2858 return !xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0));
2859 });
2860 target.addDynamicallyLegalOp<vector::InterleaveOp, vector::DeinterleaveOp>(
2861 callback: [=](Operation *op) -> bool {
2862 return !xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0));
2863 });
2864 target.markUnknownOpDynamicallyLegal(fn: [](Operation *op) { return true; });
2865 patterns
2866 .add<SgToLaneCreateNdDesc, SgToLaneLoadNd, SgToLaneStoreNd, SgToLaneDpas,
2867 SgToLaneElementWise, SgToLaneArithConstant, SgToLanePrefetchNd,
2868 SgToLaneLoadGather, SgToLaneStoreScatter, SgToLaneVectorReduction,
2869 SgToLaneMultiDimReduction, SgToLaneVectorExtract,
2870 SgToLaneVectorInsert, SgToLaneVectorExtractStridedSlice,
2871 SgToLaneVectorInsertStridedSlice, SgToLaneLoadMatrix,
2872 SgToLaneStoreMatrix, SgToLaneConvertLayout, SgToLaneVectorTranspose,
2873 SgToLaneVectorBitcast, SgToLaneVectorStep, SgToLaneVectorShapeCast,
2874 SgToLaneBroadcast, SgToLaneCreateMask<vector::CreateMaskOp>,
2875 SgToLaneCreateMask<vector::ConstantMaskOp>,
2876 SgToLaneVectorDeinterleave, SgToLaneVectorInterleave, SgToLaneDpasMx,
2877 SgToLaneConvertLayoutBroadcastExtract,
2878 SgToLaneConvertLayoutPartialBroadcastExtractShuffle,
2879 SgToLaneConvertLayoutDeinterleaveSelect>(arg&: typeConverter,
2880 args: patterns.getContext());
2881 // The SLM round-trip handles any pair of layouts but pays for a memory
2882 // round-trip, so it only runs when every pattern above has declined.
2883 patterns.add<SgToLaneConvertLayoutViaSLM>(arg&: typeConverter,
2884 args: patterns.getContext(),
2885 /*benefit=*/args: PatternBenefit(0));
2886}
2887