1//===- XeGPUWgToSgDistribute.cpp - XeGPU Workgroup to Subgroup Pass -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#include "mlir/Dialect/XeGPU/Transforms/Passes.h"
9
10#include "mlir/Dialect/Affine/Utils.h"
11#include "mlir/Dialect/Arith/IR/Arith.h"
12#include "mlir/Dialect/Arith/Utils/Utils.h"
13#include "mlir/Dialect/GPU/IR/GPUDialect.h"
14#include "mlir/Dialect/Index/IR/IndexDialect.h"
15#include "mlir/Dialect/Index/IR/IndexOps.h"
16#include "mlir/Dialect/Math/IR/Math.h"
17#include "mlir/Dialect/MemRef/IR/MemRef.h"
18#include "mlir/Dialect/SCF/Transforms/Patterns.h"
19#include "mlir/Dialect/Utils/IndexingUtils.h"
20#include "mlir/Dialect/XeGPU/IR/XeGPU.h"
21#include "mlir/Dialect/XeGPU/Transforms/Transforms.h"
22#include "mlir/Dialect/XeGPU/Transforms/XeGPULayoutImpl.h"
23#include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
24#include "mlir/Transforms/DialectConversion.h"
25#include "llvm/ADT/SetVector.h"
26#include <optional>
27
28namespace mlir {
29namespace xegpu {
30#define GEN_PASS_DEF_XEGPUWGTOSGDISTRIBUTE
31#include "mlir/Dialect/XeGPU/Transforms/Passes.h.inc"
32} // namespace xegpu
33} // namespace mlir
34
35using namespace mlir;
36
37namespace {
38
39// Retrieve the RangeAttr if it is specified.
40static xegpu::RangeAttr getRangeSpecAttr(Operation *op) {
41 Operation *parent = op->getParentOfType<scf::IfOp>();
42 while (parent) {
43 if (auto attr = llvm::dyn_cast_if_present<xegpu::RangeAttr>(
44 Val: parent->getDiscardableAttr(name: "sg_id_range")))
45 return attr;
46 parent = parent->getParentOfType<scf::IfOp>();
47 }
48 return {};
49}
50
51static std::pair<SmallVector<int64_t>, int>
52getSgShapeAndCount(ArrayRef<int64_t> shape,
53 xegpu::DistributeLayoutAttr layout) {
54 int count = 1;
55 SmallVector<int64_t> sgShape(shape);
56 auto distributedShape = layout.computeDistributedShape(
57 shape: SmallVector<int64_t>(shape.begin(), shape.end()));
58 if (failed(Result: distributedShape))
59 return std::make_pair(x&: sgShape, y&: count);
60 auto sgData = layout.getEffectiveSgDataAsInt();
61 count = computeProduct(basis: distributedShape.value()) / computeProduct(basis: sgData);
62 return std::make_pair(x&: sgData, y&: count);
63}
64
65/// Utility helper for deriving a list of offsets for each sub-TensorDescs
66/// or sub-MemDescs to be accessed by current subgroup (sgId) based on the
67/// associated distribute layout attribute, the shape, subgroup id and the
68/// original offsets of the op
69template <typename OpType,
70 typename = std::enable_if_t<llvm::is_one_of<
71 OpType, xegpu::LoadNdOp, xegpu::StoreNdOp, xegpu::PrefetchNdOp,
72 xegpu::LoadMatrixOp, xegpu::StoreMatrixOp>::value>>
73static LogicalResult
74genOffsetsList(ConversionPatternRewriter &rewriter, OpType op,
75 SmallVector<SmallVector<OpFoldResult>> &offsetsList) {
76 Location loc = op.getLoc();
77 SmallVector<OpFoldResult> origOffsets = op.getMixedOffsets();
78 // not applicable to ops without offsets operands.
79 if (origOffsets.empty())
80 return failure();
81
82 // if op is xegpu::CreateNdDescOp, call op.getDescLayoutAttr()
83 xegpu::DistributeLayoutAttr layout;
84 if constexpr (std::is_same_v<OpType, xegpu::LoadMatrixOp> ||
85 std::is_same_v<OpType, xegpu::StoreMatrixOp>) {
86 layout = op.getLayoutAttr();
87 } else {
88 layout = op.getDescLayoutAttr();
89 }
90
91 // not applicable to ops without workgroup layout attributes
92 if (!layout || !layout.isForWorkgroup())
93 return failure();
94
95 Value sgId =
96 gpu::SubgroupIdOp::create(builder&: rewriter, location: loc, /*upper_bound=*/nullptr);
97
98 // verify and adjust the sgId if the range specifier is present
99 xegpu::RangeAttr sgIdRange = getRangeSpecAttr(op);
100 if (sgIdRange) {
101 int64_t startOfRange = sgIdRange.getStart().getInt();
102 int64_t endOfRange = sgIdRange.getEnd().getInt();
103 // verify the RangeAttr against the layout attribute
104 if (layout.getNumSubgroups() != endOfRange - startOfRange)
105 return rewriter.notifyMatchFailure(
106 op, "sg_layout size must match the sg_id_range");
107 // adjust the sgId if necessary
108 if (startOfRange > 0) {
109 Value startOfRangeVal =
110 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: startOfRange);
111 sgId = index::SubOp::create(builder&: rewriter, location: loc, lhs: sgId, rhs: startOfRangeVal);
112 }
113 }
114
115 // Compute the list of subgroup-relative offsets for sub-tensors or sub-memory
116 // descriptors to be accessed, based on the layout information.
117 ArrayRef<int64_t> wgShape = op.getDataShape();
118 auto maybeDescOffsets =
119 layout.computeDistributedCoords(builder&: rewriter, loc, linearId: sgId, shape: wgShape);
120 if (failed(Result: maybeDescOffsets))
121 return failure();
122
123 // Compute the final global offsets for each accessed sub-tensor
124 // or sub-memory descriptor.
125 for (const auto &sgOffsets : *maybeDescOffsets) {
126 SmallVector<OpFoldResult> newOffsets = xegpu::addWithRightAligned(
127 builder&: rewriter, loc, lhs: getAsOpFoldResult(values: sgOffsets), rhs: origOffsets);
128 offsetsList.push_back(Elt: std::move(newOffsets));
129 }
130
131 // callback(offsetsList);
132 return success();
133}
134
135/// This pattern transforms the CreateNdDescOp to create a subgroup descriptor
136/// from a workgroup descriptor. It replaces the offsets and sizes with
137/// appropriate values for the subgroup.
138/// It uses round-robin assignment to distribute the work to the subgroups.
139/// Following create_nd_desc operation:
140/// %tdesc = xegpu.create_nd_tdesc %src : memref<24x24xf32>
141/// -> !xegpu.tensor_desc<24x24xf32, #xegpu.layout<sg_layout = [4, 4],
142/// sg_data = [2, 2], lane_layout = [2, 2], lane_data = [1, 1]>>
143/// is converted to 9 subgroup level operations based on the sg_layout &
144/// sg_data:
145/// %tdesc = xegpu.create_nd_tdesc %src : memref<24x24xf32> ->
146/// !xegpu.tensor_desc<2x2xf32, #xegpu.layout<lane_layout = [2, 2],
147/// lane_data = [1, 1]>>
148///
149/// The sg_layout and sg_data attributes are dropped after the pass as they are
150/// no longer needed.
151///
152/// 24x24 matrix distribution example:
153/// sg_layout = [4, 4], sg_data = [2, 2]
154/// Each 8x8 matrix within the 24x24 matrix is called a distribution unit.
155/// dist_unit_shape = [8, 8] --> sg_layout[i] * sg_data[i]
156///
157/// +------------------------+
158/// | 8x8 | 8x8 | 8x8 | <- 3 tiles across
159/// |-----+-----+-----|
160/// | 8x8 | 8x8 | 8x8 | <- 3 tiles down
161/// |-----+-----+-----|
162/// | 8x8 | 8x8 | 8x8 |
163/// +------------------------+
164///
165/// Each 8x8 tile is further subdivided among subgroups:
166/// +------------------------+
167/// | 2x2 2x2 2x2 2x2 | <- 4 subgroups across (each handles 2 columns)
168/// | 2x2 2x2 2x2 2x2 | <- 4 subgroups down (each handles 2 rows)
169/// | 2x2 2x2 2x2 2x2 |
170/// | 2x2 2x2 2x2 2x2 |
171/// +------------------------+
172///
173/// Since the 24x24 matrix is divided into 8x8 distribution units, there will be
174/// 9 distribution units (3x3) in total. Hence the 9 subgroup level operations.
175
176/// The pass currently has entire distribution logic in the WgToSgCreateNdOp
177/// pattern and all the other ops just follow.
178/// TODO: Decouple the distribution logic from WgToSgCreateNdOp for all the
179/// ops in the pass.
180// This pattern transforms the CreateNdDescOp to create a
181// subgroup descriptor from a workgroup descriptor.
182struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
183 using OpConversionPattern<xegpu::CreateNdDescOp>::OpConversionPattern;
184
185 LogicalResult
186 matchAndRewrite(xegpu::CreateNdDescOp op, OneToNOpAdaptor adaptor,
187 ConversionPatternRewriter &rewriter) const override {
188
189 Location loc = op.getLoc();
190 MLIRContext *ctx = op.getContext();
191 xegpu::TensorDescType tdescTy = op.getType();
192 auto layout = dyn_cast<xegpu::DistributeLayoutAttr>(Val: tdescTy.getLayout());
193 if (!layout || !layout.isForWorkgroup())
194 return failure();
195
196 Type elemTy = tdescTy.getElementType();
197 ArrayRef<int64_t> wgShape = tdescTy.getShape();
198
199 SmallVector<int64_t> sgShape;
200 int count;
201 std::tie(args&: sgShape, args&: count) = getSgShapeAndCount(shape: wgShape, layout);
202 xegpu::TensorDescType newTdescTy =
203 xegpu::TensorDescType::get(context: ctx, shape: sgShape, elementType: elemTy, encoding: tdescTy.getEncoding(),
204 layout: layout.dropSgLayoutAndData());
205
206 Value src = op.getSource();
207 SmallVector<Value> newCreateNdOps(count);
208 std::generate(first: newCreateNdOps.begin(), last: newCreateNdOps.end(), gen: [&]() -> Value {
209 if (isa<MemRefType>(Val: src.getType()))
210 return xegpu::CreateNdDescOp::create(builder&: rewriter, location: loc, tdesc: newTdescTy,
211 source: cast<TypedValue<MemRefType>>(Val&: src));
212 return xegpu::CreateNdDescOp::create(builder&: rewriter, location: loc, tdesc: newTdescTy, source: src,
213 shape: op.getMixedSizes(),
214 strides: op.getMixedStrides());
215 });
216
217 rewriter.replaceOpWithMultiple(op, newValues: {newCreateNdOps});
218 return success();
219 }
220};
221
222/// This pattern transforms the LoadNdOp to load subgroup data.
223struct WgToSgLoadNdOp : public OpConversionPattern<xegpu::LoadNdOp> {
224 using OpConversionPattern<xegpu::LoadNdOp>::OpConversionPattern;
225 LogicalResult
226 matchAndRewrite(xegpu::LoadNdOp op, OneToNOpAdaptor adaptor,
227 ConversionPatternRewriter &rewriter) const override {
228
229 SmallVector<SmallVector<OpFoldResult>> offsetsList;
230 if (failed(Result: genOffsetsList(rewriter, op, offsetsList)))
231 return failure();
232
233 xegpu::DistributeLayoutAttr layout = op.getLayoutAttr();
234 if (layout)
235 layout = layout.dropSgLayoutAndData();
236 SmallVector<Value> newOps;
237 for (auto [tdesc, offsets] :
238 llvm::zip(t: adaptor.getTensorDesc(), u&: offsetsList)) {
239 auto tdescTy = dyn_cast<xegpu::TensorDescType>(Val: tdesc.getType());
240 VectorType newResTy =
241 VectorType::get(shape: tdescTy.getShape(), elementType: tdescTy.getElementType());
242 auto newOp = xegpu::LoadNdOp::create(
243 builder&: rewriter, location: op.getLoc(), value: newResTy, TensorDesc: tdesc, offsets,
244 /*packed = */ nullptr, /*transpose = */ nullptr, l1_hint: op.getL1HintAttr(),
245 l2_hint: op.getL2HintAttr(), l3_hint: op.getL3HintAttr(), layout);
246 newOps.push_back(Elt: newOp);
247 }
248 rewriter.replaceOpWithMultiple(op, newValues: {newOps});
249
250 return success();
251 }
252};
253
254/// This pattern transforms the StoreNdOp to store subgroup data.
255struct WgToSgStoreNdOp : public OpConversionPattern<xegpu::StoreNdOp> {
256 using OpConversionPattern<xegpu::StoreNdOp>::OpConversionPattern;
257 LogicalResult
258 matchAndRewrite(xegpu::StoreNdOp op, OneToNOpAdaptor adaptor,
259 ConversionPatternRewriter &rewriter) const override {
260 SmallVector<SmallVector<OpFoldResult>> offsetsList;
261 if (failed(Result: genOffsetsList(rewriter, op, offsetsList)))
262 return failure();
263
264 xegpu::DistributeLayoutAttr layout = op.getLayoutAttr();
265 if (layout)
266 layout = layout.dropSgLayoutAndData();
267 for (auto [v, tdesc, offsets] :
268 llvm::zip(t: adaptor.getValue(), u: adaptor.getTensorDesc(), args&: offsetsList)) {
269 xegpu::StoreNdOp::create(builder&: rewriter, location: op.getLoc(), value: v, TensorDesc: tdesc, offsets,
270 l1_hint: op.getL1HintAttr(), l2_hint: op.getL2HintAttr(),
271 l3_hint: op.getL3HintAttr(), layout);
272 }
273 rewriter.eraseOp(op);
274
275 return success();
276 }
277};
278
279/// This pattern transforms the PrefetchNdOp to prefetch subgroup data.
280struct WgToSgPrefetchNdOp : public OpConversionPattern<xegpu::PrefetchNdOp> {
281 using OpConversionPattern<xegpu::PrefetchNdOp>::OpConversionPattern;
282 LogicalResult
283 matchAndRewrite(xegpu::PrefetchNdOp op, OneToNOpAdaptor adaptor,
284 ConversionPatternRewriter &rewriter) const override {
285 SmallVector<SmallVector<OpFoldResult>> offsetsList;
286 if (failed(Result: genOffsetsList(rewriter, op, offsetsList)))
287 return failure();
288
289 xegpu::DistributeLayoutAttr layout = op.getLayoutAttr();
290 if (layout)
291 layout = layout.dropSgLayoutAndData();
292 for (auto [tdesc, offsets] :
293 llvm::zip(t: adaptor.getTensorDesc(), u&: offsetsList)) {
294 xegpu::PrefetchNdOp::create(builder&: rewriter, location: op.getLoc(), TensorDesc: tdesc, offsets,
295 l1_hint: op.getL1HintAttr(), l2_hint: op.getL2HintAttr(),
296 l3_hint: op.getL3HintAttr(), layout);
297 }
298 rewriter.eraseOp(op);
299
300 return success();
301 }
302};
303
304/// This pattern transforms the DpasOp to work at subgroup level.
305struct WgToSgDpasOp : public OpConversionPattern<xegpu::DpasOp> {
306 using OpConversionPattern<xegpu::DpasOp>::OpConversionPattern;
307 LogicalResult
308 matchAndRewrite(xegpu::DpasOp op, OneToNOpAdaptor adaptor,
309 ConversionPatternRewriter &rewriter) const override {
310 Location loc = op.getLoc();
311 VectorType resultTy = op.getResult().getType();
312 if (resultTy.getRank() < 2)
313 return failure();
314
315 auto layoutCd = op.getLayoutCdAttr();
316 auto layoutA = op.getLayoutAAttr();
317 auto layoutB = op.getLayoutBAttr();
318 if (!layoutCd || !layoutA || !layoutB)
319 return failure();
320 size_t i = 0;
321 SmallVector<Value> newDpasOps;
322 for (auto aVec : adaptor.getLhs()) {
323 for (auto bVec : adaptor.getRhs()) {
324
325 Value tmpC;
326 if (op.getAcc())
327 tmpC = adaptor.getAcc()[i++];
328
329 ArrayRef<int64_t> aVecShape =
330 cast<VectorType>(Val: aVec.getType()).getShape();
331 ArrayRef<int64_t> bVecShape =
332 cast<VectorType>(Val: bVec.getType()).getShape();
333 // Build result shape: batch dims from A + [M, N] from last dims of
334 // A and B.
335 SmallVector<int64_t> resShape(aVecShape.drop_back(N: 2));
336 resShape.push_back(Elt: aVecShape[aVecShape.size() - 2]);
337 resShape.push_back(Elt: bVecShape[bVecShape.size() - 1]);
338 VectorType resTy = VectorType::get(shape: resShape, elementType: resultTy.getElementType());
339 auto newDpasOp = xegpu::DpasOp::create(
340 builder&: rewriter, location: loc, result: resTy, lhs: aVec, rhs: bVec, acc: tmpC,
341 /*layout_a=*/nullptr, /*layout_b=*/nullptr, /*layout_cd=*/nullptr);
342 newDpasOp.setLayoutCdAttr(layoutCd.dropSgLayoutAndData());
343 newDpasOp.setLayoutAAttr(layoutA.dropSgLayoutAndData());
344 newDpasOp.setLayoutBAttr(layoutB.dropSgLayoutAndData());
345
346 newDpasOps.push_back(Elt: newDpasOp);
347 }
348 }
349 rewriter.replaceOpWithMultiple(op, newValues: {newDpasOps});
350 return success();
351 }
352};
353
354/// This pattern transforms the DpasMxOp to work at subgroup level.
355struct WgToSgDpasMxOp : public OpConversionPattern<xegpu::DpasMxOp> {
356 using OpConversionPattern<xegpu::DpasMxOp>::OpConversionPattern;
357 LogicalResult
358 matchAndRewrite(xegpu::DpasMxOp op, OneToNOpAdaptor adaptor,
359 ConversionPatternRewriter &rewriter) const override {
360
361 Location loc = op.getLoc();
362 VectorType resultTy = op.getResult().getType();
363
364 if (resultTy.getRank() < 2)
365 return failure();
366
367 auto layoutCd = op.getLayoutCdAttr();
368 auto layoutA = op.getLayoutAAttr();
369 auto layoutB = op.getLayoutBAttr();
370 auto layoutAScale = op.getLayoutAScaleAttr();
371 auto layoutBScale = op.getLayoutBScaleAttr();
372
373 if (!layoutCd || !layoutA || !layoutB || !layoutAScale || !layoutBScale)
374 return failure();
375
376 size_t index_c = 0;
377 SmallVector<Value> newDpasMxOps;
378 for (auto [index_a, aVec] : llvm::enumerate(First: adaptor.getA())) {
379 for (auto [index_b, bVec] : llvm::enumerate(First: adaptor.getB())) {
380 Value accVal = (op.getAcc()) ? adaptor.getAcc()[index_c++] : Value();
381 Value scaleAVal =
382 (op.getScaleA()) ? adaptor.getScaleA()[index_a] : Value();
383 Value scaleBVal =
384 (op.getScaleB()) ? adaptor.getScaleB()[index_b] : Value();
385
386 ArrayRef<int64_t> aVecShape =
387 cast<VectorType>(Val: aVec.getType()).getShape();
388 ArrayRef<int64_t> bVecShape =
389 cast<VectorType>(Val: bVec.getType()).getShape();
390 // Build result shape: batch dims from A + [M, N]
391 SmallVector<int64_t> resShape(aVecShape.drop_back(N: 2));
392 resShape.push_back(Elt: aVecShape[aVecShape.size() - 2]);
393 resShape.push_back(Elt: bVecShape[bVecShape.size() - 1]);
394 VectorType resTy = VectorType::get(shape: resShape, elementType: resultTy.getElementType());
395 auto newDpasMxOp = xegpu::DpasMxOp::create(
396 builder&: rewriter, location: loc, result: resTy, a: aVec, b: bVec, acc: accVal, scale_a: scaleAVal, scale_b: scaleBVal,
397 layout_a: layoutA.dropSgLayoutAndData(), layout_b: layoutB.dropSgLayoutAndData(),
398 layout_cd: layoutCd.dropSgLayoutAndData(), layout_a_scale: layoutAScale.dropSgLayoutAndData(),
399 layout_b_scale: layoutBScale.dropSgLayoutAndData());
400
401 newDpasMxOps.push_back(Elt: newDpasMxOp);
402 }
403 }
404 rewriter.replaceOpWithMultiple(op, newValues: {newDpasMxOps});
405 return success();
406 }
407};
408
409/// This pattern transforms vector.broadcast ops to work at subgroup level.
410struct WgToSgVectorBroadcastOp
411 : public OpConversionPattern<vector::BroadcastOp> {
412 using OpConversionPattern<vector::BroadcastOp>::OpConversionPattern;
413
414 LogicalResult
415 matchAndRewrite(vector::BroadcastOp op, OneToNOpAdaptor adaptor,
416 ConversionPatternRewriter &rewriter) const override {
417
418 VectorType resultType = op.getResult().getType();
419 ArrayRef<int64_t> wgShape = resultType.getShape();
420
421 xegpu::DistributeLayoutAttr layout =
422 xegpu::getTemporaryLayout(operandOrResult: llvm::cast<OpResult>(Val: op.getResult()));
423 if (!layout || !layout.isForWorkgroup())
424 return failure();
425
426 SmallVector<int64_t> sgShape;
427 int count;
428 std::tie(args&: sgShape, args&: count) = getSgShapeAndCount(shape: wgShape, layout);
429 VectorType newResultType =
430 VectorType::get(shape: sgShape, elementType: resultType.getElementType());
431
432 SmallVector<Value> newBroadcastOps;
433 auto distSource = adaptor.getOperands().front();
434 int numDistributions = count / distSource.size();
435 for (int i = 0; i < numDistributions; ++i) {
436 for (auto operand : distSource) {
437 auto newBroadcast = vector::BroadcastOp::create(builder&: rewriter, location: op.getLoc(),
438 result: newResultType, source: operand);
439
440 newBroadcastOps.push_back(Elt: newBroadcast.getResult());
441 }
442 }
443 rewriter.replaceOpWithMultiple(op, newValues: {newBroadcastOps});
444 return success();
445 }
446};
447
448// This pattern transforms elementwise ops to work at subgroup level.
449struct WgToSgElementwiseOp : public ConversionPattern {
450 WgToSgElementwiseOp(MLIRContext *ctx)
451 : ConversionPattern(MatchAnyOpTypeTag(), /*benefit=*/1, ctx) {}
452
453 LogicalResult
454 matchAndRewrite(Operation *op, ArrayRef<ValueRange> operands,
455 ConversionPatternRewriter &rewriter) const override {
456 // Only match ops with elementwise trait and single result.
457 if (!OpTrait::hasElementwiseMappableTraits(op) || op->getNumResults() != 1)
458 return failure();
459
460 auto resultType = dyn_cast<VectorType>(Val: op->getResult(idx: 0).getType());
461 assert(resultType && "Expected result to be a VectorType");
462
463 ArrayRef<int64_t> wgShape = resultType.getShape();
464
465 xegpu::DistributeLayoutAttr layout =
466 xegpu::getTemporaryLayout(operandOrResult: llvm::cast<OpResult>(Val: op->getResult(idx: 0)));
467 if (!layout || !layout.isForWorkgroup())
468 return failure();
469
470 SmallVector<int64_t> sgShape = getSgShapeAndCount(shape: wgShape, layout).first;
471
472 size_t numVariants = operands.empty() ? 0 : operands.front().size();
473
474 if (llvm::any_of(Range&: operands, P: [&](const ValueRange &operandVec) {
475 return operandVec.size() != numVariants;
476 }))
477 return failure();
478
479 SmallVector<Value> newResults;
480 VectorType newResultType =
481 VectorType::get(shape: sgShape, elementType: resultType.getElementType());
482
483 for (size_t i = 0; i < numVariants; ++i) {
484 SmallVector<Value> opOperands;
485 for (auto &operandVec : operands)
486 opOperands.push_back(Elt: operandVec[i]);
487
488 OperationState state(op->getLoc(), op->getName());
489 state.addOperands(newOperands: opOperands);
490 state.addTypes(newTypes: newResultType);
491 state.addAttributes(newAttributes: op->getDiscardableAttrDictionary().getValue());
492 state.propertiesAttr = op->getPropertiesAsAttribute();
493 Operation *newOp = rewriter.create(state);
494 xegpu::removeLayoutAttrs(op: newOp);
495 newResults.push_back(Elt: newOp->getResult(idx: 0));
496 }
497
498 rewriter.replaceOpWithMultiple(op, newValues: {newResults});
499 return success();
500 }
501};
502
503// clang-format off
504// Pattern for lowering ConvertLayoutOp based on sg_layout and sg_data.
505// If input_layout and target_layout have identical sg_layout and sg_data,
506// the op is rewritten to a subgroup-level ConvertLayoutOp with these fields
507// dropped. For example:
508// #a = #xegpu.layout<sg_layout = [2, 2], sg_data = [16, 16], inst_data = [16, 16]>
509// #b = #xegpu.layout<sg_layout = [2, 2], sg_data = [16, 16], inst_data = [8, 16]>
510// xegpu.convert_layout %1 <{input_layout = #a, target_layout = #b}> : vector<32x64xf32>
511// becomes:
512// #a = #xegpu.layout<inst_data = [16, 16]>
513// #b = #xegpu.layout<inst_data = [8, 16]>
514// xegpu.convert_layout %1 <{input_layout = #a, target_layout = #b}> : vector<16x16xf32>
515// (vector<16x16xf32> is determined by sg_data = [16, 16])
516//
517// If sg_layout or sg_data differ, SLM is used to redistribute data across subgroups.
518// For example:
519// #a = #xegpu.layout<sg_layout = [1, 4], sg_data = [32, 16], inst_data = [16, 16]>
520// #b = #xegpu.layout<sg_layout = [2, 2], sg_data = [16, 32], inst_data = [8, 16]>
521// xegpu.convert_layout %1 <{input_layout = #a, target_layout = #b}> : vector<32x64xf32>
522// is lowered to:
523// #a = #xegpu.layout<inst_data = [16, 16]>
524// #b = #xegpu.layout<inst_data = [8, 16]>
525// store_matrix %1, %slm <{layout_input_0 = #a}> : vector<32x16>, mem_desc<32x64xf32>
526// %d = load_matrix %slm <{layout_result_0 = #a}> : mem_desc<32x64xf32> -> vector<16x32xf32>
527// xegpu.convert_layout %d <{input_layout = #a, target_layout = #b}> : vector<16x32xf32>
528// clang-format on
529struct WgToSgConvertLayoutOp
530 : public OpConversionPattern<xegpu::ConvertLayoutOp> {
531 using OpConversionPattern<xegpu::ConvertLayoutOp>::OpConversionPattern;
532
533 LogicalResult
534 matchAndRewrite(xegpu::ConvertLayoutOp op, OneToNOpAdaptor adaptor,
535 ConversionPatternRewriter &rewriter) const override {
536 Location loc = op.getLoc();
537 auto inputLayout = op.getEffectiveInputLayout();
538 auto targetLayout = op.getTargetLayout();
539
540 if (!inputLayout || !targetLayout || !inputLayout.isForWorkgroup() ||
541 !targetLayout.isForWorkgroup())
542 return rewriter.notifyMatchFailure(
543 arg&: op, msg: "Input and target layouts must have subgroup layout");
544
545 Type resultType = op.getResult().getType();
546 if (resultType.isIntOrFloat()) {
547 rewriter.replaceOp(op, newValues: op.getSource());
548 assert(!inputLayout.dropSgLayoutAndData() &&
549 !targetLayout.dropSgLayoutAndData() &&
550 "unexpected layout attributes for scalar type");
551 return success();
552 }
553
554 ArrayRef<int64_t> wgShape = cast<VectorType>(Val&: resultType).getShape();
555 SmallVector<int64_t> inputSgLayout =
556 inputLayout.getEffectiveSgLayoutAsInt();
557 SmallVector<int64_t> inputSgData = inputLayout.getEffectiveSgDataAsInt();
558 SmallVector<int64_t> targetSgLayout =
559 targetLayout.getEffectiveSgLayoutAsInt();
560 SmallVector<int64_t> targetSgData = targetLayout.getEffectiveSgDataAsInt();
561
562 // Fast path: if sg_layout and sg_data are identical, no SLM needed
563 SmallVector<int64_t> wgShapeVec(wgShape.begin(), wgShape.end());
564 if (inputLayout.isCompatibleWith(other: targetLayout, shape: wgShapeVec,
565 level: xegpu::LayoutKind::Subgroup)) {
566 inputLayout = inputLayout.dropSgLayoutAndData();
567 targetLayout = targetLayout.dropSgLayoutAndData();
568
569 SmallVector<Value> newOps(adaptor.getSource());
570 if (inputLayout && targetLayout) {
571 for (auto [i, src] : llvm::enumerate(First: adaptor.getSource())) {
572 auto newOp = xegpu::ConvertLayoutOp::create(
573 builder&: rewriter, location: loc, result: src.getType(), source: src, input_layout: inputLayout, target_layout: targetLayout);
574 newOps[i] = newOp;
575 }
576 }
577 rewriter.replaceOpWithMultiple(op, newValues: {newOps});
578 return success();
579 }
580
581 // SLM path: layouts differ, need cross-subgroup data redistribution
582 Type elemTy = cast<VectorType>(Val: op.getSource().getType()).getElementType();
583
584 SmallVector<int64_t> slmShape = llvm::to_vector(Range&: wgShape);
585
586 // Calculate SLM size requirements
587 auto bitWidth = elemTy.getIntOrFloatBitWidth();
588 auto bytesPerElement = bitWidth / 8;
589 auto slmSize = computeProduct(basis: slmShape) * bytesPerElement;
590
591 // Allocate SLM
592 auto slmTy = MemRefType::get(shape: {slmSize}, elementType: rewriter.getI8Type(), map: {}, memorySpaceInd: 3);
593 auto slm = memref::AllocaOp::create(builder&: rewriter, location: loc, memrefType: slmTy);
594
595 auto memDescType = xegpu::MemDescType::get(context: rewriter.getContext(), shape: slmShape,
596 elementType: elemTy, mem_layout: nullptr);
597 auto memDesc =
598 xegpu::CreateMemDescOp::create(builder&: rewriter, location: loc, mem_desc: memDescType, source: slm);
599
600 auto sgId = gpu::SubgroupIdOp::create(builder&: rewriter, location: loc,
601 result: rewriter.getIndexType(), upper_bound: nullptr);
602
603 // STORE PHASE: Each subgroup stores in SLM using input layout
604 auto storeCoords = inputLayout.computeDistributedCoords(
605 builder&: rewriter, loc, linearId: sgId.getResult(), shape: wgShape);
606 if (failed(Result: storeCoords))
607 return failure();
608
609 // Store to SLM
610 for (auto [src, coords] : llvm::zip(t: adaptor.getSource(), u&: *storeCoords)) {
611 SmallVector<OpFoldResult> storeMatrixOffsets;
612 for (Value coord : coords) {
613 storeMatrixOffsets.push_back(Elt: coord);
614 }
615 xegpu::StoreMatrixOp::create(builder&: rewriter, location: loc, data: src, mem_desc: memDesc.getResult(),
616 offsets: storeMatrixOffsets, layout: nullptr /*layout*/);
617 }
618
619 gpu::BarrierOp::create(builder&: rewriter, location: loc);
620
621 // LOAD PHASE: Each target subgroup loads from SLM using target layout
622 auto loadCoords = targetLayout.computeDistributedCoords(
623 builder&: rewriter, loc, linearId: sgId.getResult(), shape: wgShape);
624 if (failed(Result: loadCoords))
625 return failure();
626
627 VectorType loadType = VectorType::get(shape: targetSgData, elementType: elemTy);
628
629 // Load vectors from SLM
630 SmallVector<Value> finalResults;
631 for (auto coords : *loadCoords) {
632 SmallVector<OpFoldResult> loadMatrixOffsets;
633 for (Value coord : coords) {
634 loadMatrixOffsets.push_back(Elt: coord);
635 }
636 auto loadOp = xegpu::LoadMatrixOp::create(
637 builder&: rewriter, location: loc, res: loadType, mem_desc: memDesc.getResult(), offsets: loadMatrixOffsets,
638 layout: targetLayout.dropSgLayoutAndData());
639
640 finalResults.push_back(Elt: loadOp.getResult());
641 }
642
643 rewriter.replaceOpWithMultiple(op, newValues: {finalResults});
644 return success();
645 }
646};
647
648// This pattern distributes arith.constant op into subgroup-level constants
649struct WgToSgArithConstantOp : public OpConversionPattern<arith::ConstantOp> {
650 using OpConversionPattern<arith::ConstantOp>::OpConversionPattern;
651
652 LogicalResult
653 matchAndRewrite(arith::ConstantOp op, OneToNOpAdaptor adaptor,
654 ConversionPatternRewriter &rewriter) const override {
655 auto vecAttr = dyn_cast<DenseElementsAttr>(Val: op.getValue());
656 auto vecType = dyn_cast<VectorType>(Val: op.getType());
657 if (!vecAttr || !vecType)
658 return failure();
659
660 xegpu::DistributeLayoutAttr layout =
661 xegpu::getTemporaryLayout(operandOrResult: dyn_cast<OpResult>(Val: op.getResult()));
662 if (!layout || !layout.isForWorkgroup())
663 return failure();
664
665 ArrayRef<int64_t> wgShape = vecType.getShape();
666 SmallVector<int64_t> sgShape;
667 int count;
668 std::tie(args&: sgShape, args&: count) = getSgShapeAndCount(shape: wgShape, layout);
669
670 auto newType = VectorType::get(shape: sgShape, elementType: vecType.getElementType());
671 Location loc = op.getLoc();
672 auto eltType = vecType.getElementType();
673
674 if (vecAttr.isSplat()) {
675 // Splat: single value for all subgroups
676 Attribute singleVal = vecAttr.getSplatValue<Attribute>();
677 auto sgAttr = DenseElementsAttr::get(type: newType, values: singleVal);
678 SmallVector<Value> newConstOps;
679 for (int i = 0; i < count; ++i) {
680 auto cstOp = arith::ConstantOp::create(builder&: rewriter, location: loc, result: newType, value: sgAttr);
681 newConstOps.push_back(Elt: cstOp);
682 }
683 rewriter.replaceOpWithMultiple(op, newValues: {newConstOps});
684 return success();
685 } else if (sgShape == wgShape) { // if the entire vector is shared by all
686 // subgroups, don't distribute
687 auto newConstOp =
688 arith::ConstantOp::create(builder&: rewriter, location: op.getLoc(), result: vecType, value: vecAttr);
689 rewriter.replaceOp(op, newOp: newConstOp);
690 return success();
691 } else {
692 // Non-splat constant
693 // Only supports 1D & 2D
694 // TODO: support other cases that require SLM access
695 if (!eltType.isIndex())
696 return rewriter.notifyMatchFailure(
697 arg&: op, msg: "Unsupported element type for non-splat constant op.");
698
699 if (wgShape.size() > 2)
700 return rewriter.notifyMatchFailure(
701 arg&: op, msg: "Only 1D & 2D vector constant supported");
702
703 SmallVector<Attribute> values(vecAttr.getValues<Attribute>());
704 int64_t rowStride = 0, colStride = 0;
705 int64_t rows = wgShape.size() == 1 ? 1 : wgShape[0];
706 int64_t cols = wgShape.size() == 1 ? wgShape[0] : wgShape[1];
707
708 // Compute colStride and rowStride, and check for constant strides.
709 if (cols > 1) {
710 colStride = cast<IntegerAttr>(Val&: values[1]).getInt() -
711 cast<IntegerAttr>(Val&: values[0]).getInt();
712 }
713 if (rows > 1) {
714 rowStride = cast<IntegerAttr>(Val&: values[cols]).getInt() -
715 cast<IntegerAttr>(Val&: values[0]).getInt();
716 }
717
718 for (int64_t r = 0; r < rows; ++r) {
719 for (int64_t c = 0; c < cols; ++c) {
720 int64_t idx = r * cols + c;
721 // Check column stride
722 if (c > 0 && cols > 1) {
723 int64_t prevIdx = r * cols + (c - 1);
724 int64_t diff = cast<IntegerAttr>(Val&: values[idx]).getInt() -
725 cast<IntegerAttr>(Val&: values[prevIdx]).getInt();
726 if (diff != colStride)
727 return rewriter.notifyMatchFailure(
728 arg&: op, msg: "Non-constant column stride in constant op.");
729 }
730 // Check row stride
731 if (r > 0 && rows > 1) {
732 int64_t prevIdx = (r - 1) * cols + c;
733 int64_t diff = cast<IntegerAttr>(Val&: values[idx]).getInt() -
734 cast<IntegerAttr>(Val&: values[prevIdx]).getInt();
735 if (diff != rowStride)
736 return rewriter.notifyMatchFailure(
737 arg&: op, msg: "Non-constant row stride in constant op.");
738 }
739 }
740 }
741
742 // Create a constant for the base tile.
743 // For 2D case, extract the top-left sgShape[0] x sgShape[1] submatrix.
744 // For 1D case, extract the first sgShape[0] elements.
745 SmallVector<Attribute> baseTileValues;
746 int baseTileCols = sgShape[sgShape.size() - 1];
747 int64_t baseTileRows = sgShape.size() == 1 ? 1 : sgShape[0];
748 for (int64_t r = 0; r < baseTileRows; ++r) {
749 for (int64_t c = 0; c < baseTileCols; ++c) {
750 baseTileValues.push_back(Elt: values[r * cols + c]);
751 }
752 }
753
754 auto tileAttr = DenseElementsAttr::get(type: VectorType::get(shape: sgShape, elementType: eltType),
755 values: baseTileValues);
756 auto baseConstVec = arith::ConstantOp::create(builder&: rewriter, location: loc, value: tileAttr);
757
758 // Get subgroup id
759 Value sgId =
760 gpu::SubgroupIdOp::create(builder&: rewriter, location: loc, /*upper_bound=*/nullptr);
761 auto sgOffsets =
762 layout.computeDistributedCoords(builder&: rewriter, loc, linearId: sgId, shape: wgShape);
763 if (failed(Result: sgOffsets))
764 return failure();
765
766 SmallVector<Value, 2> strideConsts;
767 strideConsts.push_back(
768 Elt: arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: colStride));
769 if (rows > 1)
770 strideConsts.insert(
771 I: strideConsts.begin(),
772 Elt: arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: rowStride));
773
774 SmallVector<Value> newConstOps;
775 for (auto offsets : *sgOffsets) {
776 // Multiply offset with stride, broadcast it and add to baseConstVec
777 Value mulOffset = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
778 for (size_t i = 0; i < strideConsts.size(); ++i) {
779 Value mul =
780 arith::MulIOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(),
781 lhs: offsets[i], rhs: strideConsts[i]);
782 mulOffset = arith::AddIOp::create(
783 builder&: rewriter, location: loc, result: rewriter.getIndexType(), lhs: mulOffset, rhs: mul);
784 }
785 // Broadcast to baseConstVec size
786 auto bcastOffset = vector::BroadcastOp::create(
787 builder&: rewriter, location: loc, result: baseConstVec.getType(), source: mulOffset);
788 auto finalConst =
789 arith::AddIOp::create(builder&: rewriter, location: loc, lhs: baseConstVec, rhs: bcastOffset);
790 newConstOps.push_back(Elt: finalConst);
791 }
792 rewriter.replaceOpWithMultiple(op, newValues: {newConstOps});
793 return success();
794 }
795 }
796};
797
798// This pattern transforms the LoadGatherOp with explicit offsets to load
799// subgroup data
800struct WgToSgLoadGatherOp : public OpConversionPattern<xegpu::LoadGatherOp> {
801 using OpConversionPattern<xegpu::LoadGatherOp>::OpConversionPattern;
802 LogicalResult
803 matchAndRewrite(xegpu::LoadGatherOp op, OneToNOpAdaptor adaptor,
804 ConversionPatternRewriter &rewriter) const override {
805
806 Location loc = op.getLoc();
807 VectorType resultType = dyn_cast<VectorType>(Val: op.getResult().getType());
808 if (!resultType)
809 return failure();
810 ArrayRef<int64_t> wgShape = resultType.getShape();
811
812 xegpu::DistributeLayoutAttr layout = op.getLayoutAttr();
813
814 if (!layout || !layout.isForWorkgroup())
815 return failure();
816
817 SmallVector<int64_t> sgShape = getSgShapeAndCount(shape: wgShape, layout).first;
818
819 // The offsets need to be distributed
820 auto offsetsVecType =
821 dyn_cast<VectorType>(Val: adaptor.getOffsets().front().getType());
822 auto maskVecType =
823 dyn_cast<VectorType>(Val: adaptor.getMask().front().getType());
824 if (!offsetsVecType || !maskVecType ||
825 offsetsVecType.getShape() != maskVecType.getShape()) {
826 return rewriter.notifyMatchFailure(arg&: op,
827 msg: "offsets have not been distributed");
828 }
829
830 SmallVector<Value> newLoadOps;
831 VectorType newTy = VectorType::get(shape: sgShape, elementType: resultType.getElementType());
832 for (auto [offsets, mask] :
833 llvm::zip(t: adaptor.getOffsets(), u: adaptor.getMask())) {
834 auto newLayout = layout.dropSgLayoutAndData();
835 auto newLoadOp = xegpu::LoadGatherOp::create(
836 builder&: rewriter, location: loc, value: newTy, source: op.getSource(), offsets, mask,
837 l1_hint: op.getL1HintAttr(), l2_hint: op.getL2HintAttr(), l3_hint: op.getL3HintAttr(), layout: newLayout,
838 /*contiguity=*/nullptr);
839 newLoadOps.push_back(Elt: newLoadOp);
840 }
841 rewriter.replaceOpWithMultiple(op, newValues: {newLoadOps});
842 return success();
843 }
844};
845
846// This pattern transforms the StoreScatterOp with explicit offsets to store
847// subgroup data
848struct WgToSgStoreScatterOp
849 : public OpConversionPattern<xegpu::StoreScatterOp> {
850 using OpConversionPattern<xegpu::StoreScatterOp>::OpConversionPattern;
851 LogicalResult
852 matchAndRewrite(xegpu::StoreScatterOp op, OneToNOpAdaptor adaptor,
853 ConversionPatternRewriter &rewriter) const override {
854
855 Location loc = op.getLoc();
856 VectorType valueType = dyn_cast<VectorType>(Val: op.getValue().getType());
857 if (!valueType)
858 return failure();
859
860 xegpu::DistributeLayoutAttr layout = op.getLayoutAttr();
861
862 if (!layout || !layout.isForWorkgroup())
863 return failure();
864
865 // The offsets need to be distributed
866 auto offsetsVecType =
867 dyn_cast<VectorType>(Val: adaptor.getOffsets().front().getType());
868 auto maskVecType =
869 dyn_cast<VectorType>(Val: adaptor.getMask().front().getType());
870 if (!offsetsVecType || !maskVecType ||
871 offsetsVecType.getShape() != maskVecType.getShape()) {
872 return rewriter.notifyMatchFailure(arg&: op,
873 msg: "offsets have not been distributed");
874 }
875
876 for (auto [val, offs, mask] : llvm::zip(
877 t: adaptor.getValue(), u: adaptor.getOffsets(), args: adaptor.getMask())) {
878 xegpu::StoreScatterOp::create(
879 builder&: rewriter, location: loc, value: val, dest: op.getDest(), offsets: offs, mask, l1_hint: op.getL1HintAttr(),
880 l2_hint: op.getL2HintAttr(), l3_hint: op.getL3HintAttr(), layout: layout.dropSgLayoutAndData(),
881 /*contiguity=*/nullptr);
882 }
883 rewriter.eraseOp(op);
884 return success();
885 }
886};
887
888struct WgToSgLoadMatrixOp : public OpConversionPattern<xegpu::LoadMatrixOp> {
889 using OpConversionPattern<xegpu::LoadMatrixOp>::OpConversionPattern;
890 LogicalResult
891 matchAndRewrite(xegpu::LoadMatrixOp op, OneToNOpAdaptor adaptor,
892 ConversionPatternRewriter &rewriter) const override {
893
894 SmallVector<SmallVector<OpFoldResult>> offsetsList;
895 if (failed(Result: genOffsetsList(rewriter, op, offsetsList)))
896 return failure();
897
898 ArrayRef<int64_t> wgShape = op.getDataShape();
899 VectorType valueTy = llvm::dyn_cast<VectorType>(Val: op.getRes().getType());
900 assert(valueTy && "the value type must be vector type!");
901 Type elemTy = valueTy.getElementType();
902
903 xegpu::DistributeLayoutAttr layout = op.getLayoutAttr();
904 SmallVector<int64_t> sgShape = getSgShapeAndCount(shape: wgShape, layout).first;
905 VectorType newResTy = VectorType::get(shape: sgShape, elementType: elemTy);
906 SmallVector<Value> newOps;
907 for (auto offsets : offsetsList) {
908 auto newOp = xegpu::LoadMatrixOp::create(builder&: rewriter, location: op.getLoc(), res: newResTy,
909 mem_desc: op.getMemDesc(), offsets,
910 layout: layout.dropSgLayoutAndData());
911 newOps.push_back(Elt: newOp);
912 }
913 rewriter.replaceOpWithMultiple(op, newValues: {newOps});
914
915 return success();
916 }
917};
918
919struct WgToSgStoreMatrixOp : public OpConversionPattern<xegpu::StoreMatrixOp> {
920 using OpConversionPattern<xegpu::StoreMatrixOp>::OpConversionPattern;
921 LogicalResult
922 matchAndRewrite(xegpu::StoreMatrixOp op, OneToNOpAdaptor adaptor,
923 ConversionPatternRewriter &rewriter) const override {
924
925 SmallVector<SmallVector<OpFoldResult>> offsetsList;
926 if (failed(Result: genOffsetsList(rewriter, op, offsetsList)))
927 return failure();
928
929 xegpu::DistributeLayoutAttr layout = op.getLayoutAttr();
930 for (auto [v, offsets] : llvm::zip(t: adaptor.getData(), u&: offsetsList))
931 xegpu::StoreMatrixOp::create(builder&: rewriter, location: op.getLoc(), data: v, mem_desc: op.getMemDesc(),
932 offsets, layout: layout.dropSgLayoutAndData());
933 rewriter.eraseOp(op);
934 return success();
935 }
936};
937
938// This pattern distributes the vector.step ops to work at subgroup level
939struct WgToSgVectorStepOp : public OpConversionPattern<vector::StepOp> {
940 using OpConversionPattern<vector::StepOp>::OpConversionPattern;
941 LogicalResult
942 matchAndRewrite(vector::StepOp op, OneToNOpAdaptor adaptor,
943 ConversionPatternRewriter &rewriter) const override {
944 xegpu::DistributeLayoutAttr layout =
945 xegpu::getTemporaryLayout(operandOrResult: dyn_cast<OpResult>(Val: op.getResult()));
946 if (!layout || !layout.isForWorkgroup())
947 return failure();
948
949 Location loc = op.getLoc();
950 VectorType type = op.getResult().getType();
951 auto wgShape = type.getShape();
952 std::optional<SmallVector<int64_t>> sgShape =
953 getSgShapeAndCount(shape: wgShape, layout).first;
954 if (!sgShape)
955 return failure();
956
957 Value sgId =
958 gpu::SubgroupIdOp::create(builder&: rewriter, location: loc, /*upper_bound=*/nullptr);
959 auto sgOffsets =
960 layout.computeDistributedCoords(builder&: rewriter, loc, linearId: sgId, shape: wgShape);
961 if (failed(Result: sgOffsets))
962 return failure();
963
964 VectorType newTy = type.cloneWith(shape: *sgShape, elementType: type.getElementType());
965 auto steps = vector::StepOp::create(builder&: rewriter, location: loc, result: newTy);
966 SmallVector<Value> newOps;
967 for (auto offsets : *sgOffsets) {
968 // Broadcast the offset scalar to a vector & add to the base steps
969 auto bcastOffset =
970 vector::BroadcastOp::create(builder&: rewriter, location: loc, result: newTy, source: offsets[0]);
971 auto finalSteps =
972 arith::AddIOp::create(builder&: rewriter, location: loc, lhs: steps, rhs: bcastOffset);
973 newOps.push_back(Elt: finalSteps);
974 }
975
976 rewriter.replaceOpWithMultiple(op, newValues: {newOps});
977 return success();
978 }
979};
980
981// This pattern transforms vector.shape_cast ops to work at subgroup level.
982struct WgToSgVectorShapeCastOp
983 : public OpConversionPattern<vector::ShapeCastOp> {
984 using OpConversionPattern<vector::ShapeCastOp>::OpConversionPattern;
985
986 LogicalResult
987 matchAndRewrite(vector::ShapeCastOp op, OneToNOpAdaptor adaptor,
988 ConversionPatternRewriter &rewriter) const override {
989
990 VectorType resultType = dyn_cast<VectorType>(Val: op.getResult().getType());
991 if (!resultType)
992 return failure();
993
994 ArrayRef<int64_t> wgShape = resultType.getShape();
995 xegpu::DistributeLayoutAttr layout =
996 xegpu::getTemporaryLayout(operandOrResult: dyn_cast<OpResult>(Val: op.getResult()));
997 if (!layout || !layout.isForWorkgroup())
998 return failure();
999
1000 // Check that srcShape and destShape, if they differ, only differ by
1001 // expand of unit dimensions.
1002 auto srcType = dyn_cast<VectorType>(Val: op.getSource().getType());
1003 if (!srcType)
1004 return failure();
1005
1006 ArrayRef<int64_t> srcShape = srcType.getShape();
1007
1008 xegpu::DistributeLayoutAttr layoutToDistribute = layout;
1009 SmallVector<int64_t> expandedUnitDims;
1010 if (xegpu::matchUnitDimExpansion(src: srcShape, dst: wgShape, expandedUnitDims)) {
1011 xegpu::DistributeLayoutAttr sourceLayout =
1012 xegpu::getTemporaryLayout(operandOrResult: op->getOpOperand(idx: 0));
1013
1014 if (!sourceLayout.isSliceOf(other: layout))
1015 return rewriter.notifyMatchFailure(
1016 arg&: op, msg: "The ShapeCast op only expands dimensions, the input layout "
1017 "must be a slice of the result layout.");
1018
1019 assert(layoutToDistribute.isEqualTo(
1020 layoutToDistribute.setUnitDimData(expandedUnitDims)) &&
1021 "The sg_data for unit dimensions should be set as 1");
1022 }
1023
1024 SmallVector<int64_t> sgShape =
1025 getSgShapeAndCount(shape: wgShape, layout: layoutToDistribute).first;
1026 VectorType newResultType =
1027 VectorType::get(shape: sgShape, elementType: resultType.getElementType());
1028
1029 SmallVector<Value> newShapeCastOps;
1030 for (auto src : adaptor.getSource()) {
1031 auto newShapeCast = vector::ShapeCastOp::create(builder&: rewriter, location: op.getLoc(),
1032 result: newResultType, source: src);
1033 newShapeCastOps.push_back(Elt: newShapeCast.getResult());
1034 }
1035
1036 rewriter.replaceOpWithMultiple(op, newValues: {newShapeCastOps});
1037 return success();
1038 }
1039};
1040
1041/// This pattern transforms vector.multi_dim_reduction operations from
1042/// workgroup-level to subgroup-level execution with support for multiple
1043/// reduction dimensions.
1044///
1045/// Steps include:
1046/// 1. LOCAL REDUCTION :
1047/// - Each subgroup performs local reduction on its data slice
1048/// - Uses ZERO accumulator to avoid double-counting during cross-subgroup
1049/// phase
1050///
1051/// 2. CROSS-SUBGROUP :
1052/// - Determines if cross-subgroup reduction is needed (when sg_layout > 1 in
1053/// reduction dims & sgData[reduction dims] < wgData[reduction dims])
1054/// - If not needed, adds original accumulator and returns local results
1055///
1056/// 3. SHARED LOCAL MEMORY (SLM) PHASE (when cross-subgroup reduction needed):
1057/// a) SLM Layout Design:
1058/// - Rows: subgroups participating in reduction (product of sg_layout in
1059/// reduction dims)
1060/// - Cols: total result elements across non-reduction dimensions
1061///
1062/// b) Store Phase:
1063/// - Each subgroup stores its local reduction result to SLM
1064/// - Row offset: linearized index of subgroup in reduction dimensions
1065/// - Col offset: linearized index of subgroup in non-reduction dimensions
1066///
1067/// c) Load and Final Reduction Phase:
1068/// - Each subgroup loads a column of data (all reduction participants for
1069/// its position)
1070/// - Performs final reduction along the loaded dimension
1071/// - Adds original accumulator to get final result
1072///
1073struct WgToSgMultiDimReductionOp
1074 : public OpConversionPattern<vector::MultiDimReductionOp> {
1075 using OpConversionPattern<vector::MultiDimReductionOp>::OpConversionPattern;
1076
1077 LogicalResult
1078 matchAndRewrite(vector::MultiDimReductionOp op, OneToNOpAdaptor adaptor,
1079 ConversionPatternRewriter &rewriter) const override {
1080 Location loc = op.getLoc();
1081
1082 VectorType srcType = op.getSourceVectorType();
1083 Type resultTy = op.getResult().getType();
1084 VectorType dstVecType = dyn_cast<VectorType>(Val&: resultTy);
1085 bool isScalarResult = !dstVecType;
1086
1087 auto originalSrcShape = srcType.getShape();
1088 Type elemTy = srcType.getElementType();
1089
1090 xegpu::DistributeLayoutAttr layout =
1091 xegpu::getTemporaryLayout(operandOrResult: dyn_cast<OpResult>(Val: op.getResult()));
1092 if (!layout || !layout.isForWorkgroup())
1093 return failure();
1094
1095 auto reductionDims = llvm::to_vector(Range: op.getReductionDims());
1096
1097 // Get sg_layout and sg_data from the parent layout
1098 SmallVector<int64_t> sgLayout;
1099 SmallVector<int64_t> sgData;
1100 xegpu::DistributeLayoutAttr parentLayout;
1101 if (auto sliceAttr = dyn_cast<xegpu::SliceAttr>(Val&: layout)) {
1102 parentLayout = sliceAttr.getParent();
1103 sgLayout = parentLayout.getEffectiveSgLayoutAsInt();
1104 sgData = parentLayout.getEffectiveSgDataAsInt();
1105 } else
1106 return rewriter.notifyMatchFailure(
1107 arg&: op, msg: "Reduction should have SliceAttr layout");
1108
1109 // Step 1: perform local subgroup reductions with neutral accumulator
1110 SmallVector<Value> localReductions;
1111 auto sgSrcs = adaptor.getSource();
1112 auto sgSrcType = dyn_cast<VectorType>(Val: sgSrcs.front().getType());
1113 SmallVector<int64_t> sgSrcShape(sgSrcType.getShape().begin(),
1114 sgSrcType.getShape().end());
1115
1116 // Determine the SG-level destination type.
1117 // For scalar results (all dims reduced), the sg result is also scalar.
1118 // For vector results, compute the sg destination shape from layout.
1119 Type sgDstType;
1120 if (dstVecType) {
1121 auto originalDstShape = dstVecType.getShape();
1122 SmallVector<int64_t> sgDstShape =
1123 getSgShapeAndCount(shape: originalDstShape, layout).first;
1124 sgDstType = VectorType::get(shape: sgDstShape, elementType: elemTy);
1125 } else {
1126 sgDstType = elemTy;
1127 }
1128
1129 for (auto sgSrc : sgSrcs) {
1130 // Create neutral accumulator for local reduction
1131 Value neutralLocalAcc = xegpu::createReductionNeutralValue(
1132 builder&: rewriter, loc, type: sgDstType, kind: op.getKind(), fastmath: op.getFastmathAttr());
1133 // Local reduction with neutral accumulator
1134 auto localReduce = vector::MultiDimReductionOp::create(
1135 builder&: rewriter, location: loc, result: sgDstType, kind: op.getKind(), source: sgSrc, acc: neutralLocalAcc,
1136 reduction_dims: reductionDims, fastmath: op.getFastmath());
1137 localReductions.push_back(Elt: localReduce.getResult());
1138 }
1139
1140 // Check if cross-subgroup reduction is needed for any reduction dimension
1141 SmallVector<int64_t> crossSgReductionDims;
1142 for (int64_t reductionDim : reductionDims) {
1143 bool needsCrossSubgroupReduction =
1144 (sgLayout[reductionDim] > 1) &&
1145 (sgData[reductionDim] < originalSrcShape[reductionDim]);
1146
1147 if (needsCrossSubgroupReduction) {
1148 crossSgReductionDims.push_back(Elt: reductionDim);
1149 }
1150 }
1151
1152 // If no cross-subgroup reduction needed, add accumulator and return
1153 if (crossSgReductionDims.empty()) {
1154 SmallVector<Value> results;
1155 for (auto localResult : localReductions) {
1156 auto finalResult = vector::makeArithReduction(
1157 b&: rewriter, loc, kind: op.getKind(), v1: localResult, acc: adaptor.getAcc()[0],
1158 fastmath: op.getFastmathAttr());
1159 results.push_back(Elt: finalResult);
1160 }
1161 rewriter.replaceOpWithMultiple(op, newValues: {results});
1162 return success();
1163 }
1164
1165 // Step 2: cross-subgroup reduction using SLM - allocating slm memory
1166 auto slmStoreDataShape = sgSrcShape;
1167 for (int64_t dim : reductionDims)
1168 slmStoreDataShape[dim] = 1;
1169 VectorType slmStoreDataType = VectorType::get(shape: slmStoreDataShape, elementType: elemTy);
1170 SmallVector<Value> slmStoreData;
1171 for (auto localResult : localReductions) {
1172 if (isScalarResult) {
1173 // Scalar result: broadcast scalar to vector<1x...x1> for SLM store
1174 slmStoreData.push_back(Elt: vector::BroadcastOp::create(
1175 builder&: rewriter, location: loc, result: slmStoreDataType, source: localResult));
1176 } else {
1177 slmStoreData.push_back(Elt: vector::ShapeCastOp::create(
1178 builder&: rewriter, location: loc, result: slmStoreDataType, source: localResult));
1179 }
1180 }
1181 // for reduction dimension, SLM stores partial results from each subgroup
1182 SmallVector<int64_t> slmShape(originalSrcShape.begin(),
1183 originalSrcShape.end());
1184 SmallVector<int> slmSgData(sgData.begin(), sgData.end());
1185 SmallVector<int> slmSgLayout(sgLayout.begin(), sgLayout.end());
1186 for (int dim : reductionDims) {
1187 slmShape[dim] = sgLayout[dim];
1188 slmSgData[dim] = 1;
1189 }
1190 xegpu::LayoutAttr slmStoreLayout =
1191 xegpu::LayoutAttr::get(context: rewriter.getContext(), lane_layout: slmSgLayout, lane_data: slmSgData);
1192
1193 // Allocate SLM
1194 auto bitWidth = elemTy.getIntOrFloatBitWidth();
1195 auto bytesPerElement = bitWidth / 8;
1196 auto slmSize = computeProduct(basis: slmShape) * bytesPerElement;
1197 auto slmTy = MemRefType::get(shape: {slmSize}, elementType: rewriter.getI8Type(), map: {}, memorySpaceInd: 3);
1198 auto slm = memref::AllocaOp::create(builder&: rewriter, location: loc, memrefType: slmTy);
1199
1200 auto memDescType = xegpu::MemDescType::get(context: rewriter.getContext(), shape: slmShape,
1201 elementType: elemTy, mem_layout: nullptr);
1202 auto memDesc =
1203 xegpu::CreateMemDescOp::create(builder&: rewriter, location: loc, mem_desc: memDescType, source: slm);
1204
1205 // Step 3: Store local results to SLM
1206 auto sgId = gpu::SubgroupIdOp::create(builder&: rewriter, location: loc,
1207 result: rewriter.getIndexType(), upper_bound: nullptr);
1208
1209 auto slmStoreCoords =
1210 slmStoreLayout.computeDistributedCoords(builder&: rewriter, loc, linearId: sgId, shape: slmShape);
1211 if (failed(Result: slmStoreCoords))
1212 return failure();
1213 for (auto [data, coord] : llvm::zip(t&: slmStoreData, u&: *slmStoreCoords)) {
1214 SmallVector<OpFoldResult> coordOfr(coord.begin(), coord.end());
1215 xegpu::StoreMatrixOp::create(builder&: rewriter, location: loc, data, mem_desc: memDesc.getResult(),
1216 offsets: coordOfr,
1217 /*layout=*/nullptr);
1218 }
1219
1220 gpu::BarrierOp::create(builder&: rewriter, location: loc);
1221
1222 // Step 4: Load from SLM for final reduction
1223 SmallVector<int64_t> slmLoadDataShape(sgSrcShape.begin(), sgSrcShape.end());
1224 for (int64_t dim : reductionDims) {
1225 slmLoadDataShape[dim] = slmShape[dim];
1226 slmSgData[dim] = slmShape[dim];
1227 }
1228 xegpu::LayoutAttr slmLoadLayout =
1229 xegpu::LayoutAttr::get(context: rewriter.getContext(), lane_layout: slmSgLayout, lane_data: slmSgData);
1230 auto slmLoadCoords =
1231 slmLoadLayout.computeDistributedCoords(builder&: rewriter, loc, linearId: sgId, shape: slmShape);
1232 if (failed(Result: slmLoadCoords))
1233 return failure();
1234
1235 VectorType slmLoadType = VectorType::get(shape: slmLoadDataShape, elementType: elemTy);
1236 SmallVector<Value> slmLoadData;
1237 for (auto coord : *slmLoadCoords) {
1238 SmallVector<OpFoldResult> coordOfr(coord.begin(), coord.end());
1239 slmLoadData.push_back(Elt: xegpu::LoadMatrixOp::create(
1240 builder&: rewriter, location: loc, res: slmLoadType, mem_desc: memDesc.getResult(), offsets: coordOfr,
1241 /*layout=*/nullptr));
1242 }
1243
1244 // Step 5: Perform final reduction with neutral accumulator and add the
1245 // original accumulator at the end
1246 Value neutralFinalAcc = xegpu::createReductionNeutralValue(
1247 builder&: rewriter, loc, type: sgDstType, kind: op.getKind(), fastmath: op.getFastmathAttr());
1248
1249 SmallVector<Value> finalResults;
1250 for (size_t i = 0; i < slmLoadData.size(); ++i) {
1251 auto loaded = slmLoadData[i];
1252 auto finalReduce = vector::MultiDimReductionOp::create(
1253 builder&: rewriter, location: loc, result: sgDstType, kind: op.getKind(), source: loaded, acc: neutralFinalAcc,
1254 reduction_dims: reductionDims, fastmath: op.getFastmath());
1255 finalResults.push_back(Elt: vector::makeArithReduction(
1256 b&: rewriter, loc, kind: op.getKind(), v1: finalReduce.getResult(),
1257 acc: adaptor.getAcc()[i], fastmath: op.getFastmathAttr()));
1258 }
1259 rewriter.replaceOpWithMultiple(op, newValues: {finalResults});
1260 return success();
1261 }
1262};
1263
1264// This pattern transforms vector.transpose ops to work at subgroup level.
1265struct WgToSgVectorTransposeOp
1266 : public OpConversionPattern<vector::TransposeOp> {
1267 using OpConversionPattern<vector::TransposeOp>::OpConversionPattern;
1268
1269 LogicalResult
1270 matchAndRewrite(vector::TransposeOp op, OneToNOpAdaptor adaptor,
1271 ConversionPatternRewriter &rewriter) const override {
1272 VectorType resultType = op.getResultVectorType();
1273
1274 ArrayRef<int64_t> wgShape = resultType.getShape();
1275 xegpu::DistributeLayoutAttr layout =
1276 xegpu::getTemporaryLayout(operandOrResult: dyn_cast<OpResult>(Val: op.getResult()));
1277 if (!layout || !layout.isForWorkgroup())
1278 return failure();
1279 xegpu::DistributeLayoutAttr sourceLayout =
1280 xegpu::getTemporaryLayout(operandOrResult: op->getOpOperand(idx: 0));
1281 if (!sourceLayout || !sourceLayout.isForWorkgroup())
1282 return failure();
1283
1284 SmallVector<int64_t> sourceSgLayout =
1285 sourceLayout.getEffectiveSgLayoutAsInt();
1286 SmallVector<int64_t> resultSgLayout = layout.getEffectiveSgLayoutAsInt();
1287
1288 ArrayRef<int64_t> permutation = op.getPermutation();
1289 size_t permutationSize = permutation.size();
1290 if (sourceSgLayout.size() != permutationSize ||
1291 resultSgLayout.size() != permutationSize) {
1292 return rewriter.notifyMatchFailure(
1293 arg&: op, msg: "Layouts and permutation must have the same rank");
1294 }
1295
1296 // Check that sgLayout, sgData & order are properly transposed for source
1297 // and result
1298 if (!layout.isTransposeOf(other: sourceLayout, perm: permutation,
1299 kind: xegpu::LayoutKind::Subgroup))
1300 return rewriter.notifyMatchFailure(
1301 arg&: op, msg: "Result layout is not a valid transpose of source layout "
1302 "according to permutation");
1303
1304 SmallVector<int64_t> sgShape = getSgShapeAndCount(shape: wgShape, layout).first;
1305 VectorType newResultType =
1306 VectorType::get(shape: sgShape, elementType: resultType.getElementType());
1307
1308 SmallVector<Value> newTransposeOps;
1309 for (auto src : adaptor.getVector()) {
1310 auto newTranspose = vector::TransposeOp::create(
1311 builder&: rewriter, location: op.getLoc(), result: newResultType, vector: src, permutation);
1312 newTransposeOps.push_back(Elt: newTranspose.getResult());
1313 }
1314 rewriter.replaceOpWithMultiple(op, newValues: {newTransposeOps});
1315 return success();
1316 }
1317};
1318
1319// Distribute vector mask ops to work at subgroup level.
1320template <typename MaskOpType>
1321struct WgToSgVectorMaskOp : public OpConversionPattern<MaskOpType> {
1322 using OpConversionPattern<MaskOpType>::OpConversionPattern;
1323
1324 LogicalResult matchAndRewrite(
1325 MaskOpType op,
1326 typename OpConversionPattern<MaskOpType>::OneToNOpAdaptor adaptor,
1327 ConversionPatternRewriter &rewriter) const override {
1328 xegpu::DistributeLayoutAttr layout =
1329 xegpu::getTemporaryLayout(dyn_cast<OpResult>(op.getResult()));
1330 if (!layout || !layout.isForWorkgroup())
1331 return failure();
1332
1333 Location loc = op.getLoc();
1334 VectorType type = op.getResult().getType();
1335 auto wgShape = type.getShape();
1336
1337 SmallVector<Value> wgMaskDimSizes;
1338 if constexpr (std::is_same_v<MaskOpType, vector::ConstantMaskOp>) {
1339 for (int64_t maskSize : op.getMaskDimSizes()) {
1340 wgMaskDimSizes.push_back(
1341 Elt: arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: maskSize));
1342 }
1343 } else if constexpr (std::is_same_v<MaskOpType, vector::CreateMaskOp>) {
1344 wgMaskDimSizes = llvm::to_vector(op.getOperands());
1345 }
1346
1347 Value sgId =
1348 gpu::SubgroupIdOp::create(builder&: rewriter, location: loc, /*upper_bound=*/nullptr);
1349 auto sgOffsets =
1350 layout.computeDistributedCoords(builder&: rewriter, loc, linearId: sgId, shape: wgShape);
1351 if (failed(Result: sgOffsets))
1352 return failure();
1353
1354 SmallVector<int64_t> sgShape = getSgShapeAndCount(shape: wgShape, layout).first;
1355 VectorType resultType = VectorType::get(shape: sgShape, elementType: type.getElementType());
1356
1357 // In each dimension, each subgroup computes its local mask size as:
1358 // min(max(wgMaskDimSize[d] - offset[d], 0), sgDimSize[d])
1359 SmallVector<Value> newCreateMaskOps;
1360 for (auto offsetSet : *sgOffsets) {
1361 SmallVector<Value> maskOperands;
1362
1363 for (auto [i, wgMaskDimSize] : llvm::enumerate(First&: wgMaskDimSizes)) {
1364 Value dimSizeVal =
1365 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: sgShape[i]);
1366 Value offset = offsetSet[i];
1367 Value adjustedMaskSize =
1368 arith::SubIOp::create(builder&: rewriter, location: loc, lhs: wgMaskDimSize, rhs: offset);
1369 Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
1370 Value nonNegative =
1371 arith::MaxSIOp::create(builder&: rewriter, location: loc, lhs: adjustedMaskSize, rhs: zero);
1372 Value sgMaskSize =
1373 arith::MinSIOp::create(builder&: rewriter, location: loc, lhs: nonNegative, rhs: dimSizeVal);
1374 maskOperands.push_back(Elt: sgMaskSize);
1375 }
1376
1377 auto newCreateMaskOp =
1378 vector::CreateMaskOp::create(builder&: rewriter, location: loc, resultType0: resultType, mask_dim_sizes: maskOperands);
1379 newCreateMaskOps.push_back(Elt: newCreateMaskOp.getResult());
1380 }
1381
1382 rewriter.replaceOpWithMultiple(op, {newCreateMaskOps});
1383 return success();
1384 }
1385};
1386
1387using WgToSgVectorConstantMaskOp = WgToSgVectorMaskOp<vector::ConstantMaskOp>;
1388using WgToSgVectorCreateMaskOp = WgToSgVectorMaskOp<vector::CreateMaskOp>;
1389
1390// This pattern transforms vector.bitcast ops to work at subgroup level.
1391struct WgToSgVectorBitCastOp : public OpConversionPattern<vector::BitCastOp> {
1392 using OpConversionPattern<vector::BitCastOp>::OpConversionPattern;
1393
1394 LogicalResult
1395 matchAndRewrite(vector::BitCastOp op, OneToNOpAdaptor adaptor,
1396 ConversionPatternRewriter &rewriter) const override {
1397 VectorType resultType = op.getResultVectorType();
1398
1399 ArrayRef<int64_t> wgShape = resultType.getShape();
1400 xegpu::DistributeLayoutAttr layout =
1401 xegpu::getTemporaryLayout(operandOrResult: dyn_cast<OpResult>(Val: op.getResult()));
1402 if (!layout || !layout.isForWorkgroup())
1403 return failure();
1404
1405 SmallVector<int64_t> sgShape = getSgShapeAndCount(shape: wgShape, layout).first;
1406 VectorType newResultType =
1407 VectorType::get(shape: sgShape, elementType: resultType.getElementType());
1408
1409 SmallVector<Value> newBitCastOps;
1410 for (auto src : adaptor.getSource()) {
1411 auto newBitCast =
1412 vector::BitCastOp::create(builder&: rewriter, location: op.getLoc(), result: newResultType, source: src);
1413 newBitCastOps.push_back(Elt: newBitCast.getResult());
1414 }
1415
1416 rewriter.replaceOpWithMultiple(op, newValues: {newBitCastOps});
1417 return success();
1418 }
1419};
1420
1421// This pattern transforms vector.interleave ops to work at subgroup level.
1422struct WgToSgVectorInterleaveOp
1423 : public OpConversionPattern<vector::InterleaveOp> {
1424 using OpConversionPattern<vector::InterleaveOp>::OpConversionPattern;
1425
1426 LogicalResult
1427 matchAndRewrite(vector::InterleaveOp op, OneToNOpAdaptor adaptor,
1428 ConversionPatternRewriter &rewriter) const override {
1429 VectorType resultType = op.getResultVectorType();
1430
1431 ArrayRef<int64_t> wgShape = resultType.getShape();
1432 xegpu::DistributeLayoutAttr layout =
1433 xegpu::getTemporaryLayout(operandOrResult: dyn_cast<OpResult>(Val: op.getResult()));
1434 if (!layout || !layout.isForWorkgroup())
1435 return failure();
1436
1437 SmallVector<int64_t> sgShape = getSgShapeAndCount(shape: wgShape, layout).first;
1438 VectorType newResultType =
1439 VectorType::get(shape: sgShape, elementType: resultType.getElementType());
1440
1441 SmallVector<Value> newInterleaveOps;
1442 // Interleave operates pairwise: each lhs value is interleaved with
1443 // corresponding rhs value
1444 for (auto [lhs, rhs] : llvm::zip(t: adaptor.getLhs(), u: adaptor.getRhs())) {
1445 auto newInterleave = vector::InterleaveOp::create(
1446 builder&: rewriter, location: op.getLoc(), result: newResultType, lhs, rhs);
1447 newInterleaveOps.push_back(Elt: newInterleave.getResult());
1448 }
1449
1450 rewriter.replaceOpWithMultiple(op, newValues: {newInterleaveOps});
1451 return success();
1452 }
1453};
1454
1455// This pattern transforms vector.deinterleave ops to work at subgroup level.
1456struct WgToSgVectorDeinterleaveOp
1457 : public OpConversionPattern<vector::DeinterleaveOp> {
1458 using OpConversionPattern<vector::DeinterleaveOp>::OpConversionPattern;
1459
1460 LogicalResult
1461 matchAndRewrite(vector::DeinterleaveOp op, OneToNOpAdaptor adaptor,
1462 ConversionPatternRewriter &rewriter) const override {
1463 SmallVector<Value> newRes1Ops;
1464 SmallVector<Value> newRes2Ops;
1465
1466 for (auto src : adaptor.getSource()) {
1467 auto newDeinterleave =
1468 vector::DeinterleaveOp::create(builder&: rewriter, location: op.getLoc(), source: src);
1469 newRes1Ops.push_back(Elt: newDeinterleave.getResultEven());
1470 newRes2Ops.push_back(Elt: newDeinterleave.getResultOdd());
1471 }
1472
1473 SmallVector<SmallVector<Value>> results = {newRes1Ops, newRes2Ops};
1474 rewriter.replaceOpWithMultiple(op, newValues&: results);
1475 return success();
1476 }
1477};
1478
1479} // namespace
1480
1481namespace mlir {
1482namespace xegpu {
1483void populateXeGPUWgToSgDistributeTypeConversions(TypeConverter &converter,
1484 Operation *topLevelOp) {
1485 // Pass through all types by default.
1486 converter.addConversion(callback: [](Type type) -> Type { return type; });
1487
1488 // For TensorDescType, convert WG-level tensor descs to N SG-level descs.
1489 converter.addConversion(
1490 callback: [](xegpu::TensorDescType type,
1491 SmallVectorImpl<Type> &result) -> std::optional<LogicalResult> {
1492 xegpu::DistributeLayoutAttr layout = type.getLayoutAttr();
1493 if (!layout || !layout.isForWorkgroup())
1494 return std::nullopt;
1495
1496 Type elemTy = type.getElementType();
1497 ArrayRef<int64_t> shape = type.getShape();
1498
1499 int count;
1500 SmallVector<int64_t> subShape;
1501 std::tie(args&: subShape, args&: count) = getSgShapeAndCount(shape, layout);
1502
1503 layout = layout.dropSgLayoutAndData();
1504
1505 auto newTy = xegpu::TensorDescType::get(
1506 context: type.getContext(), shape: subShape, elementType: elemTy, encoding: type.getEncoding(), layout);
1507 result.append(NumInputs: count, Elt: newTy);
1508 return success();
1509 });
1510
1511 // Context-aware VectorType conversion based on sg_layout/sg_data
1512 // (1:1 shape-changing or 1:N).
1513 auto getSubShapeAndCount = [](VectorType vecTy,
1514 xegpu::DistributeLayoutAttr layout)
1515 -> std::pair<SmallVector<int64_t>, int> {
1516 if (!layout.isForWorkgroup())
1517 return {{}, 0};
1518 return getSgShapeAndCount(shape: vecTy.getShape(), layout);
1519 };
1520 auto loopArgTypes =
1521 xegpu::precomputeLoopBlockArgTypes(topLevelOp, getSubShapeAndCount);
1522 xegpu::addVectorTypeConversion(converter, getSubShapeAndCount,
1523 loopArgTypes: std::move(loopArgTypes));
1524}
1525
1526void populateXeGPUWgToSgDistributePatterns(RewritePatternSet &patterns) {
1527 patterns.add<WgToSgCreateNdOp, WgToSgLoadNdOp, WgToSgStoreNdOp, WgToSgDpasOp,
1528 WgToSgDpasMxOp, WgToSgPrefetchNdOp, WgToSgElementwiseOp,
1529 WgToSgVectorBroadcastOp, WgToSgConvertLayoutOp,
1530 WgToSgArithConstantOp, WgToSgLoadGatherOp, WgToSgStoreScatterOp,
1531 WgToSgLoadMatrixOp, WgToSgStoreMatrixOp, WgToSgVectorStepOp,
1532 WgToSgVectorShapeCastOp, WgToSgMultiDimReductionOp,
1533 WgToSgVectorTransposeOp, WgToSgVectorConstantMaskOp,
1534 WgToSgVectorCreateMaskOp, WgToSgVectorBitCastOp,
1535 WgToSgVectorInterleaveOp, WgToSgVectorDeinterleaveOp>(
1536 arg: patterns.getContext());
1537}
1538} // namespace xegpu
1539} // namespace mlir
1540
1541namespace {
1542struct XeGPUWgToSgDistributePass
1543 : public xegpu::impl::XeGPUWgToSgDistributeBase<XeGPUWgToSgDistributePass> {
1544 void runOnOperation() override;
1545};
1546} // namespace
1547
1548void XeGPUWgToSgDistributePass::runOnOperation() {
1549
1550 Operation *op = getOperation();
1551 if (!xegpu::recoverTemporaryLayouts(rootOp: op)) {
1552 signalPassFailure();
1553 return;
1554 }
1555
1556 // Collect existing UnrealizedConversionCastOps. These must be preserved.
1557 llvm::SmallSetVector<UnrealizedConversionCastOp, 8> existingCasts;
1558 getOperation()->walk(
1559 callback: [&](UnrealizedConversionCastOp castOp) { existingCasts.insert(X: castOp); });
1560
1561 // Perform workgroup to subgroup distribution for TensorDesc and Vector
1562 // values, as well as XeGPU, Arith, and Vector operations. Uses a
1563 // context-aware type converter that inspects Values to retrieve the
1564 // distribute layout attribute for 1:N type conversion.
1565 MLIRContext *ctx = &getContext();
1566 RewritePatternSet patterns(ctx);
1567 ConversionTarget target(*ctx);
1568 TypeConverter converter;
1569 // Source (N:1) and target (1:1) materializations using
1570 // UnrealizedConversionCastOp.
1571 auto materializeCast = [](OpBuilder &builder, Type type, ValueRange inputs,
1572 Location loc) -> Value {
1573 return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: type, operands: inputs)
1574 .getResult(i: 0);
1575 };
1576 converter.addSourceMaterialization(callback&: materializeCast);
1577 converter.addTargetMaterialization(callback&: materializeCast);
1578 xegpu::populateXeGPUWgToSgDistributeTypeConversions(converter,
1579 topLevelOp: getOperation());
1580
1581 auto getTensorDescType = [](Operation *op) -> xegpu::TensorDescType {
1582 if (auto createOp = dyn_cast<xegpu::CreateNdDescOp>(Val: op))
1583 return createOp.getType();
1584 if (auto loadOp = dyn_cast<xegpu::LoadNdOp>(Val: op))
1585 return loadOp.getTensorDescType();
1586 if (auto storeOp = dyn_cast<xegpu::StoreNdOp>(Val: op))
1587 return storeOp.getTensorDescType();
1588 if (auto prefetchOp = dyn_cast<xegpu::PrefetchNdOp>(Val: op))
1589 return prefetchOp.getTensorDescType();
1590 return xegpu::TensorDescType();
1591 };
1592
1593 auto isLegal = [&](xegpu::DistributeLayoutAttr layout) -> bool {
1594 return !layout || !layout.isForWorkgroup();
1595 };
1596
1597 target.addDynamicallyLegalOp<xegpu::CreateNdDescOp, xegpu::LoadNdOp,
1598 xegpu::StoreNdOp, xegpu::PrefetchNdOp>(
1599 callback: [=](Operation *op) -> bool {
1600 auto tdescTy = getTensorDescType(op);
1601 auto layout = dyn_cast_if_present<xegpu::DistributeLayoutAttr>(
1602 Val: tdescTy.getLayout());
1603 return isLegal(layout);
1604 });
1605
1606 target.addDynamicallyLegalOp<xegpu::DpasOp>(callback: [=](xegpu::DpasOp op) -> bool {
1607 auto layout = op.getLayoutCdAttr();
1608 return isLegal(layout);
1609 });
1610
1611 target.addDynamicallyLegalOp<xegpu::DpasMxOp>(
1612 callback: [=](xegpu::DpasMxOp op) -> bool {
1613 auto layout = op.getLayoutCdAttr();
1614 return isLegal(layout);
1615 });
1616
1617 target.addDynamicallyLegalOp<xegpu::LoadMatrixOp>(
1618 callback: [=](xegpu::LoadMatrixOp op) -> bool {
1619 return isLegal(op.getLayoutAttr());
1620 });
1621
1622 target.addDynamicallyLegalOp<xegpu::StoreMatrixOp>(
1623 callback: [=](xegpu::StoreMatrixOp op) -> bool {
1624 return isLegal(op.getLayoutAttr());
1625 });
1626
1627 target.addDynamicallyLegalOp<arith::ConstantOp>(
1628 callback: [=](arith::ConstantOp op) -> bool {
1629 auto vecType = dyn_cast<VectorType>(Val: op.getType());
1630 if (!vecType)
1631 return true;
1632
1633 auto layout =
1634 xegpu::getTemporaryLayout(operandOrResult: dyn_cast<OpResult>(Val: op.getResult()));
1635 return isLegal(layout);
1636 });
1637
1638 target.addDynamicallyLegalOp<
1639 vector::ShapeCastOp, vector::StepOp, vector::TransposeOp,
1640 vector::BroadcastOp, vector::MultiDimReductionOp, vector::ConstantMaskOp,
1641 vector::CreateMaskOp, vector::BitCastOp, vector::InterleaveOp,
1642 vector::DeinterleaveOp>(callback: [=](Operation *op) -> bool {
1643 // Check for either a SliceAttr or LayoutAttr on the result.
1644 auto layout =
1645 xegpu::getTemporaryLayout(operandOrResult: dyn_cast<OpResult>(Val: op->getResult(idx: 0)));
1646 return isLegal(layout);
1647 });
1648
1649 target.addDynamicallyLegalOp<xegpu::LoadGatherOp>(
1650 callback: [=](xegpu::LoadGatherOp op) -> bool {
1651 auto layout = op.getLayoutAttr();
1652 return isLegal(layout);
1653 });
1654
1655 target.addDynamicallyLegalOp<xegpu::StoreScatterOp>(
1656 callback: [=](xegpu::StoreScatterOp op) -> bool {
1657 auto layout = op.getLayoutAttr();
1658 return isLegal(layout);
1659 });
1660
1661 target.addDynamicallyLegalOp<xegpu::ConvertLayoutOp>(
1662 callback: [=](xegpu::ConvertLayoutOp op) -> bool {
1663 return isLegal(op.getEffectiveInputLayout()) &&
1664 isLegal(op.getTargetLayout());
1665 });
1666
1667 target.addDynamicallyLegalDialect<math::MathDialect, arith::ArithDialect>(
1668 callback: [=](Operation *op) -> std::optional<bool> {
1669 // Only handle elementwise mappable ops
1670 if (!OpTrait::hasElementwiseMappableTraits(op))
1671 return true;
1672
1673 VectorType resultType =
1674 dyn_cast<VectorType>(Val: op->getResult(idx: 0).getType());
1675 if (!resultType)
1676 return true;
1677
1678 // Check if all operands are vectors of the same shape
1679 // TODO: Support other types.
1680 for (Value operand : op->getOperands()) {
1681 VectorType operandType = dyn_cast<VectorType>(Val: operand.getType());
1682 if (!operandType || operandType.getShape() != resultType.getShape()) {
1683 return true;
1684 }
1685 }
1686
1687 xegpu::DistributeLayoutAttr layout =
1688 xegpu::getTemporaryLayout(operandOrResult: op->getResult(idx: 0));
1689 return isLegal(layout);
1690 });
1691
1692 target.addLegalOp<UnrealizedConversionCastOp>();
1693
1694 target.markUnknownOpDynamicallyLegal(fn: [](Operation *) { return true; });
1695
1696 scf::populateSCFStructuralTypeConversionsAndLegality(typeConverter: converter, patterns,
1697 target);
1698 xegpu::populateXeGPUWgToSgDistributePatterns(patterns);
1699 if (failed(
1700 Result: applyPartialConversion(op: getOperation(), target, patterns: std::move(patterns))))
1701 return signalPassFailure();
1702
1703 // Fold cancelling cast chains and erase dead casts.
1704 xegpu::cleanupUnrealizedConversionCasts(root: getOperation(), existingCasts);
1705 xegpu::removeTemporaryLayoutAttrs(op: getOperation());
1706}
1707