1//===---- XeGPULayoutImpl.cpp - MLIR Utilities for XeGPUOps
2//------------------===//
3//
4// Part of the MLIR Project, under the Apache License v2.0 with LLVM Exceptions.
5// See https://llvm.org/LICENSE.txt for license information.
6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements layout utility functions for XeGPU dialect
11// transformation.
12//
13//===----------------------------------------------------------------------===//
14
15#include "mlir/Dialect/XeGPU/Transforms/XeGPULayoutImpl.h"
16#include "mlir/Dialect/Arith/IR/Arith.h"
17#include "mlir/Dialect/Func/IR/FuncOps.h"
18#include "mlir/Dialect/GPU/IR/GPUDialect.h"
19#include "mlir/Dialect/LLVMIR/XeVMDialect.h"
20#include "mlir/Dialect/SCF/Transforms/Patterns.h"
21#include "mlir/Dialect/Utils/IndexingUtils.h"
22#include "mlir/Dialect/Vector/IR/VectorOps.h"
23#include "mlir/Dialect/XeGPU/IR/XeGPU.h"
24#include "mlir/IR/Builders.h"
25#include "mlir/IR/IRMapping.h"
26#include "mlir/IR/Operation.h"
27#include "mlir/IR/ValueRange.h"
28#include "mlir/Interfaces/ControlFlowInterfaces.h"
29#include "mlir/Interfaces/LoopLikeInterface.h"
30#include "mlir/Interfaces/SideEffectInterfaces.h"
31#include "mlir/Transforms/DialectConversion.h"
32#include "llvm/ADT/DenseSet.h"
33#include "llvm/ADT/PostOrderIterator.h"
34#include "llvm/Support/FormatVariadic.h"
35#include <cstdint>
36#include <functional>
37#include <numeric>
38
39using namespace mlir;
40
41SmallVector<NamedAttribute>
42xegpu::dropSgLayoutAndDataOnAttrs(ArrayRef<NamedAttribute> attrs) {
43 SmallVector<NamedAttribute> out;
44 out.reserve(N: attrs.size());
45
46 for (auto attr : attrs) {
47 if (auto dist = dyn_cast<xegpu::DistributeLayoutAttr>(Val: attr.getValue())) {
48 auto newLayout = dist.dropSgLayoutAndData();
49 if (newLayout)
50 out.emplace_back(Args: attr.getName(), Args&: newLayout);
51 } else {
52 out.push_back(Elt: attr);
53 }
54 }
55
56 return out;
57}
58
59SmallVector<NamedAttribute>
60xegpu::dropInstDataOnAttrs(ArrayRef<NamedAttribute> attrs) {
61 SmallVector<NamedAttribute> out;
62 out.reserve(N: attrs.size());
63
64 for (auto attr : attrs) {
65 if (auto dist = dyn_cast<xegpu::DistributeLayoutAttr>(Val: attr.getValue())) {
66 auto newLayout = dist.dropInstData();
67 if (newLayout)
68 out.emplace_back(Args: attr.getName(), Args&: newLayout);
69 } else {
70 out.push_back(Elt: attr);
71 }
72 }
73
74 return out;
75}
76
77void xegpu::dropInstDataOnInherentAttrs(Operation *op) {
78 op->getName().walkInherentAttrs(op, visitor: [](StringRef, Attribute &attr) {
79 if (auto dist = dyn_cast<xegpu::DistributeLayoutAttr>(Val&: attr))
80 attr = dist.dropInstData();
81 });
82}
83
84// Sets the layout on a TensorDesc value by updating its type to include
85// the given layout, if the type does not already have a layout attached.
86static void setTensorDescLayout(Value val, xegpu::DistributeLayoutAttr layout) {
87 auto tensorDescTy = dyn_cast<xegpu::TensorDescType>(Val: val.getType());
88 if (!tensorDescTy || tensorDescTy.getLayoutAttr())
89 return;
90 auto typeWithLayout = xegpu::TensorDescType::get(
91 context: tensorDescTy.getContext(), shape: tensorDescTy.getShape(),
92 elementType: tensorDescTy.getElementType(), encoding: tensorDescTy.getEncoding(), layout);
93 val.setType(typeWithLayout);
94}
95
96// the walkRegionBackward() is a recursive function
97// the input rootOp is the function operation, which is also a region op.
98// it recursively processes the region op in reverse topological order.
99static void walkRegionBackward(Region &region,
100 llvm::function_ref<void(Operation *)> visit) {
101
102 // Use post-order traversal to process blocks in reverse topological order.
103 // This ensures that use blocks are visited before def blocks, which is
104 // required for backward layout propagation.
105 if (region.empty())
106 return;
107 llvm::ReversePostOrderTraversal<Region *> rpot(&region);
108 SmallVector<Block *> blocks(rpot.begin(), rpot.end());
109 for (Block *block : llvm::reverse(C&: blocks)) {
110 // ops: back -> front
111 for (Operation &op : llvm::reverse(C&: *block)) {
112 // make sure we first visit inside the region op (so yield op first)
113 // and then move to region op itself
114 // Regions are iterated in forward order so that for multi-region ops
115 // like scf.while, earlier regions (e.g., "before/cond") are processed
116 // first. This ensures that when a later region's terminator (e.g., "do"
117 // yield) needs the layout of an earlier region's block args, those
118 // layouts are already available from use points.
119 for (Region &nested : op.getRegions())
120 walkRegionBackward(region&: nested, visit);
121
122 visit(&op);
123 }
124 }
125}
126
127static xegpu::DistributeLayoutAttr getLayoutFromUsePoints(Value result) {
128 xegpu::DistributeLayoutAttr layout = nullptr;
129 for (OpOperand &use : result.getUses()) {
130 if (auto tmpLayout = xegpu::getDistributeLayoutAttr(opr: use)) {
131 if (!layout)
132 layout = tmpLayout;
133 break;
134 }
135 }
136 return layout;
137}
138
139static constexpr unsigned kMaxRematerializableChainOps = 16;
140
141/// Accept pure vector generators and elementwise ops that preserve vector
142/// shape. Generators have no vector operands; elementwise ops share their
143/// result layout with their vector operands, so the chain needs no layout
144/// inference.
145static bool isLayoutPreservingVectorProducer(Operation *op) {
146 if (op->getNumResults() != 1 || op->getNumRegions() != 0 ||
147 !isa<VectorType>(Val: op->getResult(idx: 0).getType()) || !isMemoryEffectFree(op))
148 return false;
149 if (auto constant = dyn_cast<arith::ConstantOp>(Val: op)) {
150 auto value = dyn_cast<DenseElementsAttr>(Val: constant.getValue());
151 return value && value.isSplat();
152 }
153 if (isa<vector::StepOp, vector::CreateMaskOp>(Val: op))
154 return true;
155 if (auto broadcast = dyn_cast<vector::BroadcastOp>(Val: op))
156 return !isa<VectorType>(Val: broadcast.getSource().getType());
157 return OpTrait::hasElementwiseMappableTraits(op);
158}
159
160/// Collect each eligible producer once, producers first, for cloning with an
161/// IRMapping. Reject chains exceeding the duplication limit or reaching an
162/// unsupported producer or vector block argument.
163static bool collectRematerializableChain(Operation *root,
164 SmallVectorImpl<Operation *> &chain) {
165 // Re-push each op below its producers to obtain an iterative post-order walk.
166 SmallVector<std::pair<Operation *, bool>> worklist{{root, false}};
167 DenseSet<Operation *> visited;
168 while (!worklist.empty()) {
169 auto [op, expanded] = worklist.pop_back_val();
170 if (expanded) {
171 chain.push_back(Elt: op);
172 continue;
173 }
174 if (!visited.insert(V: op).second)
175 continue;
176 if (visited.size() > kMaxRematerializableChainOps ||
177 !isLayoutPreservingVectorProducer(op))
178 return false;
179 worklist.emplace_back(Args&: op, Args: true);
180 for (Value operand : op->getOperands()) {
181 if (!isa<VectorType>(Val: operand.getType()))
182 continue;
183 Operation *producer = operand.getDefiningOp();
184 if (!producer)
185 return false;
186 if (!visited.contains(V: producer))
187 worklist.emplace_back(Args&: producer, Args: false);
188 }
189 }
190 return true;
191}
192
193bool xegpu::isTriviallyRematerializable(Operation *op) {
194 if (!op)
195 return false;
196 SmallVector<Operation *> chain;
197 return collectRematerializableChain(root: op, chain);
198}
199
200Value xegpu::rematerialize(OpBuilder &builder, Value value) {
201 Operation *producer = value.getDefiningOp();
202 SmallVector<Operation *> chain;
203 if (!producer || !collectRematerializableChain(root: producer, chain))
204 return nullptr;
205 // Clone producers first so vector operands map to clones; scalars stay
206 // shared.
207 OpBuilder::InsertionGuard guard(builder);
208 IRMapping mapping;
209 for (Operation *op : chain) {
210 builder.setInsertionPointAfter(op);
211 Operation *clone = builder.clone(op&: *op, mapper&: mapping);
212 removeLayoutAttrs(op: clone);
213 }
214 return mapping.lookup(from: value);
215}
216
217// For regular operations: First the result layouts are propagated from uses.
218// Then the result layouts are propagated to uses (operands).
219static void propagateResultsToRegularOperands(Operation *op) {
220 if (op->getNumResults() == 0)
221 return;
222 if (op->getNumResults() > 1 && !isa<vector::DeinterleaveOp>(Val: op))
223 return;
224 OpResult result = op->getResult(idx: 0);
225 xegpu::DistributeLayoutAttr resLayout = getLayoutFromUsePoints(result);
226 Type resultType = result.getType();
227
228 if (!resLayout)
229 return;
230
231 // Recover layout for TensorDesc type results by updating the type to include
232 // the layout. For vector type
233 if (isa<xegpu::TensorDescType>(Val: resultType))
234 setTensorDescLayout(val: result, layout: resLayout);
235
236 // Recover layout for vector type results, or for multi-reduction ops which
237 // may reduce to a scalar that still needs a layout.
238 if (isa<VectorType>(Val: resultType) || isa<vector::MultiDimReductionOp>(Val: op))
239 xegpu::setTemporaryLayout(operandOrResult: result, layout: resLayout);
240
241 if (isa<vector::DeinterleaveOp>(Val: op))
242 xegpu::setTemporaryLayout(operandOrResult: op->getResult(idx: 1), layout: resLayout);
243
244 for (OpOperand &opr : op->getOpOperands()) {
245 xegpu::DistributeLayoutAttr operandLayout =
246 xegpu::inferSourceLayoutFromResultForNonAnchorOp(operand&: opr, resLayout);
247 if (isa<VectorType>(Val: opr.get().getType()) && operandLayout)
248 xegpu::setTemporaryLayout(operandOrResult: opr, layout: operandLayout);
249 }
250}
251
252// Propagate layout from region op results and sibling region block args
253// to yield/condition operands. For each successor of this terminator:
254// - Parent successor: propagate from parent op's result layouts (use points).
255// - Region successor: propagate from target region's block arg layouts (use
256// points), e.g., scf.yield in "after/do" region propagates to "before/cond"
257// block args.
258static void propagateRegionResultsToYieldOperands(
259 mlir::RegionBranchTerminatorOpInterface yieldOp) {
260 auto regionBranchOp =
261 dyn_cast<RegionBranchOpInterface>(Val: yieldOp->getParentOp());
262 if (!regionBranchOp)
263 return;
264
265 SmallVector<RegionSuccessor> successors;
266 SmallVector<Attribute> operandAttrs(yieldOp->getNumOperands(), nullptr);
267 yieldOp.getSuccessorRegions(operands: operandAttrs, regions&: successors);
268
269 for (const RegionSuccessor &successor : successors) {
270 OperandRange succOps = yieldOp.getSuccessorOperands(successor);
271 if (succOps.empty())
272 continue;
273 unsigned beginIdx = succOps.getBeginOperandIndex();
274 ValueRange successorInputs = regionBranchOp.getSuccessorInputs(successor);
275 unsigned count = std::min<unsigned>(a: succOps.size(), b: successorInputs.size());
276
277 for (unsigned i = 0; i < count; ++i) {
278 xegpu::DistributeLayoutAttr layout;
279 if (successor.isOperation()) {
280 // For parent successor, get layout from external use points of the
281 // parent op's results.
282 auto regionResult = regionBranchOp->getResult(idx: i);
283 layout = getLayoutFromUsePoints(result: regionResult);
284 if (layout) {
285 // set layout for the region op, like scf.loop
286 xegpu::setTemporaryLayout(operandOrResult: regionResult, layout);
287 if (isa<xegpu::TensorDescType>(Val: regionResult.getType()))
288 setTensorDescLayout(val: regionResult, layout);
289 }
290 } else {
291 // For region successor, get layout from the target region's block
292 // arg use points (e.g., "before/cond" region args for scf.while
293 // "after/do" yield).
294 layout = getLayoutFromUsePoints(result: successorInputs[i]);
295 }
296 if (!layout)
297 continue;
298 auto operandType = succOps[i].getType();
299 if (isa<VectorType>(Val: operandType) ||
300 dyn_cast<xegpu::TensorDescType>(Val&: operandType))
301 // recover layout for yield op operands
302 xegpu::setTemporaryLayout(operandOrResult: yieldOp->getOpOperand(idx: beginIdx + i), layout);
303 }
304 }
305}
306
307/// Assign a layout to a region op's results (e.g. scf.for) using the layout of
308/// the terminator operands that the region forwards to them. For each operand a
309/// terminator (e.g. scf.yield) forwards to a successor input, if that input is
310/// a region op result, the operand's layout is written onto the result.
311/// clang-format off
312/// Example: scf.for ... iter_args(...) -> (out types) {
313/// ...
314/// scf.yield ... : (yield types)
315/// }
316/// clang-format on
317/// Having a layout on the region op result lets a later step attach a
318/// convert_layout as a use to resolve the region op's no-use case.
319/// Block-argument successors are left untouched.
320LogicalResult xegpu::propagateYieldOperandsToRegionResults(
321 mlir::RegionBranchTerminatorOpInterface terminator,
322 xegpu::GetLayoutFnTy getLayoutOfValue) {
323 // Only process if the terminator is inside a region branch op.
324 auto branchOp = dyn_cast<RegionBranchOpInterface>(Val: terminator->getParentOp());
325 if (!branchOp)
326 return success();
327
328 RegionBranchSuccessorMapping mapping;
329 branchOp.getSuccessorOperandInputMapping(mapping,
330 src: RegionBranchPoint(terminator));
331 for (const auto &[successorOperand, successorInputs] : mapping) {
332 for (Value successorInput : successorInputs) {
333 Type inputType = successorInput.getType();
334 // We only need to operate on vector types.
335 if (!isa<VectorType>(Val: inputType))
336 continue;
337 xegpu::DistributeLayoutAttr successorOperandLayout =
338 getLayoutOfValue(successorOperand->get());
339
340 // The forwarded operand must carry a layout to propagate.
341 if (!successorOperandLayout)
342 return failure();
343 // Assign the yield operand's layout to the region op result it feeds.
344 if (auto result = dyn_cast<OpResult>(Val&: successorInput))
345 xegpu::setDistributeLayoutAttr(Result: result, layout: successorOperandLayout);
346 // Restrict the input IR: a successor argument that is not tied to an init
347 // operand (scf.while's "after" arguments) must be fed by a pass-through,
348 // because nothing else identifies which value the region carries. Its
349 // layout is then that of the forwarded argument's init operand.
350 if (auto arg = dyn_cast<BlockArgument>(Val&: successorInput)) {
351 auto loop =
352 dyn_cast<LoopLikeOpInterface>(Val: arg.getOwner()->getParentOp());
353 bool tiedToInit = loop && loop.getTiedLoopInit(bbArg: arg);
354 if (!tiedToInit && !isa<BlockArgument>(Val: successorOperand->get()))
355 return terminator->emitError(
356 message: "unsupported region structure: the successor argument it feeds "
357 "is not tied to an init operand, so its value must be passed "
358 "through from predecessor region argument.");
359 }
360 }
361 }
362 return success();
363}
364
365// Propagate layout from region arguments to region op's init operands. This
366// sets the temporary layout for region arguments and init operands.
367LogicalResult
368xegpu::propagateRegionArgsToInits(mlir::RegionBranchOpInterface regionOp,
369 xegpu::GetLayoutFnTy getLayoutOfValue) {
370 // Iterate all regions of the region op. For each block argument that has a
371 // layout (obtained via `getLayoutOfValue`), trace back to find the
372 // corresponding init operand of the regionOp and set the layout on it.
373 // This works generically for scf.for, scf.while, and other
374 // RegionBranchOpInterface ops.
375 for (Region &region : regionOp->getRegions()) {
376 RegionSuccessor regionSuccessor(&region);
377 // Use getSuccessorInputs to get the block arguments that correspond to
378 // predecessor operands. This correctly handles ops like scf.for where
379 // the induction variable is a block arg but not a successor input.
380 ValueRange successorInputs = regionOp.getSuccessorInputs(successor: regionSuccessor);
381 for (auto [inputIdx, regionArg] : llvm::enumerate(First&: successorInputs)) {
382 auto layout = getLayoutOfValue(regionArg);
383 if (!layout)
384 continue;
385
386 // Recover layout for tensor_desc block args by updating the type.
387 if (isa<xegpu::TensorDescType>(Val: regionArg.getType()))
388 setTensorDescLayout(val: regionArg, layout);
389
390 // Recover layout for region op operands, like scf.for's init operands.
391 // Find all predecessor values that flow into this block argument.
392 SmallVector<Value> predValues;
393 regionOp.getPredecessorValues(successor: regionSuccessor, index: inputIdx, predecessorValues&: predValues);
394 for (Value predVal : predValues) {
395 // Match predecessor value to an operand of the regionOp.
396 for (OpOperand &operand : regionOp->getOpOperands()) {
397 if (operand.get() == predVal)
398 xegpu::setTemporaryLayout(operandOrResult: operand, layout);
399 }
400 }
401 }
402 }
403 return success();
404}
405
406// Prerequisite for Layout Recovery
407// It relies on the following invariant:
408// 1. there is no layout conflict between different uses of the same definition.
409// 2. each definition has a well-defined layout requirement at its use point.
410// - Every definition must have at least one use that appears after it in
411// topological order.
412// - TODO: If a definition has no such use (e.g., a loop result or region
413// output), an explicit convert_layout operation is inserted to create a
414// use.
415// - Only the result of convert_layout is permitted to have no subsequent
416// use.
417//
418// The recovery proceeds by scanning the operation in reverse topological order
419// as follows:
420// For regular operations: First the result layouts are propagated from uses.
421// Then the result layouts are propagated to operands.
422//
423// For region operations (e.g., loops):
424// - When backward propagation reaches a region op, it sets the layout of
425// the region op’s results according to use points like regular ops.
426// - Then, the result layouts (such as a loop output) are propagated to
427// their corresponding operands in the yield.
428// - When backward propagation reaches the first operation inside the
429// region, the pass examines the region op’s initialization list,
430// propagating from region arguments to the corresponding initialization
431// operands.
432// - This ensures that layouts are consistently propagated
433// across region boundaries while preserving a single well-defined use for
434// each definition at the region-op level.
435bool xegpu::recoverTemporaryLayouts(Operation *rootOp) {
436 auto processFunc = [&](Region &body, StringRef funcName) {
437 walkRegionBackward(region&: body, visit: [&](Operation *op) {
438 if (auto regionOp = dyn_cast<mlir::RegionBranchOpInterface>(Val: op)) {
439 (void)xegpu::propagateRegionArgsToInits(regionOp,
440 getLayoutOfValue: getLayoutFromUsePoints);
441 } else if (auto yieldOp =
442 dyn_cast<mlir::RegionBranchTerminatorOpInterface>(Val: op)) {
443 propagateRegionResultsToYieldOperands(yieldOp);
444 } else if (!dyn_cast<xegpu::AnchorLayoutInterface>(Val: op)) {
445 propagateResultsToRegularOperands(op);
446 }
447 });
448 };
449 removeTemporaryLayoutAttrs(op: rootOp);
450 rootOp->walk(callback: [&](func::FuncOp func) {
451 processFunc(func.getBody(), func.getSymName());
452 });
453 rootOp->walk(callback: [&](gpu::GPUFuncOp func) {
454 processFunc(func.getBody(), func.getName());
455 });
456
457 return true;
458}
459
460template <typename T, typename>
461void xegpu::removeLayoutAttr(const T &operandOrResult) {
462 Operation *owner = operandOrResult.getOwner();
463 std::string name = xegpu::getTemporaryLayoutName(operandOrResult);
464 if (owner->hasDiscardableAttrOfType<DistributeLayoutAttr>(name))
465 owner->removeDiscardableAttr(name);
466}
467
468// Explicit instantiation for OpResult
469template void
470xegpu::removeLayoutAttr<mlir::OpResult>(const mlir::OpResult &result);
471
472// Explicit instantiation for OpOperand
473template void
474xegpu::removeLayoutAttr<mlir::OpOperand>(const mlir::OpOperand &operand);
475
476void xegpu::removeLayoutAttrs(Operation *op) {
477 op->walk(callback: [&](Operation *nestOp) {
478 // Remove all attributes of DistributeLayoutAttr type
479 SmallVector<StringAttr> attrsToRemove;
480 for (auto namedAttr : nestOp->getDiscardableAttrDictionary().getValue()) {
481 if (isa<DistributeLayoutAttr>(Val: namedAttr.getValue()))
482 attrsToRemove.push_back(Elt: namedAttr.getName());
483 }
484 for (auto attrName : attrsToRemove)
485 nestOp->removeDiscardableAttr(name: attrName);
486 });
487}
488
489void xegpu::removeTemporaryLayoutAttrs(Operation *op) {
490 op->walk(callback: [&](Operation *nestOp) {
491 SmallVector<StringAttr> attrsToRemove;
492 for (auto namedAttr : nestOp->getDiscardableAttrDictionary().getValue()) {
493 if (isa<xegpu::DistributeLayoutAttr>(Val: namedAttr.getValue()))
494 attrsToRemove.push_back(Elt: namedAttr.getName());
495 }
496 for (auto attrName : attrsToRemove)
497 nestOp->removeDiscardableAttr(name: attrName);
498 });
499}
500
501static xegpu::LayoutAttr buildInstDataLayoutWithLane(
502 mlir::MLIRContext *context, ArrayRef<int64_t> instData,
503 ArrayRef<int64_t> laneLayout, ArrayRef<int64_t> laneData,
504 DenseI32ArrayAttr orderAttr = nullptr) {
505 auto toI32Attr = [&](auto range) {
506 SmallVector<int32_t> v(range.begin(), range.end());
507 return DenseI32ArrayAttr::get(context, content: v);
508 };
509 return xegpu::LayoutAttr::get(context, /*sg_layout=*/nullptr,
510 /*sg_data=*/nullptr, inst_data: toI32Attr(instData),
511 lane_layout: toI32Attr(laneLayout), lane_data: toI32Attr(laneData),
512 order: orderAttr);
513}
514
515static bool isValidLaneLayout(ArrayRef<int64_t> dataShape,
516 ArrayRef<int64_t> laneLayout,
517 ArrayRef<int64_t> laneData) {
518 return !llvm::any_of(Range: llvm::seq<int>(Begin: 0, End: dataShape.size()), P: [&](int dim) {
519 return dataShape[dim] % (laneLayout[dim] * laneData[dim]) != 0;
520 });
521}
522
523static xegpu::LayoutAttr
524buildLaneLayout(mlir::MLIRContext *context, ArrayRef<int64_t> laneLayout,
525 ArrayRef<int64_t> laneData,
526 DenseI32ArrayAttr orderAttr = nullptr) {
527 auto toI32Attr = [&](auto range) {
528 SmallVector<int32_t> v(range.begin(), range.end());
529 return DenseI32ArrayAttr::get(context, content: v);
530 };
531 return xegpu::LayoutAttr::get(context, /*sg_layout=*/nullptr,
532 /*sg_data=*/nullptr,
533 /*inst_data=*/nullptr, lane_layout: toI32Attr(laneLayout),
534 lane_data: toI32Attr(laneData), order: orderAttr);
535}
536
537static xegpu::LayoutAttr
538buildLayout(mlir::MLIRContext *context, ArrayRef<int64_t> sgLayout,
539 ArrayRef<int64_t> sgData, ArrayRef<int64_t> instData,
540 ArrayRef<int64_t> laneLayout, ArrayRef<int64_t> laneData,
541 DenseI32ArrayAttr orderAttr = nullptr) {
542 auto toI32Attr = [&](auto range) {
543 SmallVector<int32_t> v(range.begin(), range.end());
544 return DenseI32ArrayAttr::get(context, content: v);
545 };
546 return xegpu::LayoutAttr::get(
547 context, sg_layout: sgLayout.empty() ? nullptr : toI32Attr(sgLayout),
548 sg_data: sgData.empty() ? nullptr : toI32Attr(sgData),
549 inst_data: instData.empty() ? nullptr : toI32Attr(instData),
550 lane_layout: laneLayout.empty() ? nullptr : toI32Attr(laneLayout),
551 lane_data: laneData.empty() ? nullptr : toI32Attr(laneData), order: orderAttr);
552}
553
554static xegpu::LayoutAttr buildSgLayout(mlir::MLIRContext *context,
555 ArrayRef<int64_t> wgTileShape,
556 ArrayRef<int64_t> sgLayout,
557 int dimK = -1,
558 DenseI32ArrayAttr orderAttr = nullptr) {
559 SmallVector<int64_t> sgData(sgLayout.size());
560 for (int dim = 0; dim < (int)sgLayout.size(); ++dim) {
561 if (dim == dimK)
562 sgData[dim] = wgTileShape[dim];
563 else
564 sgData[dim] = wgTileShape[dim] / sgLayout[dim];
565 }
566 return buildLayout(context, sgLayout, sgData,
567 /*inst_data=*/instData: {}, /*lane_layout=*/laneLayout: {},
568 /*lane_data=*/laneData: {}, /*order=*/orderAttr: nullptr);
569}
570
571/// Infers the source layout attribute for a broadcast operation given the
572/// result layout attribute, result shape, source shape.
573xegpu::DistributeLayoutAttr
574xegpu::inferBroadcastSourceLayout(xegpu::DistributeLayoutAttr resLayout,
575 ArrayRef<int64_t> resShape,
576 ArrayRef<int64_t> srcShape) {
577
578 SmallVector<int64_t> bcastDims;
579 size_t dimDiff = resShape.size() - srcShape.size();
580 auto bcastSourceLayout = resLayout;
581
582 // Right-aligned source in result, look for stretched unit dims.
583 for (size_t i = dimDiff; i < resShape.size(); i++) {
584 if ((srcShape[i - dimDiff] == 1) && (resShape[i] != 1))
585 bcastDims.push_back(Elt: i);
586 }
587
588 // Case UnitDimStretch (e.g., 1x4 -> 4x4): the source layout data field must
589 // be 1.
590 if (!bcastDims.empty())
591 bcastSourceLayout = bcastSourceLayout.setUnitDimData(bcastDims);
592
593 // Case RankDiff:
594 if (dimDiff) {
595 SmallVector<int64_t> sliceDims;
596 bool isOuterDimDiffUnitDims = llvm::all_of(
597 Range: resShape.take_front(N: dimDiff), P: [&](int64_t dim) { return dim == 1; });
598 if (dimDiff && bcastDims.size() == dimDiff && isOuterDimDiffUnitDims) {
599 // Case RankDiffInnerDims (e.g., 1x4 -> 1x16x4):
600 // slice the expanded inner dims
601 sliceDims.assign(in_start: bcastDims.begin(), in_end: bcastDims.end());
602 } else {
603 // Case RankDiffOuterDims (e.g., 1x4 -> 1x1x4):
604 // slice the outer dims
605 llvm::append_range(C&: sliceDims, R: llvm::seq<int64_t>(Begin: 0, End: dimDiff));
606 }
607 bcastSourceLayout = xegpu::SliceAttr::get(
608 context: resLayout.getContext(), parent: bcastSourceLayout,
609 dims: DenseI64ArrayAttr::get(context: resLayout.getContext(), content: sliceDims));
610 }
611 return bcastSourceLayout;
612}
613
614/// Infers the source layout attribute for a reduction operation given the
615/// result layout attribute and reduced dims.
616xegpu::DistributeLayoutAttr
617xegpu::inferMultiReductionSourceLayout(xegpu::DistributeLayoutAttr resLayout,
618 SmallVector<int64_t> reduceDims) {
619
620 assert(isa<xegpu::SliceAttr>(resLayout) &&
621 "reduction result layout must be slice layout");
622
623 xegpu::SliceAttr sliceLayout = dyn_cast<xegpu::SliceAttr>(Val&: resLayout);
624
625 assert((reduceDims == sliceLayout.getDims().asArrayRef()) &&
626 "reduction dims must match with slice dims");
627
628 return sliceLayout.getParent();
629}
630
631xegpu::DistributeLayoutAttr
632xegpu::inferReductionSourceLayout(xegpu::DistributeLayoutAttr resLayout) {
633 return xegpu::inferMultiReductionSourceLayout(resLayout, reduceDims: {0});
634}
635
636/// Infers the source layout attribute for a transpose operation given the
637/// result layout attribute and permutation.
638///
639/// vector.transpose semantics is `result[i] = source[permutation[i]]`, so
640/// `result_layout[i] = source_layout[permutation[i]]`. To recover the source
641/// layout from the result layout we must apply the inverse permutation.
642xegpu::DistributeLayoutAttr
643xegpu::inferTransposeSourceLayout(xegpu::DistributeLayoutAttr resLayout,
644 ArrayRef<int64_t> permutation) {
645 SmallVector<int64_t> inversePermutation =
646 invertPermutationVector(permutation);
647 return resLayout.transposeDims(permutation: inversePermutation);
648}
649
650/// Infers the source layout attribute for a bitcast operation given the
651/// result layout attribute, result element type bitwidth, and source element
652/// type bitwidth.
653xegpu::DistributeLayoutAttr
654xegpu::inferBitCastSourceLayout(xegpu::DistributeLayoutAttr resLayout,
655 int resElemTyBitWidth, int srcElemTyBitWidth) {
656
657 SmallVector<int64_t> sgData = resLayout.getEffectiveSgDataAsInt();
658 SmallVector<int64_t> instData = resLayout.getEffectiveInstDataAsInt();
659 SmallVector<int64_t> laneData = resLayout.getEffectiveLaneDataAsInt();
660 size_t sgDataSize = sgData.size();
661 size_t instDataSize = instData.size();
662 size_t laneDataSize = laneData.size();
663 int64_t sgDataValue = -1;
664 int64_t instDataValue = -1;
665 int64_t laneDataValue = -1;
666 int64_t dim = resLayout.getRank() - 1;
667
668 if (srcElemTyBitWidth <= resElemTyBitWidth) {
669 int bitWidthRatio = resElemTyBitWidth / srcElemTyBitWidth;
670 if (sgDataSize)
671 sgDataValue = sgData.back() * bitWidthRatio;
672 if (instDataSize)
673 instDataValue = instData.back() * bitWidthRatio;
674 if (laneDataSize)
675 laneDataValue = laneData.back() * bitWidthRatio;
676 } else {
677 int bitWidthRatio = srcElemTyBitWidth / resElemTyBitWidth;
678 if (sgDataSize) {
679 assert((sgData.back() % bitWidthRatio) == 0 &&
680 "sgData not divisible by bitWidthRatio");
681 sgDataValue = sgData.back() / bitWidthRatio;
682 }
683 if (instDataSize) {
684 assert((instData.back() % bitWidthRatio) == 0 &&
685 "instData not divisible by bitWidthRatio");
686 instDataValue = instData.back() / bitWidthRatio;
687 }
688 if (laneDataSize) {
689 assert((laneData.back() % bitWidthRatio) == 0 &&
690 "laneData not divisible by bitWidthRatio");
691 laneDataValue = laneData.back() / bitWidthRatio;
692 }
693 }
694
695 xegpu::DistributeLayoutAttr finalSrcLayout;
696 finalSrcLayout =
697 resLayout.setDimData(dim, sgData: sgDataValue, instData: instDataValue, laneData: laneDataValue);
698
699 return finalSrcLayout;
700}
701
702/// Infers the source layout attribute for an interleave operation given the
703/// result layout attribute. Interleave doubles the size of the innermost
704/// dimension, so the layout inference is similar to bitcast where the source
705/// element type is larger than the result element type (ratio = 2).
706xegpu::DistributeLayoutAttr
707xegpu::inferInterleaveSourceLayout(xegpu::DistributeLayoutAttr resLayout) {
708
709 SmallVector<int64_t> sgData = resLayout.getEffectiveSgDataAsInt();
710 SmallVector<int64_t> instData = resLayout.getEffectiveInstDataAsInt();
711 SmallVector<int64_t> laneData = resLayout.getEffectiveLaneDataAsInt();
712 size_t sgDataSize = sgData.size();
713 size_t instDataSize = instData.size();
714 size_t laneDataSize = laneData.size();
715 int64_t sgDataValue = -1;
716 int64_t instDataValue = -1;
717 int64_t laneDataValue = -1;
718 int64_t dim = resLayout.getRank() - 1;
719
720 // Interleave doubles the innermost dimension, so we need to halve the
721 // layout values (similar to bitcast with ratio = 2)
722 constexpr int ratio = 2;
723 if (sgDataSize) {
724 assert((sgData.back() % ratio) == 0 &&
725 "sgData not divisible by interleave ratio");
726 sgDataValue = sgData.back() / ratio;
727 }
728 if (instDataSize) {
729 assert((instData.back() % ratio) == 0 &&
730 "instData not divisible by interleave ratio");
731 instDataValue = instData.back() / ratio;
732 }
733 if (laneDataSize) {
734 assert((laneData.back() % ratio) == 0 &&
735 "laneData not divisible by interleave ratio");
736 laneDataValue = laneData.back() / ratio;
737 }
738
739 return resLayout.setDimData(dim, sgData: sgDataValue, instData: instDataValue, laneData: laneDataValue);
740}
741
742/// Infers the source layout attribute for a deinterleave operation given the
743/// result layout attribute. Deinterleave halves the size of the innermost
744/// dimension, so the layout inference is similar to bitcast where the source
745/// element type is smaller than the result element type (ratio = 2).
746xegpu::DistributeLayoutAttr
747xegpu::inferDeinterleaveSourceLayout(xegpu::DistributeLayoutAttr resLayout) {
748
749 SmallVector<int64_t> sgData = resLayout.getEffectiveSgDataAsInt();
750 SmallVector<int64_t> instData = resLayout.getEffectiveInstDataAsInt();
751 SmallVector<int64_t> laneData = resLayout.getEffectiveLaneDataAsInt();
752 size_t sgDataSize = sgData.size();
753 size_t instDataSize = instData.size();
754 size_t laneDataSize = laneData.size();
755 int64_t sgDataValue = -1;
756 int64_t instDataValue = -1;
757 int64_t laneDataValue = -1;
758 int64_t dim = resLayout.getRank() - 1;
759
760 // Deinterleave halves the innermost dimension, so we need to double the
761 // layout values (similar to bitcast with ratio = 2)
762 constexpr int ratio = 2;
763 if (sgDataSize)
764 sgDataValue = sgData.back() * ratio;
765 if (instDataSize)
766 instDataValue = instData.back() * ratio;
767 if (laneDataSize)
768 laneDataValue = laneData.back() * ratio;
769
770 return resLayout.setDimData(dim, sgData: sgDataValue, instData: instDataValue, laneData: laneDataValue);
771}
772
773/// Infers the source layout attribute for an insert strided slice operation
774/// given the result layout attribute, result shape, and source shape. Removes
775/// leading dimensions from the result layout to match the source shape size.
776xegpu::DistributeLayoutAttr xegpu::inferInsertStridedSliceSourceLayout(
777 xegpu::DistributeLayoutAttr resLayout, ArrayRef<int64_t> resShape,
778 ArrayRef<int64_t> srcShape) {
779
780 int srcShapeSize = srcShape.size();
781 int resShapeSize = resShape.size();
782 int dimDiff = resShapeSize - srcShapeSize;
783
784 if (dimDiff > 0) {
785 // assert that the leading dimensions being sliced off are not distributed
786 // (i.e. sg_layout and lane_layout for those dimensions are all 1)
787 auto resSgLayout = resLayout.getEffectiveSgLayoutAsInt();
788 auto resLaneLayout = resLayout.getEffectiveLaneLayoutAsInt();
789 for (int i = 0; i < dimDiff; i++) {
790 assert((resSgLayout.size() == 0 || resSgLayout[i] == 1) &&
791 (resLaneLayout.size() == 0 || resLaneLayout[i] == 1) &&
792 "Leading dimensions being sliced off must not be distributed");
793 }
794 return resLayout.dropDims(dimGroup: llvm::to_vector(Range: llvm::seq<int64_t>(Begin: 0, End: dimDiff)));
795 }
796 return resLayout;
797}
798
799/// Infers the source layout attribute for an insert operation
800/// given the result layout attribute, result shape, and source shape. Removes
801/// leading dimensions from the result layout to match the source shape size.
802// TODO: add propagation support for insert op
803xegpu::DistributeLayoutAttr
804xegpu::inferInsertSourceLayout(xegpu::DistributeLayoutAttr resLayout,
805 ArrayRef<int64_t> resShape,
806 ArrayRef<int64_t> srcShape) {
807
808 int srcShapeSize = srcShape.size();
809 int resShapeSize = resShape.size();
810 int dimDiff = resShapeSize - srcShapeSize;
811
812 if (dimDiff > 0) {
813 // assert that the leading dimensions being sliced off are not distributed
814 // (i.e. sg_layout and lane_layout for those dimensions are all 1)
815 auto resSgLayout = resLayout.getEffectiveSgLayoutAsInt();
816 auto resLaneLayout = resLayout.getEffectiveLaneLayoutAsInt();
817 for (int i = 0; i < dimDiff; i++) {
818 assert((resSgLayout.size() == 0 || resSgLayout[i] == 1) &&
819 (resLaneLayout.size() == 0 || resLaneLayout[i] == 1) &&
820 "Leading dimensions being sliced off must not be distributed");
821 }
822 return resLayout.dropDims(dimGroup: llvm::to_vector(Range: llvm::seq<int64_t>(Begin: 0, End: dimDiff)));
823 }
824 return resLayout;
825}
826
827/// Infers the source layout attribute for extract operation
828/// given the result layout attribute, result shape, and source shape. Adds
829/// leading dimensions to the source layout to match the source shape size.
830// TODO: add layout attribute interface: expandDim() and use it here.
831// TODO: add propagation support for extract op
832xegpu::DistributeLayoutAttr
833xegpu::inferExtractSourceLayout(xegpu::DistributeLayoutAttr resLayout,
834 ArrayRef<int64_t> resShape,
835 ArrayRef<int64_t> srcShape) {
836
837 int srcShapeSize = srcShape.size();
838 int resShapeSize = resShape.size();
839 int dimDiff = srcShapeSize - resShapeSize;
840 auto context = resLayout.getContext();
841 // construct the source layout by adding unit dimensions to the front of
842 // result layout
843 if (dimDiff > 0) {
844 auto sgLayout = resLayout.getEffectiveSgLayoutAsInt();
845 auto sgData = resLayout.getEffectiveSgDataAsInt();
846 auto instData = resLayout.getEffectiveInstDataAsInt();
847 auto laneLayout = resLayout.getEffectiveLaneLayoutAsInt();
848 auto laneData = resLayout.getEffectiveLaneDataAsInt();
849 auto order = resLayout.getEffectiveOrderAsInt();
850
851 // Example: result shape is 3D with order [1, 2, 0], source shape is 5D
852 // (adding 2 leading dimensions). Expected source order: [3, 4, 2, 1, 0]
853 // Step 1: shift existing order by dimDiff: [1, 2, 0] -> [3, 4, 2]
854 // Step 2: append new leading dims in reverse (slowest first): [3, 4, 2, 1,
855 // 0]
856
857 // Shift existing dimension indices in order by dimDiff to account for the
858 // new leading dimensions being added to the source shape
859 for (auto &o : order)
860 o += dimDiff;
861
862 // Add unit dimensions to the front of non-empty layout vectors and append
863 // the new dimension indices to the order array in reverse (slowest
864 // dimension has the lowest index and appears last in the order array)
865 for (int i = 0; i < dimDiff; i++) {
866 if (!sgLayout.empty())
867 sgLayout.insert(I: sgLayout.begin(), Elt: 1);
868 if (!sgData.empty())
869 sgData.insert(I: sgData.begin(), Elt: 1);
870 if (!instData.empty())
871 instData.insert(I: instData.begin(), Elt: 1);
872 if (!laneLayout.empty())
873 laneLayout.insert(I: laneLayout.begin(), Elt: 1);
874 if (!laneData.empty())
875 laneData.insert(I: laneData.begin(), Elt: 1);
876 order.push_back(Elt: dimDiff - 1 - i);
877 }
878
879 DenseI32ArrayAttr orderAttr = DenseI32ArrayAttr::get(
880 context, content: SmallVector<int32_t>(order.begin(), order.end()));
881 if (!resLayout.getOrder())
882 orderAttr = nullptr;
883
884 return buildLayout(context, sgLayout, sgData, instData, laneLayout,
885 laneData, orderAttr);
886 }
887 return resLayout;
888}
889
890/// Infers the source layout attribute for a shape cast operation given the
891/// result layout attribute, result shape, and source shape.
892xegpu::DistributeLayoutAttr
893xegpu::inferShapeCastSourceLayout(xegpu::DistributeLayoutAttr resLayout,
894 ArrayRef<int64_t> resShape,
895 ArrayRef<int64_t> srcShape) {
896
897 // There are three use cases:
898 // 1. expand dims of low-rank dimensions (e.g., 1D to 2D): to set up the
899 // tensor before broadcast
900 // 2. split dim of a high-rank dimension (e.g., 1D to 2D): to setup tensor
901 // for multi-stage reduction
902 // 3. combines all dims to a single dim and put in the innermost dim in 2d as
903 // [1, combinedData] or [combinedData]. Say, [2, 4, 8] -> [1, 64] or [64]
904 // Use cases are only supported after workgroup distribution,
905 // like cross-sg reduction saves multidimension data to
906 // 1D slm buffer, shapecast inserted by cse/canonicalization passes.
907
908 // Use case 1: Shapes only differ by expanding unit dimensions, for broadcast
909 SmallVector<int64_t> expandedUnitDims;
910
911 if (xegpu::matchUnitDimExpansion(src: srcShape, dst: resShape, expandedUnitDims)) {
912 // create a slice layout for the source by removing the expanded unit dims
913 auto sliceDimsAttr = DenseI64ArrayAttr::get(
914 context: resLayout.getContext(), content: ArrayRef<int64_t>(expandedUnitDims));
915 auto srcLayout =
916 xegpu::SliceAttr::get(context: resLayout.getContext(), parent: resLayout, dims: sliceDimsAttr);
917 return srcLayout;
918 }
919
920 // Use case 2: Dim split from source to result, for multi-stage reduction
921 SmallVector<SmallVector<int64_t>> splitDimGroups;
922 if (xegpu::matchSplitDimExpansion(src: srcShape, dst: resShape, splitDimGroups)) {
923 auto srcLayout = resLayout;
924 for (const auto &dimGroup : splitDimGroups)
925 srcLayout = srcLayout.collapseDims(dimGroup);
926
927 return srcLayout;
928 }
929
930 // Use case 3: General dim collapse, for cross-sg reduction to SLM and other
931 // shape casts where consecutive src dims fold into a single dst dim.
932 SmallVector<SmallVector<int64_t>> collapseDims;
933 if (xegpu::matchDimCollapse(src: srcShape, dst: resShape, collapseDims)) {
934 auto srcLayout = resLayout;
935 for (int64_t dstIdx = static_cast<int64_t>(collapseDims.size()) - 1;
936 dstIdx >= 0; --dstIdx) {
937 ArrayRef<int64_t> srcDims = collapseDims[dstIdx];
938 if (srcDims.empty()) {
939 srcLayout = srcLayout.dropDims(dimGroup: {dstIdx});
940 continue;
941 }
942 if (srcDims.size() == 1)
943 continue;
944 SmallVector<int64_t> targetShape;
945 targetShape.reserve(N: srcDims.size());
946 for (int64_t d : srcDims)
947 targetShape.push_back(Elt: srcShape[d]);
948 srcLayout = srcLayout.expandDim(dim: dstIdx, targetShape);
949 }
950 return srcLayout;
951 }
952 return nullptr;
953}
954
955//===----------------------------------------------------------------------===//
956// Forward layout inference (source layout -> result layout)
957//===----------------------------------------------------------------------===//
958
959/// Infers the result layout attribute for a transpose operation given the
960/// source layout attribute and permutation.
961///
962/// vector.transpose semantics is `result[i] = source[permutation[i]]`, so
963/// `result_layout[i] = source_layout[permutation[i]]`, which is exactly
964/// `srcLayout.transposeDims(permutation)`. This is the inverse of
965/// inferTransposeSourceLayout (which applies the inverse permutation).
966xegpu::DistributeLayoutAttr
967xegpu::inferTransposeResultLayout(xegpu::DistributeLayoutAttr srcLayout,
968 ArrayRef<int64_t> permutation) {
969 return srcLayout.transposeDims(permutation);
970}
971
972/// Infers the result layout attribute for a shape cast operation given the
973/// source layout attribute, source shape, and result shape. This is the
974/// inverse of inferShapeCastSourceLayout: a dim-split (src -> res) is undone by
975/// collapsing the split groups, and a dim-collapse (src -> res) is undone by
976/// expanding the collapsed groups. The unit-dim-expansion case is not inverted
977/// here because recovering which result dims are the expanded unit dims would
978/// require the SliceAttr the backward direction produces; such patterns return
979/// nullptr (leaving the result un-laid-out).
980xegpu::DistributeLayoutAttr
981xegpu::inferShapeCastResultLayout(xegpu::DistributeLayoutAttr srcLayout,
982 ArrayRef<int64_t> srcShape,
983 ArrayRef<int64_t> resShape) {
984 // Case: source dims were split into result dims (forward of use case 2 in
985 // inferShapeCastSourceLayout). Undo by expanding each source dim into its
986 // group of result dims.
987 SmallVector<SmallVector<int64_t>> splitDimGroups;
988 if (xegpu::matchSplitDimExpansion(src: srcShape, dst: resShape, splitDimGroups)) {
989 auto resLayout = srcLayout;
990 // Process source dims from innermost to outermost so that expanding a dim
991 // does not shift the indices of dims not yet processed.
992 for (int64_t srcIdx = static_cast<int64_t>(splitDimGroups.size()) - 1;
993 srcIdx >= 0; --srcIdx) {
994 ArrayRef<int64_t> resDims = splitDimGroups[srcIdx];
995 if (resDims.size() <= 1)
996 continue;
997 SmallVector<int64_t> targetShape;
998 targetShape.reserve(N: resDims.size());
999 for (int64_t d : resDims)
1000 targetShape.push_back(Elt: resShape[d]);
1001 resLayout = resLayout.expandDim(dim: srcIdx, targetShape);
1002 }
1003 return resLayout;
1004 }
1005
1006 // Case: source dims were collapsed into result dims (forward of use case 3).
1007 // Undo by collapsing each group of source dims into its single result dim.
1008 SmallVector<SmallVector<int64_t>> collapseDims;
1009 if (xegpu::matchDimCollapse(src: srcShape, dst: resShape, collapseDims)) {
1010 auto resLayout = srcLayout;
1011 // Process result dims from innermost to outermost so that collapsing a
1012 // group does not shift the indices of groups not yet processed.
1013 for (int64_t dstIdx = static_cast<int64_t>(collapseDims.size()) - 1;
1014 dstIdx >= 0; --dstIdx) {
1015 ArrayRef<int64_t> srcDims = collapseDims[dstIdx];
1016 // A result dim with no backing source dims is a trailing/leading unit
1017 // dim; its forward inference is ambiguous, so bail out.
1018 if (srcDims.empty())
1019 return nullptr;
1020 if (srcDims.size() == 1)
1021 continue;
1022 resLayout = resLayout.collapseDims(dimGroup: llvm::to_vector(Range&: srcDims));
1023 }
1024 return resLayout;
1025 }
1026
1027 return nullptr;
1028}
1029
1030/// Infers the result layout attribute for a non-anchor operation from the
1031/// layouts of its source operands. Forward counterpart of
1032/// inferSourceLayoutFromResultForNonAnchorOp.
1033xegpu::DistributeLayoutAttr xegpu::inferResultLayoutFromSourceForNonAnchorOp(
1034 Operation *op, ArrayRef<xegpu::DistributeLayoutAttr> operandLayouts) {
1035 if (op->getNumResults() != 1)
1036 return nullptr;
1037
1038 // For vector::TransposeOp, infer the result layout from the source layout.
1039 if (auto transpose = dyn_cast<vector::TransposeOp>(Val: op)) {
1040 if (!operandLayouts[0])
1041 return nullptr;
1042 return xegpu::inferTransposeResultLayout(srcLayout: operandLayouts[0],
1043 permutation: transpose.getPermutation());
1044 }
1045
1046 // For vector::ShapeCastOp, infer the result layout from the source layout.
1047 if (auto shapeCast = dyn_cast<vector::ShapeCastOp>(Val: op)) {
1048 if (!operandLayouts[0])
1049 return nullptr;
1050 return xegpu::inferShapeCastResultLayout(
1051 srcLayout: operandLayouts[0], srcShape: shapeCast.getSourceVectorType().getShape(),
1052 resShape: shapeCast.getResultVectorType().getShape());
1053 }
1054
1055 // For elementwise operations, all operands and the result share the same
1056 // layout. Use the first operand that carries a layout.
1057 if (OpTrait::hasElementwiseMappableTraits(op)) {
1058 for (xegpu::DistributeLayoutAttr layout : operandLayouts)
1059 if (layout)
1060 return layout;
1061 return nullptr;
1062 }
1063
1064 // TODO: add forward inference rules for the remaining ops; their result is
1065 // left un-laid-out until then.
1066 // - vector::BroadcastOp: the forward direction is under-determined. The
1067 // backward rule (inferBroadcastSourceLayout) either sets broadcast dims to
1068 // unit data (losing the original data on those dims) or wraps the result
1069 // in a SliceAttr; neither is generally invertible from the source layout
1070 // alone, so a forward rule must decide how to distribute the new/stretched
1071 // dims.
1072 // - vector::BitCastOp, vector::MultiDimReductionOp / vector::ReductionOp,
1073 // vector::InterleaveOp / vector::DeinterleaveOp, and the insert / extract
1074 // / strided-slice family.
1075 return nullptr;
1076}
1077
1078//===----------------------------------------------------------------------===//
1079// Layout derivation helpers: factorize sgCount into
1080// sg_layout candidates, then
1081// compute per-subgroup (sgData) and per-lane
1082// (lane_layout/lane_data/inst_data).
1083//===----------------------------------------------------------------------===//
1084
1085using LayoutRepresentation = SmallVector<int64_t>;
1086
1087/// Enumerates all ways to split `total` into `rank` factors whose product
1088/// equals `total`. Returns the list of all such factorizations.
1089static SmallVector<LayoutRepresentation> enumerateFactorizations(int64_t total,
1090 int64_t rank) {
1091 SmallVector<LayoutRepresentation> results;
1092 SmallVector<int64_t> current(rank, 0);
1093
1094 // Returns all divisors of `n` in ascending order.
1095 auto getDivisors = [](int64_t n) {
1096 SmallVector<int64_t> divs;
1097 for (int64_t i = 1; i * i <= n; ++i) {
1098 if (n % i == 0) {
1099 divs.push_back(Elt: i);
1100 if (i != n / i)
1101 divs.push_back(Elt: n / i);
1102 }
1103 }
1104 llvm::sort(C&: divs);
1105 return divs;
1106 };
1107
1108 std::function<void(int64_t, int64_t)> generate = [&](int64_t dim,
1109 int64_t remaining) {
1110 if (dim == rank - 1) {
1111 current[dim] = remaining;
1112 results.push_back(Elt: LayoutRepresentation(current));
1113 return;
1114 }
1115 for (int64_t factor : getDivisors(remaining)) {
1116 current[dim] = factor;
1117 generate(dim + 1, remaining / factor);
1118 }
1119 };
1120
1121 generate(0, total);
1122 return results;
1123}
1124
1125// Computes all valid N-dimensional sg_layout candidates for the given
1126// sgCount, whose sgData (= wgShape / sgLayout):
1127// 1. Evenly divides wgShape (i.e., wgShape[d] % sgLayout[d] == 0).
1128// 2. Is a multiple of instData (i.e., sgData[d] % instData[d] == 0).
1129// Results are sorted by balance (smallest max-min spread first), with
1130// lexicographic order as a tiebreaker.
1131//
1132// `broadcastDim` (default -1 = none) marks a dimension broadcast across
1133// subgroups rather than distributed (e.g. the K/contraction dim of a DPAS
1134// operand). Its full extent stays in every subgroup, so rule 1 is skipped for
1135// it, but rule 2 (multiple of instData) still applies.
1136//
1137// Example (2D):
1138// wgShape = [128, 64], instData = [8, 16], sgCount = 32
1139// Returns: [[8,4], [16,2]], corresponding to sgData [16,16] and [8,32].
1140static SmallVector<LayoutRepresentation>
1141getSgLayoutCandidates(ArrayRef<int64_t> wgShape, ArrayRef<int64_t> instData,
1142 int64_t sgCount, int64_t broadcastDim = -1) {
1143 int64_t rank = wgShape.size();
1144 assert(rank > 0 && "wgShape must be non-empty");
1145 assert(static_cast<int64_t>(instData.size()) == rank &&
1146 "instData rank must match wgShape rank");
1147
1148 // Step 1: Get all N-D factorizations of sgCount.
1149 auto allFactorizations = enumerateFactorizations(total: sgCount, rank);
1150
1151 // Step 2: Filter to keep only valid candidates.
1152 SmallVector<LayoutRepresentation> candidates;
1153 for (const auto &sgLayout : allFactorizations) {
1154 bool valid = true;
1155 for (int64_t dim = 0; dim < rank; ++dim) {
1156 // A broadcast dim keeps its full extent in every subgroup; others are
1157 // split evenly by sgLayout[dim].
1158 int64_t sgData;
1159 if (dim == broadcastDim) {
1160 sgData = wgShape[dim];
1161 } else {
1162 if (wgShape[dim] % sgLayout[dim] != 0) {
1163 valid = false;
1164 break;
1165 }
1166 sgData = wgShape[dim] / sgLayout[dim];
1167 }
1168 if (sgData % instData[dim] != 0) {
1169 valid = false;
1170 break;
1171 }
1172 }
1173 if (valid)
1174 candidates.push_back(Elt: sgLayout);
1175 }
1176
1177 // Step 3: Sort by balance (smallest max-min spread), then lexicographic.
1178 llvm::sort(C&: candidates, Comp: [](const LayoutRepresentation &lhs,
1179 const LayoutRepresentation &rhs) {
1180 int64_t spreadLhs = *llvm::max_element(Range: lhs) - *llvm::min_element(Range: lhs);
1181 int64_t spreadRhs = *llvm::max_element(Range: rhs) - *llvm::min_element(Range: rhs);
1182 if (spreadLhs != spreadRhs)
1183 return spreadLhs < spreadRhs;
1184 return lhs < rhs;
1185 });
1186 return candidates;
1187}
1188
1189/// Helper function to compute inst_data vectors for DPAS operands A, B, and
1190/// C/D.
1191static std::optional<SmallVector<int64_t>> get2DBlockIOInstDataLayout(
1192 ArrayRef<int64_t> dataShape, Type elemTy,
1193 const xegpu::uArch::BlockIOInstructionInterface *uArchInstruction,
1194 bool transform = false, bool transpose = false) {
1195 int rank = dataShape.size();
1196 auto blockWHC =
1197 uArchInstruction->getBlockWidthHeightCount(elemTy, hasTransform: transform, hasTranspose: transpose);
1198 if (!blockWHC)
1199 return std::nullopt;
1200 auto [bWidths, bHeights, bCounts] = blockWHC.value();
1201 // Compute inst_data from hardware block params. For Nd ops, the lane
1202 // factorization above (laneLayout / laneData) is rigid; inst_data must be
1203 // a multiple of lane_layout * lane_data on each dim (Category A
1204 // invariant).
1205 SmallVector<int64_t> instData(rank, 1);
1206 assert(rank >= 2 && "dataShape must be at least 2D for 2D-block IO");
1207 int instWidth =
1208 xegpu::getLargestDivisor(dim: static_cast<int>(dataShape.back()), candidates: bWidths);
1209 int instHeight =
1210 xegpu::getLargestDivisor(dim: static_cast<int>(dataShape[rank - 2]), candidates: bHeights);
1211 // No supported hardware block size divides the data dim (e.g. innermost dim
1212 // of 1 vs. minimum block width 16): not realizable as a 2D-block instruction.
1213 if (instWidth < 0 || instHeight < 0)
1214 return std::nullopt;
1215 instData.back() = instWidth;
1216 instData[rank - 2] = instHeight;
1217
1218 return instData;
1219}
1220
1221/// Picks the subgroup layout for a 2D block IO op (store_nd / prefetch_nd).
1222/// Every supported block (height, width) dividing the shape is tried, largest
1223/// area first and the more square block first among equal areas, until one
1224/// admits a subgroup layout.
1225static xegpu::DistributeLayoutAttr get2DBlockIOSubgroupLayout(
1226 MLIRContext *context, ArrayRef<int64_t> dataShape, Type elemTy,
1227 const xegpu::uArch::BlockIOInstructionInterface *uArchInstruction,
1228 int numSg) {
1229 auto blockWHC = uArchInstruction->getBlockWidthHeightCount(elemTy);
1230 if (!blockWHC)
1231 return nullptr;
1232 auto [bWidths, bHeights, bCounts] = blockWHC.value();
1233 int rank = dataShape.size();
1234 assert(rank >= 2 && "dataShape must be at least 2D for 2D-block IO");
1235
1236 SmallVector<SmallVector<int64_t>> instDataCandidates;
1237 for (int width : bWidths) {
1238 if (width <= 0 || dataShape[rank - 1] % width != 0)
1239 continue;
1240 for (int height : bHeights) {
1241 if (height <= 0 || dataShape[rank - 2] % height != 0)
1242 continue;
1243 SmallVector<int64_t> instData(rank, 1);
1244 instData[rank - 1] = width;
1245 instData[rank - 2] = height;
1246 instDataCandidates.push_back(Elt: std::move(instData));
1247 }
1248 }
1249 // Largest area first; among equal areas, the more square block first (e.g.
1250 // 16x16 before 8x32 before 4x64), then the wider one.
1251 llvm::stable_sort(Range&: instDataCandidates, C: [&](ArrayRef<int64_t> lhs,
1252 ArrayRef<int64_t> rhs) {
1253 int64_t hL = lhs[rank - 2], wL = lhs[rank - 1];
1254 int64_t hR = rhs[rank - 2], wR = rhs[rank - 1];
1255 if (hL * wL != hR * wR)
1256 return hL * wL > hR * wR;
1257 if (std::min(a: hL, b: wL) != std::min(a: hR, b: wR))
1258 return std::min(a: hL, b: wL) > std::min(a: hR, b: wR);
1259 return wL > wR;
1260 });
1261
1262 for (ArrayRef<int64_t> instData : instDataCandidates) {
1263 auto sgLayouts = getSgLayoutCandidates(wgShape: dataShape, instData, sgCount: numSg);
1264 if (!sgLayouts.empty())
1265 return buildSgLayout(context, wgTileShape: dataShape, sgLayout: sgLayouts.front(), /*dimK=*/-1);
1266 }
1267 return nullptr;
1268}
1269
1270/// Helper function to compute inst_data vectors for DPAS operands A, B, and
1271/// C/D. Look up the uArch table and search for the largest supported block size
1272/// that divides the data shape
1273static std::optional<std::tuple<SmallVector<int64_t>, SmallVector<int64_t>,
1274 SmallVector<int64_t>>>
1275getDpasInstDataLayouts(
1276 VectorType aTy, VectorType bTy, VectorType cdTy,
1277 const xegpu::uArch::MMAInstructionInterface *uArchInstruction) {
1278
1279 // M dimension is the second-to-last dim of A (handles batch dims).
1280 const unsigned dataALen = aTy.getShape()[aTy.getRank() - 2];
1281 auto supportedALen = uArchInstruction->getSupportedM(type: aTy.getElementType());
1282 const int maxALen =
1283 xegpu::getLargestDivisor(dim: dataALen, candidates: ArrayRef<unsigned>(supportedALen));
1284
1285 // N dimension is the last dim of B.
1286 const unsigned dataBLen = bTy.getShape().back();
1287 auto supportedBLen = uArchInstruction->getSupportedN(type: bTy.getElementType());
1288 const int maxBLen =
1289 xegpu::getLargestDivisor(dim: dataBLen, candidates: ArrayRef<unsigned>(supportedBLen));
1290
1291 auto supportedCLen = uArchInstruction->getSupportedN(type: cdTy.getElementType());
1292 const int maxCLen =
1293 xegpu::getLargestDivisor(dim: dataBLen, candidates: ArrayRef<unsigned>(supportedCLen));
1294 if (maxALen == -1 || maxBLen == -1 || maxCLen == -1)
1295 return std::nullopt;
1296
1297 auto supportedKLen = uArchInstruction->getSupportedK(type: aTy.getElementType());
1298 if (supportedKLen.empty())
1299 return std::nullopt;
1300 auto kDimSize = supportedKLen[0];
1301
1302 SmallVector<int64_t> instDataA(aTy.getRank(), 1);
1303 instDataA[aTy.getRank() - 2] = maxALen;
1304 instDataA[aTy.getRank() - 1] = kDimSize;
1305 SmallVector<int64_t> instDataB(bTy.getRank(), 1);
1306 instDataB[bTy.getRank() - 2] = kDimSize;
1307 instDataB[bTy.getRank() - 1] = maxBLen;
1308 SmallVector<int64_t> instDataCD(cdTy.getRank(), 1);
1309 instDataCD[cdTy.getRank() - 2] = maxALen;
1310 instDataCD[cdTy.getRank() - 1] = maxCLen;
1311 return std::make_tuple(args&: instDataA, args&: instDataB, args&: instDataCD);
1312}
1313
1314/// Computes lane_layout and lane_data for scatter-style store anchor layouts
1315/// (store scatter, store matrix). Lanes and the per-lane vector both live on
1316/// the innermost dim:
1317/// - laneLayout[innermost] = min(subgroupSize, srcShape[innermost])
1318/// - laneData[innermost] = min(srcShape[innermost] / laneLayout[innermost],
1319/// maxChunkSize)
1320/// All other entries are 1.
1321static std::pair<SmallVector<int64_t>, SmallVector<int64_t>>
1322computeScatterIOLaneLayoutAndData(ArrayRef<int64_t> instShape,
1323 int64_t subgroupSize, int64_t maxChunkSize) {
1324 int64_t rank = instShape.size();
1325 SmallVector<int64_t> laneLayout(rank, 1), laneData(rank, 1);
1326 int64_t innermost = rank - 1;
1327 laneLayout[innermost] = std::min(a: subgroupSize, b: instShape[innermost]);
1328 laneData[innermost] =
1329 std::min(a: instShape[innermost] / laneLayout[innermost], b: maxChunkSize);
1330 return {laneLayout, laneData};
1331}
1332
1333// Computes the (lane_layout, lane_data, order) a 2D block instruction can
1334// deliver over `instShape`, from the element `bitwidth` and the variant's
1335// `packingSize`. lane_data packs elements narrower than the packing size along
1336// the packing dim: the innermost one, or the one above it when transformed
1337// (VNNI). Lanes are spread along the innermost dim, or the one above it when
1338// transposed, which makes that dim fastest-changing and so returns a reversed
1339// `order`. `order` is empty for the variants that keep the default. Leading
1340// dims are unit.
1341static std::tuple<SmallVector<int64_t>, SmallVector<int64_t>,
1342 SmallVector<int64_t>>
1343compute2DBlockIOLaneLayout(ArrayRef<int64_t> instShape, int64_t subgroupSize,
1344 int64_t bitwidth, int64_t packingSize,
1345 bool transform = false, bool transpose = false) {
1346 int64_t rank = instShape.size();
1347 assert(rank >= 2 && "Expected at least a 2D shape for a 2D block op");
1348 SmallVector<int64_t> laneLayout(rank, 1), laneData(rank, 1), order;
1349
1350 int64_t packDim = transform ? rank - 2 : rank - 1;
1351 laneData[packDim] = llvm::divideCeil(Numerator: packingSize, Denominator: bitwidth);
1352
1353 int64_t laneDim = transpose ? rank - 2 : rank - 1;
1354 laneLayout[laneDim] =
1355 std::min(a: subgroupSize, b: instShape[laneDim] / laneData[laneDim]);
1356
1357 if (transpose) {
1358 for (int64_t i = 0; i < rank; ++i)
1359 order.push_back(Elt: rank - 1 - i);
1360 std::swap(a&: order[0], b&: order[1]);
1361 }
1362
1363 // assert that the lane layout and data fit in the inst shape
1364 for (int64_t i = 0; i < rank; ++i) {
1365 int64_t laneProduct = laneLayout[i] * laneData[i];
1366 assert(instShape[i] % laneProduct == 0 &&
1367 "lane_layout * lane_data must evenly divide the inst shape");
1368 (void)laneProduct;
1369 }
1370 return {laneLayout, laneData, order};
1371}
1372
1373/// Derives (lane_layout, lane_data, inst_data) for a vector.multi_reduction
1374/// source from the layout requested by its result's consumer. Each returned
1375/// vector has one entry per source dimension.
1376///
1377/// Reuse the consumer's slice parent when the slice dimensions exactly match
1378/// `reductionDims`, the parent's effective lane_layout and lane_data each have
1379/// one entry per source dimension, and the lane layout uses all `subgroupSize`
1380/// lanes. Clamp reused lane counts to srcShape[i] / lane_data[i], since the
1381/// sliced dimensions may have different extents in the reduction source.
1382///
1383/// If the consumer's slice parent cannot be reused, initialize lane_layout and
1384/// lane_data to 1, then copy available consumer entries onto non-reduced source
1385/// dimensions in order, with the same clamp: a consumer that broadcasts along
1386/// a unit dimension may place lanes on it.
1387///
1388/// For a derived (not reused) layout, assign any unused lanes to the innermost
1389/// dimension if it is reduced, up to the same bound. If that dimension still
1390/// has only one lane, then the lane can reduce a vector limited by
1391/// maxReduceVectorSize and the source dimension's size. The vector budget
1392/// (maxReduceVectorSize) applies to the product of lane_data across all
1393/// dimensions. If that product already meets or exceeds the budget, leave
1394/// lane_data unchanged.
1395///
1396/// Set inst_data[i] to lane_layout[i] * lane_data[i]. On non-reduced
1397/// dimensions, preserve a reused parent's larger inst_data[i].
1398///
1399/// Source layout examples with subgroupSize = maxReduceVectorSize = 16:
1400/// \code
1401/// // Reuse a matching slice parent's lane layout and data.
1402/// srcShape=[32, 64], reductionDims=[1], consumer=slice(parent, dims=[1])
1403/// parent: lane_layout=[1, 16], lane_data=[1, 1]
1404/// source: lane_layout=[1, 16], lane_data=[1, 1], inst_data=[1, 16]
1405///
1406/// // Preserve dim 0 blocking and limit dim 1 lanes to the source size.
1407/// srcShape=[16, 4], reductionDims=[1], consumer=slice(parent, dims=[1])
1408/// parent: lane_layout=[1, 16], lane_data=[1, 1], inst_data=[8, 16]
1409/// source: lane_layout=[1, 4], lane_data=[1, 1], inst_data=[8, 4]
1410///
1411/// // Assign spare lanes to the innermost dimension when it is reduced.
1412/// srcShape=[16, 32, 32], reductionDims=[2]
1413/// consumer: lane_layout=[1, 2], lane_data=[1, 1]
1414/// source: lane_layout=[1, 2, 8], lane_data=[1, 1, 1], inst_data=[1, 2, 8]
1415///
1416/// // Keep the consumer's lanes on the non-reduced dimension.
1417/// srcShape=[32, 64], reductionDims=[0]
1418/// consumer: lane_layout=[16], lane_data=[1]
1419/// source: lane_layout=[1, 16], lane_data=[1, 1], inst_data=[1, 16]
1420///
1421/// // For a scalar reduction, put lanes on the innermost dimension.
1422/// srcShape=[32, 64], reductionDims=[0, 1]
1423/// consumer: none
1424/// source: lane_layout=[1, 16], lane_data=[1, 1], inst_data=[1, 16]
1425/// \endcode
1426static std::tuple<SmallVector<int64_t>, SmallVector<int64_t>,
1427 SmallVector<int64_t>>
1428computeReductionLaneLayoutAndData(
1429 ArrayRef<int64_t> srcShape, ArrayRef<int64_t> reductionDims,
1430 int subgroupSize, int64_t maxReduceVectorSize,
1431 xegpu::DistributeLayoutAttr consumerLayout = nullptr) {
1432 int srcRank = srcShape.size();
1433 SmallVector<int64_t> laneLayout(srcRank, 1), laneData(srcRank, 1);
1434 auto isReduced = [&](int dim) {
1435 return llvm::is_contained(Range&: reductionDims, Element: dim);
1436 };
1437
1438 // A matching slice parent provides lane information for reduced dimensions
1439 // as well as the dimensions present in the result.
1440 SmallVector<int64_t> parentInstData;
1441 auto consumerSlice = dyn_cast_if_present<xegpu::SliceAttr>(Val&: consumerLayout);
1442 bool reusedParent = false;
1443 if (consumerSlice &&
1444 consumerSlice.getDims().asArrayRef().equals(RHS: reductionDims)) {
1445 auto parent = consumerSlice.getParent();
1446 SmallVector<int64_t> parentLaneLayout =
1447 parent.getEffectiveLaneLayoutAsInt();
1448 SmallVector<int64_t> parentLaneData = parent.getEffectiveLaneDataAsInt();
1449 if (static_cast<int>(parentLaneLayout.size()) == srcRank &&
1450 static_cast<int>(parentLaneData.size()) == srcRank &&
1451 computeProduct(basis: parentLaneLayout) == subgroupSize) {
1452 laneLayout = std::move(parentLaneLayout);
1453 laneData = std::move(parentLaneData);
1454 reusedParent = true;
1455 parentInstData = parent.getEffectiveInstDataAsInt();
1456 if (static_cast<int>(parentInstData.size()) != srcRank)
1457 parentInstData.clear();
1458
1459 // The parent may come from a broadcast to a larger shape. Bound its
1460 // lane counts by the source extent, accounting for per-lane packing.
1461 for (int i = 0; i < srcRank; ++i)
1462 laneLayout[i] = std::min(
1463 a: laneLayout[i], b: std::max<int64_t>(a: 1, b: srcShape[i] / laneData[i]));
1464 }
1465 }
1466
1467 // Without a reusable parent, map result dimensions to non-reduced source
1468 // dimensions.
1469 if (!reusedParent) {
1470 SmallVector<int64_t> consumerLaneLayout, consumerLaneData;
1471 if (consumerLayout) {
1472 consumerLaneLayout = consumerLayout.getEffectiveLaneLayoutAsInt();
1473 consumerLaneData = consumerLayout.getEffectiveLaneDataAsInt();
1474 }
1475 for (int i = 0, c = 0; i < srcRank; ++i) {
1476 if (isReduced(i))
1477 continue;
1478 if (c < static_cast<int>(consumerLaneLayout.size()))
1479 laneLayout[i] = consumerLaneLayout[c];
1480 if (c < static_cast<int>(consumerLaneData.size()))
1481 laneData[i] = consumerLaneData[c];
1482 ++c;
1483 }
1484
1485 // Cap laneLayout using srcShape.
1486 for (int i = 0; i < srcRank; ++i)
1487 laneLayout[i] = std::min(a: laneLayout[i],
1488 b: std::max<int64_t>(a: 1, b: srcShape[i] / laneData[i]));
1489
1490 // Assign spare lanes only when the innermost source dimension is reduced,
1491 // matching the distribution typically used by memory loads and DPAS
1492 // results.
1493 int innermost = srcRank - 1;
1494 if (isReduced(innermost)) {
1495 int64_t spareLanes =
1496 std::max<int64_t>(a: 1, b: subgroupSize / computeProduct(basis: laneLayout));
1497 if (spareLanes > 1)
1498 laneLayout[innermost] =
1499 std::min(a: spareLanes, b: std::max<int64_t>(a: 1, b: srcShape[innermost]));
1500
1501 if (laneLayout[innermost] == 1) {
1502 // Adjust lane_data on this dimension using the remaining vector budget,
1503 // capped by the source extent.
1504 int64_t vectorBudget = maxReduceVectorSize / computeProduct(basis: laneData);
1505 if (vectorBudget > 1)
1506 laneData[innermost] =
1507 std::min(a: vectorBudget * laneData[innermost], b: srcShape[innermost]);
1508 }
1509 }
1510 }
1511
1512 // Preserve compatible consumer blocking on non-reduced dimensions so the
1513 // reduction result can use the same instruction tiles as its consumer.
1514 SmallVector<int64_t> instData(srcRank);
1515 for (int i = 0; i < srcRank; ++i) {
1516 instData[i] = laneLayout[i] * laneData[i];
1517 if (isReduced(i) || parentInstData.empty())
1518 continue;
1519 // The lane product must divide the instruction tile, which must divide
1520 // the source extent.
1521 int64_t wider = parentInstData[i];
1522 if (wider > instData[i] && wider % instData[i] == 0 &&
1523 srcShape[i] % wider == 0)
1524 instData[i] = wider;
1525 }
1526
1527 return {laneLayout, laneData, instData};
1528}
1529
1530//===----------------------------------------------------------------------===//
1531// Result/anchor-layout setup. Each op category derives lane_layout/lane_data
1532// (and inst_data / sgData) differently. Two things vary across ops:
1533//
1534// * Consumer dependence: consumer-driven ops prefer the layout requested by
1535// their downstream uses and fall back to uArch defaults only when it is
1536// absent/invalid; sinks (StoreNd, PrefetchNd) have no consumer and always
1537// pick their own layout from uArch.
1538//
1539// * Derivation direction between inst_data and lane_layout/lane_data. Both
1540// obey the invariant inst_data = k * lane_layout * lane_data, where `k` is
1541// a per-dim integer >= 1 giving how many times each lane repeats its
1542// access to cover one instruction's data tile (k == 1 means one lane
1543// position per element; k > 1 means the instruction loads/stores several
1544// elements per lane along that dim). Ops solve this invariant from
1545// opposite ends:
1546// - Rigid-lane ops (Nd block IO, DPAS): hardware fixes lane_layout /
1547// lane_data first, then inst_data is built as a multiple of their
1548// product (using get2DBlockIOInstDataLayout / getDpasInstDataLayouts).
1549// - inst_data-first ops (scatter load): take inst_data from the consumer
1550// and derive lane_layout/lane_data underneath it.
1551//
1552// - DPAS (+DPAS_MX) : rigid lanes — inst_data from HW block dims; A/B/C/D
1553// lanes/data follow each operand's matmul role; DPAS_MX
1554// additionally lays out the scale operand.
1555// - LoadNd : consumer-driven, rigid lanes — honors the consumer's
1556// inst_data / lane / sg_layout (incl. transpose & VNNI
1557// packing) when it satisfies uArch block constraints,
1558// else falls back to the default 2D-block scheme (lanes
1559// on the last dim, rank-2 if transposed). The fallback
1560// picks the LARGEST uArch block that divides the data
1561// shape, so the resulting inst_data block can be bigger
1562// than what the consumer asked for (fewer, wider
1563// loads).
1564// - StoreNd/PrefetchNd: data sinks, no consumer, rigid lanes — pick the
1565// 2D-block layout directly from uArch (no VNNI
1566// packing).
1567// - Load (scatter) : load_gather / load_matrix, consumer-driven,
1568// inst_data-first — reuse the consumer's inst_data and
1569// derive lane_layout/lane_data, else default to lanes +
1570// per-lane chunk on the innermost dim (chunk capped by
1571// maxChunkSize).
1572// - Store (scatter) : store_scatter / store_matrix — same scatter scheme,
1573// but always self-derived from the scatter default.
1574// - Reduction : multi_reduction derives source lanes and instruction
1575// blocking from the consumer, assigning spare lanes to
1576// the innermost dim if reduced. The result layout is a
1577// slice over the reduction dims. See
1578// setupMultiReductionResultLayout for each layout kind.
1579// - BitCast/Interleave: scale the innermost data field by the bitwidth /
1580// interleave ratio so the source layout divides back
1581// out.
1582// - InsertStridedSlice: clamp lane_data per dim to fit the inserted slice
1583// (Lane kind only; sg/inst layouts unsupported).
1584//===----------------------------------------------------------------------===//
1585
1586/// Helper function to set up subgroup layouts for DPAS operands A, B, and
1587/// C/D. Compute subgroup layout candidates based on wgtile and instData, and
1588/// then pick the best one that satisfies all operands and the consumer (if
1589/// specified).
1590static std::optional<
1591 std::tuple<xegpu::DistributeLayoutAttr, xegpu::DistributeLayoutAttr,
1592 xegpu::DistributeLayoutAttr>>
1593getDpasSubgroupLayouts(
1594 mlir::MLIRContext *context, VectorType aTy, VectorType bTy, VectorType cdTy,
1595 xegpu::DistributeLayoutAttr consumerLayout, int numSg,
1596 std::tuple<SmallVector<int64_t>, SmallVector<int64_t>, SmallVector<int64_t>>
1597 instDataVecs,
1598 const xegpu::uArch::MMAInstructionInterface *uArchInstruction) {
1599 SmallVector<int64_t> instDataA = std::get<0>(t&: instDataVecs);
1600 SmallVector<int64_t> instDataB = std::get<1>(t&: instDataVecs);
1601 SmallVector<int64_t> instDataCD = std::get<2>(t&: instDataVecs);
1602
1603 std::optional<LayoutRepresentation> consumerSgLayout = std::nullopt;
1604 if (consumerLayout && consumerLayout.isForWorkgroup()) {
1605 consumerSgLayout = consumerLayout.getEffectiveSgLayoutAsInt();
1606 }
1607
1608 // Picks the best subgroup layout common to A, B and C/D for the given
1609 // inst_data, or std::nullopt if there is none.
1610 auto pickSgLayout = [&]() -> std::optional<LayoutRepresentation> {
1611 auto layoutsA = getSgLayoutCandidates(wgShape: aTy.getShape(), instData: instDataA, sgCount: numSg,
1612 /*broadcastDim=*/aTy.getRank() - 1);
1613 auto layoutsB = getSgLayoutCandidates(wgShape: bTy.getShape(), instData: instDataB, sgCount: numSg,
1614 /*broadcastDim=*/bTy.getRank() - 2);
1615 auto layoutsCD = getSgLayoutCandidates(wgShape: cdTy.getShape(), instData: instDataCD, sgCount: numSg);
1616 if (layoutsA.empty() || layoutsB.empty() || layoutsCD.empty())
1617 return std::nullopt;
1618
1619 std::optional<LayoutRepresentation> bestPick;
1620 for (auto &sgLayout : layoutsB) {
1621 if (llvm::is_contained(Range&: layoutsA, Element: sgLayout) &&
1622 llvm::is_contained(Range&: layoutsCD, Element: sgLayout)) {
1623 // Is in (A and B and CD) and matches consumer -> best pick
1624 if (consumerSgLayout.has_value() && sgLayout == *consumerSgLayout)
1625 return sgLayout;
1626 // Is in (A and B and CD) layoutsB is ordered from most
1627 // balanced to least. So the first one we see is the most balanced
1628 // one, remember it and later only update if there is one that matches
1629 // the consumer.
1630 if (!bestPick)
1631 bestPick = sgLayout;
1632 }
1633 }
1634 return bestPick;
1635 };
1636
1637 // Try supported M (repeat count) values, largest first, until one admits a
1638 // common subgroup layout. N and K are fixed, so only M can shrink.
1639 int64_t mDimA = aTy.getRank() - 2;
1640 int64_t mDimCD = cdTy.getRank() - 2;
1641 auto supportedM = uArchInstruction->getSupportedM(type: aTy.getElementType());
1642 llvm::sort(C&: supportedM, Comp: std::greater<uint32_t>());
1643
1644 std::optional<LayoutRepresentation> bestPick;
1645 for (unsigned m : supportedM) {
1646 if (m == 0 || aTy.getShape()[mDimA] % m != 0)
1647 continue;
1648 instDataA[mDimA] = m;
1649 instDataCD[mDimCD] = m;
1650 if ((bestPick = pickSgLayout()))
1651 break;
1652 }
1653 if (!bestPick)
1654 return std::nullopt;
1655
1656 const auto &picked = *bestPick;
1657
1658 auto dpasALayout = buildSgLayout(context, wgTileShape: aTy.getShape(), sgLayout: picked,
1659 /*dimK=*/aTy.getRank() - 1);
1660 auto dpasBLayout = buildSgLayout(context, wgTileShape: bTy.getShape(), sgLayout: picked,
1661 /*dimK=*/bTy.getRank() - 2);
1662 auto dpasCDLayout = buildSgLayout(context, wgTileShape: cdTy.getShape(), sgLayout: picked);
1663 return std::make_tuple(args&: dpasALayout, args&: dpasBLayout, args&: dpasCDLayout);
1664}
1665
1666/// Sets up the anchor layouts for dpas operands (A, B, and C/D).
1667/// The numSg and consumerLayout (optional) are only used by sg layout
1668/// creation.
1669std::optional<
1670 std::tuple<xegpu::DistributeLayoutAttr, xegpu::DistributeLayoutAttr,
1671 xegpu::DistributeLayoutAttr>>
1672xegpu::setupDpasLayout(xegpu::LayoutKind layoutKind, VectorType aTy,
1673 VectorType bTy, VectorType cdTy,
1674 xegpu::DistributeLayoutAttr consumerLayout, int numSg,
1675 const xegpu::uArch::uArch *uArch) {
1676 auto context = aTy.getContext();
1677 const auto *uArchInstruction =
1678 dyn_cast<xegpu::uArch::SubgroupMatrixMultiplyAcc>(Val: uArch->getInstruction(
1679 instKind: xegpu::uArch::InstructionKind::SubgroupMatrixMultiplyAcc));
1680 if (!uArchInstruction)
1681 return std::nullopt;
1682 auto subgroupSize = uArch->getSubgroupSize();
1683
1684 auto [laneLayoutA, laneDataA, orderA] =
1685 compute2DBlockIOLaneLayout(instShape: aTy.getShape(), subgroupSize,
1686 bitwidth: aTy.getElementType().getIntOrFloatBitWidth(),
1687 packingSize: uArchInstruction->getPackedFormatBitSizeA());
1688 auto [laneLayoutB, laneDataB, orderB] = compute2DBlockIOLaneLayout(
1689 instShape: bTy.getShape(), subgroupSize,
1690 bitwidth: bTy.getElementType().getIntOrFloatBitWidth(),
1691 packingSize: uArchInstruction->getPackedFormatBitSizeB(), /*vnni=*/transform: true);
1692 auto [laneLayoutCD, laneDataCD, orderCD] =
1693 compute2DBlockIOLaneLayout(instShape: cdTy.getShape(), subgroupSize,
1694 bitwidth: cdTy.getElementType().getIntOrFloatBitWidth(),
1695 packingSize: cdTy.getElementType().getIntOrFloatBitWidth());
1696
1697 auto instDataVecs = getDpasInstDataLayouts(aTy, bTy, cdTy, uArchInstruction);
1698 if (!instDataVecs)
1699 return std::nullopt;
1700
1701 if (layoutKind == xegpu::LayoutKind::Subgroup) {
1702 assert(numSg > 0 &&
1703 "Number of subgroups must be provided for sg layout creation.");
1704 return getDpasSubgroupLayouts(context, aTy, bTy, cdTy, consumerLayout,
1705 numSg, instDataVecs: *instDataVecs, uArchInstruction);
1706 } else if (layoutKind == xegpu::LayoutKind::InstData) {
1707 auto [instDataA, instDataB, instDataCD] = *instDataVecs;
1708 return std::make_tuple(
1709 args: buildInstDataLayoutWithLane(context, instData: instDataA, laneLayout: laneLayoutA, laneData: laneDataA),
1710 args: buildInstDataLayoutWithLane(context, instData: instDataB, laneLayout: laneLayoutB, laneData: laneDataB),
1711 args: buildInstDataLayoutWithLane(context, instData: instDataCD, laneLayout: laneLayoutCD,
1712 laneData: laneDataCD));
1713 } else if (layoutKind == xegpu::LayoutKind::Lane) {
1714 auto aLayout = buildLaneLayout(context, laneLayout: laneLayoutA, laneData: laneDataA);
1715 auto bLayout = buildLaneLayout(context, laneLayout: laneLayoutB, laneData: laneDataB);
1716 auto cdLayout = buildLaneLayout(context, laneLayout: laneLayoutCD, laneData: laneDataCD);
1717 return std::make_tuple(args&: aLayout, args&: bLayout, args&: cdLayout);
1718 }
1719 return std::nullopt;
1720}
1721
1722/// Helper to create a scale layout derived from a matrix operand layout.
1723/// The scale layout is computed by mapping each dimension of the matrix
1724/// layout to the corresponding scale tensor dimension using the ratio
1725/// between the matrix and scale shapes.
1726static xegpu::DistributeLayoutAttr
1727createScaleLayout(mlir::MLIRContext *context, VectorType matrixTy,
1728 VectorType scaleTy, xegpu::DistributeLayoutAttr matrixLayout,
1729 bool isBScale, const xegpu::uArch::uArch *uArch) {
1730 if (!scaleTy || !matrixLayout)
1731 return nullptr;
1732
1733 // Calculate scaling factor by dividing matrix shape by scale shape
1734 ArrayRef<int64_t> matrixShape = matrixTy.getShape();
1735 ArrayRef<int64_t> scaleShape = scaleTy.getShape();
1736
1737 // Scale shapes can be 1D or 2D, handle both cases
1738 if (scaleShape.empty())
1739 return nullptr;
1740
1741 auto uArchInstruction =
1742 dyn_cast<xegpu::uArch::SubgroupScaledMatrixMultiplyAcc>(
1743 Val: uArch->getInstruction(
1744 instKind: xegpu::uArch::InstructionKind::SubgroupScaledMatrixMultiplyAcc));
1745
1746 int64_t rank = matrixLayout.getRank();
1747 assert(rank >= 2 && "dpas layouts must be at least two dimensions");
1748
1749 SmallVector<int64_t> sgLayout = matrixLayout.getEffectiveSgLayoutAsInt();
1750 SmallVector<int64_t> sgData = matrixLayout.getEffectiveSgDataAsInt();
1751 SmallVector<int64_t> instData = matrixLayout.getEffectiveInstDataAsInt();
1752 SmallVector<int64_t> laneLayout = matrixLayout.getEffectiveLaneLayoutAsInt();
1753 SmallVector<int64_t> laneData = matrixLayout.getEffectiveLaneDataAsInt();
1754 auto order = matrixLayout.getOrder();
1755
1756 SmallVector<int64_t> scaleSgLayout;
1757 SmallVector<int64_t> scaleSgData;
1758 if (!sgLayout.empty() && !sgData.empty()) {
1759 scaleSgLayout.assign(in_start: sgLayout.begin(), in_end: sgLayout.end());
1760 scaleSgData.assign(in_start: sgData.begin(), in_end: sgData.end());
1761 scaleSgData[rank - 2] = std::max<int64_t>(
1762 a: scaleShape[rank - 2] / (matrixShape[rank - 2] / sgData[rank - 2]), b: 1);
1763 scaleSgData[rank - 1] = std::max<int64_t>(
1764 a: scaleShape[rank - 1] / (matrixShape[rank - 1] / sgData[rank - 1]), b: 1);
1765 }
1766
1767 // For DPAS_MX scales: if matrix has inst_data, scale needs adjusted
1768 // inst_data. Scale inst_data is derived from matrix inst_data divided by
1769 // scale factor.
1770 SmallVector<int64_t> scaleInstData;
1771 if (!instData.empty()) {
1772 scaleInstData.assign(in_start: instData.begin(), in_end: instData.end());
1773 if (isBScale)
1774 scaleInstData[rank - 2] = std::max<int64_t>(
1775 a: scaleShape[rank - 2] / (matrixShape[rank - 2] / instData[rank - 2]),
1776 b: 1);
1777 else
1778 scaleInstData[rank - 1] = std::max<int64_t>(
1779 a: scaleShape[rank - 1] / (matrixShape[rank - 1] / instData[rank - 1]),
1780 b: 1);
1781 }
1782
1783 SmallVector<int64_t> scaleLaneLayout;
1784 SmallVector<int64_t> scaleLaneData;
1785 if (!laneLayout.empty() && !laneData.empty()) {
1786 scaleLaneLayout.assign(in_start: laneLayout.begin(), in_end: laneLayout.end());
1787 scaleLaneData.assign(NumElts: laneData.size(), Elt: 1);
1788
1789 bool isRowMajor = uArchInstruction->isLaneLayoutRowMajorOrder();
1790 if (isBScale ^ isRowMajor)
1791 std::swap(a&: scaleLaneLayout[rank - 2], b&: scaleLaneLayout[rank - 1]);
1792 // Cap lane_layout by the per-instruction tile (inst_data) on each dim.
1793 // Then derive lane_data = inst_data / lane_layout so the Category A
1794 // invariant inst_data = lane_layout * lane_data * k (with k = 1) holds
1795 // for the scale operand's load_nd consumer.
1796 auto layoutCap = scaleInstData.empty() ? scaleShape : scaleInstData;
1797 for (int64_t d = rank - 2; d < rank; ++d)
1798 scaleLaneLayout[d] = std::min<int64_t>(a: layoutCap[d], b: scaleLaneLayout[d]);
1799 }
1800 return buildLayout(context, sgLayout: scaleSgLayout, sgData: scaleSgData, instData: scaleInstData,
1801 laneLayout: scaleLaneLayout, laneData: scaleLaneData, orderAttr: order);
1802}
1803
1804/// Sets up the anchor layouts for dpas_mx operands (A, B, C/D, A_scale, and
1805/// B_scale). The numSg and consumerLayout (optional) are only used by sg
1806/// layout creation.
1807std::optional<
1808 std::tuple<xegpu::DistributeLayoutAttr, xegpu::DistributeLayoutAttr,
1809 xegpu::DistributeLayoutAttr, xegpu::DistributeLayoutAttr,
1810 xegpu::DistributeLayoutAttr>>
1811xegpu::setupDpasMxLayout(xegpu::LayoutKind layoutKind, VectorType aTy,
1812 VectorType bTy, VectorType cdTy, VectorType aScaleTy,
1813 VectorType bScaleTy,
1814 xegpu::DistributeLayoutAttr consumerLayout, int numSg,
1815 const xegpu::uArch::uArch *uArch) {
1816 auto context = aTy.getContext();
1817 const auto *uArchInstruction =
1818 dyn_cast<xegpu::uArch::SubgroupMatrixMultiplyAcc>(Val: uArch->getInstruction(
1819 instKind: xegpu::uArch::InstructionKind::SubgroupMatrixMultiplyAcc));
1820 if (!uArchInstruction)
1821 return std::nullopt;
1822 auto subgroupSize = uArch->getSubgroupSize();
1823
1824 auto [laneLayoutA, laneDataA, orderA] =
1825 compute2DBlockIOLaneLayout(instShape: aTy.getShape(), subgroupSize,
1826 bitwidth: aTy.getElementType().getIntOrFloatBitWidth(),
1827 packingSize: uArchInstruction->getPackedFormatBitSizeA());
1828 auto [laneLayoutB, laneDataB, orderB] = compute2DBlockIOLaneLayout(
1829 instShape: bTy.getShape(), subgroupSize,
1830 bitwidth: bTy.getElementType().getIntOrFloatBitWidth(),
1831 packingSize: uArchInstruction->getPackedFormatBitSizeB(), /*vnni=*/transform: true);
1832 auto [laneLayoutCD, laneDataCD, orderCD] =
1833 compute2DBlockIOLaneLayout(instShape: cdTy.getShape(), subgroupSize,
1834 bitwidth: cdTy.getElementType().getIntOrFloatBitWidth(),
1835 packingSize: cdTy.getElementType().getIntOrFloatBitWidth());
1836 auto instDataVecs = getDpasInstDataLayouts(aTy, bTy, cdTy, uArchInstruction);
1837 if (!instDataVecs)
1838 return std::nullopt;
1839
1840 if (layoutKind == xegpu::LayoutKind::Subgroup) {
1841 assert(numSg > 0 &&
1842 "Number of subgroups must be provided for sg layout creation.");
1843 auto dpasLayouts =
1844 getDpasSubgroupLayouts(context, aTy, bTy, cdTy, consumerLayout, numSg,
1845 instDataVecs: *instDataVecs, uArchInstruction);
1846 if (!dpasLayouts)
1847 return std::nullopt;
1848
1849 auto [dpasALayout, dpasBLayout, dpasCDLayout] = *dpasLayouts;
1850
1851 // Create scale layouts
1852 auto aScaleLayout =
1853 createScaleLayout(context, matrixTy: aTy, scaleTy: aScaleTy, matrixLayout: dpasALayout, isBScale: false, uArch);
1854
1855 auto bScaleLayout =
1856 createScaleLayout(context, matrixTy: bTy, scaleTy: bScaleTy, matrixLayout: dpasBLayout, isBScale: true, uArch);
1857
1858 return std::make_tuple(args&: dpasALayout, args&: dpasBLayout, args&: dpasCDLayout, args&: aScaleLayout,
1859 args&: bScaleLayout);
1860 } else if (layoutKind == xegpu::LayoutKind::InstData) {
1861
1862 auto [instDataA, instDataB, instDataCD] = *instDataVecs;
1863
1864 auto dpasALayout =
1865 buildInstDataLayoutWithLane(context, instData: instDataA, laneLayout: laneLayoutA, laneData: laneDataA);
1866 auto dpasBLayout =
1867 buildInstDataLayoutWithLane(context, instData: instDataB, laneLayout: laneLayoutB, laneData: laneDataB);
1868 auto dpasCDLayout = buildInstDataLayoutWithLane(context, instData: instDataCD,
1869 laneLayout: laneLayoutCD, laneData: laneDataCD);
1870
1871 auto aScaleLayout =
1872 createScaleLayout(context, matrixTy: aTy, scaleTy: aScaleTy, matrixLayout: dpasALayout, isBScale: false, uArch);
1873 auto bScaleLayout =
1874 createScaleLayout(context, matrixTy: bTy, scaleTy: bScaleTy, matrixLayout: dpasBLayout, isBScale: true, uArch);
1875
1876 return std::make_tuple(args&: dpasALayout, args&: dpasBLayout, args&: dpasCDLayout, args&: aScaleLayout,
1877 args&: bScaleLayout);
1878 } else if (layoutKind == xegpu::LayoutKind::Lane) {
1879 auto dpasALayout = buildLaneLayout(context, laneLayout: laneLayoutA, laneData: laneDataA);
1880 auto dpasBLayout = buildLaneLayout(context, laneLayout: laneLayoutB, laneData: laneDataB);
1881 auto dpasCDLayout = buildLaneLayout(context, laneLayout: laneLayoutCD, laneData: laneDataCD);
1882
1883 auto aScaleLayout =
1884 createScaleLayout(context, matrixTy: aTy, scaleTy: aScaleTy, matrixLayout: dpasALayout, isBScale: false, uArch);
1885 auto bScaleLayout =
1886 createScaleLayout(context, matrixTy: bTy, scaleTy: bScaleTy, matrixLayout: dpasBLayout, isBScale: true, uArch);
1887
1888 return std::make_tuple(args&: dpasALayout, args&: dpasBLayout, args&: dpasCDLayout, args&: aScaleLayout,
1889 args&: bScaleLayout);
1890 }
1891 return std::nullopt;
1892}
1893
1894/// Sets up the anchor layout for a store_nd operation. StoreNd picks its
1895/// own layout based on uArch block parameters (it does not take a consumer
1896/// layout, since it is a data sink).
1897xegpu::DistributeLayoutAttr
1898xegpu::setupStoreNdAnchorLayout(xegpu::LayoutKind layoutKind,
1899 VectorType srcVecTy, int numSg,
1900 const xegpu::uArch::uArch *uArch) {
1901 const auto *uArchInstruction =
1902 dyn_cast<xegpu::uArch::Subgroup2DBlockStoreInstruction>(
1903 Val: uArch->getInstruction(
1904 instKind: xegpu::uArch::InstructionKind::Subgroup2DBlockStore));
1905 if (!uArchInstruction)
1906 return nullptr;
1907
1908 auto context = srcVecTy.getContext();
1909 Type elemTy = srcVecTy.getElementType();
1910 auto subgroupSize = uArch->getSubgroupSize();
1911 auto dataShape = srcVecTy.getShape();
1912 [[maybe_unused]] int rank = srcVecTy.getRank();
1913 assert(rank >= 2 && "Expected at least 2D shape for ND op");
1914
1915 // Compute the default 2D block IO lane layout / lane data.
1916 unsigned bitwidth = elemTy.getIntOrFloatBitWidth();
1917 auto [laneLayout, laneData, order] =
1918 compute2DBlockIOLaneLayout(instShape: dataShape, subgroupSize, bitwidth,
1919 packingSize: uArchInstruction->getPackedFormatBitSize());
1920
1921 if (layoutKind == xegpu::LayoutKind::Lane)
1922 return buildLaneLayout(context, laneLayout, laneData);
1923
1924 auto instData =
1925 get2DBlockIOInstDataLayout(dataShape, elemTy, uArchInstruction);
1926 // Shape not realizable as a 2D-block instruction; let the caller report it.
1927 if (!instData)
1928 return nullptr;
1929
1930 if (layoutKind == xegpu::LayoutKind::InstData) {
1931 assert(isValidLaneLayout(*instData, laneLayout, laneData) &&
1932 "Expected the store layout to satisfy uArch block constraints");
1933 return buildInstDataLayoutWithLane(context, instData: *instData, laneLayout,
1934 laneData);
1935 }
1936
1937 if (layoutKind == xegpu::LayoutKind::Subgroup) {
1938 assert(numSg > 0 &&
1939 "Number of subgroups must be provided for sg layout creation.");
1940 return get2DBlockIOSubgroupLayout(context, dataShape, elemTy,
1941 uArchInstruction, numSg);
1942 }
1943
1944 return nullptr;
1945}
1946
1947/// Sets up the anchor layout for a prefetch_nd operation. PrefetchNd has no
1948/// consumer (it produces no value), so it picks its own layout from uArch
1949/// block parameters.
1950xegpu::DistributeLayoutAttr
1951xegpu::setupPrefetchNdAnchorLayout(xegpu::LayoutKind layoutKind,
1952 xegpu::TensorDescType tdescTy, int numSg,
1953 const xegpu::uArch::uArch *uArch) {
1954
1955 const auto *uArchInstruction =
1956 dyn_cast<xegpu::uArch::Subgroup2DBlockPrefetchInstruction>(
1957 Val: uArch->getInstruction(
1958 instKind: xegpu::uArch::InstructionKind::Subgroup2DBlockPrefetch));
1959 if (!uArchInstruction)
1960 return nullptr;
1961
1962 auto context = tdescTy.getContext();
1963 Type elemTy = tdescTy.getElementType();
1964 auto subgroupSize = uArch->getSubgroupSize();
1965 auto dataShape = tdescTy.getShape();
1966 [[maybe_unused]] int rank = tdescTy.getRank();
1967 assert(rank >= 2 && "Expected at least 2D shape for ND op");
1968
1969 // Compute the default 2D block IO lane layout / lane data.
1970 unsigned bitwidth = elemTy.getIntOrFloatBitWidth();
1971 auto [laneLayout, laneData, order] =
1972 compute2DBlockIOLaneLayout(instShape: dataShape, subgroupSize, bitwidth,
1973 packingSize: uArchInstruction->getPackedFormatBitSize());
1974
1975 if (layoutKind == xegpu::LayoutKind::Lane)
1976 return buildLaneLayout(context, laneLayout, laneData);
1977
1978 auto instData =
1979 get2DBlockIOInstDataLayout(dataShape, elemTy, uArchInstruction);
1980 // Shape not realizable as a 2D-block instruction; let the caller report it.
1981 if (!instData)
1982 return nullptr;
1983
1984 if (layoutKind == xegpu::LayoutKind::InstData) {
1985 assert(isValidLaneLayout(*instData, laneLayout, laneData) &&
1986 "Expected the prefetch layout to satisfy uArch block constraints");
1987 return buildInstDataLayoutWithLane(context, instData: *instData, laneLayout,
1988 laneData);
1989 }
1990
1991 if (layoutKind == xegpu::LayoutKind::Subgroup) {
1992 assert(numSg > 0 &&
1993 "Number of subgroups must be provided for sg layout creation.");
1994 return get2DBlockIOSubgroupLayout(context, dataShape, elemTy,
1995 uArchInstruction, numSg);
1996 }
1997
1998 return nullptr;
1999}
2000
2001/// Sets up the anchor layout for a load_nd operation. LoadNd takes a
2002/// consumer layout (from its result's downstream uses) and validates it
2003/// against uArch constraints; if valid, the consumer's `inst_data` /
2004/// `sg_layout` are honored. Otherwise the helper falls back to defaults
2005/// derived from uArch block parameters.
2006xegpu::DistributeLayoutAttr
2007xegpu::setupLoadNdAnchorLayout(xegpu::LayoutKind layoutKind,
2008 VectorType resVecTy,
2009 xegpu::DistributeLayoutAttr consumerLayout,
2010 int numSg, const xegpu::uArch::uArch *uArch) {
2011
2012 assert(consumerLayout && "Expected a valid consumer layout");
2013 if (layoutKind == xegpu::LayoutKind::Subgroup) {
2014 assert(consumerLayout.isForWorkgroup() &&
2015 "Expected consumer layout to be a complete workgroup-level layout");
2016 return consumerLayout;
2017 }
2018
2019 auto context = resVecTy.getContext();
2020 Type elemTy = resVecTy.getElementType();
2021 auto subgroupSize = uArch->getSubgroupSize();
2022 auto dataShape = resVecTy.getShape();
2023 const auto *uArchInstruction =
2024 dyn_cast<xegpu::uArch::Subgroup2DBlockLoadInstruction>(
2025 Val: uArch->getInstruction(
2026 instKind: xegpu::uArch::InstructionKind::Subgroup2DBlockLoad));
2027 if (!uArchInstruction)
2028 return nullptr;
2029
2030 int rank = resVecTy.getRank();
2031 SmallVector<int64_t> consumerInstData =
2032 consumerLayout.getEffectiveInstDataAsInt();
2033 SmallVector<int64_t> consumerLaneLayout =
2034 consumerLayout.getEffectiveLaneLayoutAsInt();
2035 SmallVector<int64_t> consumerLaneData =
2036 consumerLayout.getEffectiveLaneDataAsInt();
2037
2038 assert(!consumerLaneLayout.empty() && !consumerLaneData.empty() &&
2039 "Expected consumer layout to have lane_layout and lane_data");
2040
2041 // vertical lane layout means that the blockload must be transposed
2042 // note scaleA on PVC has vertical lane layout even without transposed order
2043 // attr
2044 bool hasTranspose =
2045 consumerLaneLayout[rank - 2] > 1 && consumerLaneLayout[rank - 1] == 1;
2046 // VNNI exists only below 32 bits.
2047 bool canTransform =
2048 elemTy.isIntOrFloat() &&
2049 elemTy.getIntOrFloatBitWidth() < uArch->getGeneralPackedFormatBitSize();
2050 bool hasTransform = !hasTranspose && canTransform &&
2051 consumerLaneData[rank - 2] > 1 &&
2052 consumerLaneData[rank - 1] == 1;
2053 assert((consumerLaneData[rank - 2] == 1 || consumerLaneData[rank - 1] == 1) &&
2054 "Expected consumer lane data to have at most one non-unit dim");
2055
2056 if (layoutKind == xegpu::LayoutKind::InstData) {
2057 auto blockWHC = uArchInstruction->getBlockWidthHeightCount(
2058 elemTy, hasTransform, hasTranspose,
2059 /*upConv=*/false);
2060 if (!blockWHC)
2061 return nullptr;
2062 auto [bWidths, bHeights, bCounts] = blockWHC.value();
2063
2064 // lane_layout and lane_data are the block load's own, not the consumer's: a
2065 // consumer may ask for more elements per lane than the hardware can pack,
2066 // e.g. an f32 multi_reduction wanting 16.
2067 unsigned packingSize = hasTransform || hasTranspose
2068 ? uArch->getGeneralPackedFormatBitSize()
2069 : uArchInstruction->getPackedFormatBitSize();
2070 auto [laneLayout, laneData, order] = compute2DBlockIOLaneLayout(
2071 instShape: dataShape, subgroupSize, bitwidth: elemTy.getIntOrFloatBitWidth(), packingSize,
2072 transform: hasTransform, transpose: hasTranspose);
2073 DenseI32ArrayAttr orderAttr =
2074 order.empty()
2075 ? nullptr
2076 : DenseI32ArrayAttr::get(
2077 context, content: SmallVector<int32_t>(order.begin(), order.end()));
2078
2079 // See whether the consumer's inst_data satisfies the block constraints.
2080 int64_t height = consumerInstData[rank - 2];
2081 int64_t width = consumerInstData[rank - 1];
2082 auto maxBlockCount = *llvm::max_element(Range&: bCounts);
2083 auto maxWidth = *llvm::max_element(Range&: bWidths);
2084 if (llvm::is_contained(Range&: bWidths, Element: static_cast<int>(width)) ||
2085 (width % maxWidth == 0 && width / maxWidth < maxBlockCount)) {
2086 if (llvm::is_contained(Range&: bHeights, Element: static_cast<int>(height))) {
2087 assert(isValidLaneLayout(consumerInstData, laneLayout, laneData) &&
2088 "Expected the load layout to satisfy uArch block constraints");
2089 return buildInstDataLayoutWithLane(context, instData: consumerInstData,
2090 laneLayout, laneData, orderAttr);
2091 }
2092 }
2093
2094 // The consumer's inst_data is not a usable block, so take the uArch's.
2095 // DPAS_MX scales land here. A scale whose innermost dim is too narrow to
2096 // spread packed lanes over (width 16 for an 8-bit scale needing 16 lanes of
2097 // 2 elements) cannot be loaded regularly; transformed, it packs down the
2098 // column instead. A vertical lane_layout is already transposed.
2099 // Only switch if a transformed load is possible (checking via a call
2100 // to 'get2DBlockIOInstDataLayout') otherwise keep the regularone and leave
2101 // the conflict to a convert_layout.
2102 if (!hasTranspose && !hasTransform &&
2103 consumerInstData[rank - 1] %
2104 (laneLayout[rank - 1] * laneData[rank - 1]) !=
2105 0 &&
2106 get2DBlockIOInstDataLayout(dataShape, elemTy, uArchInstruction,
2107 /*transform=*/true,
2108 /*transpose=*/false)) {
2109 hasTransform = true;
2110 std::tie(args&: laneLayout, args&: laneData, args&: order) = compute2DBlockIOLaneLayout(
2111 instShape: dataShape, subgroupSize, bitwidth: elemTy.getIntOrFloatBitWidth(),
2112 packingSize: uArch->getGeneralPackedFormatBitSize(), transform: hasTransform, transpose: hasTranspose);
2113 }
2114
2115 auto instData = get2DBlockIOInstDataLayout(
2116 dataShape, elemTy, uArchInstruction, transform: hasTransform, transpose: hasTranspose);
2117 // Shape not realizable as a 2D-block instruction; let the caller report it.
2118 if (!instData)
2119 return nullptr;
2120 assert(isValidLaneLayout(*instData, laneLayout, laneData) &&
2121 "Expected the load layout to satisfy uArch block constraints");
2122 return buildInstDataLayoutWithLane(context, instData: *instData, laneLayout, laneData,
2123 orderAttr);
2124 }
2125 if (layoutKind == xegpu::LayoutKind::Lane) {
2126 assert(isValidLaneLayout(dataShape, consumerLaneLayout, consumerLaneData) &&
2127 "Expected the lane layout to satisfy uArch block constraints");
2128 return consumerLayout;
2129 }
2130 return nullptr;
2131}
2132
2133/// Sets up the anchor layout for load gather and load matrix operation.
2134/// load matrix lowers to load gather and 1d block load. All of them share the
2135/// same layout setup logic.
2136///
2137/// For Subgroup layout, uses the consumer layout directly.
2138///
2139/// For InstData layout, takes consumer's inst_data as-is; lane_layout and
2140/// lane_data are taken from the consumer.
2141///
2142/// For Lane layout, lane_layout/lane_data are taken from the consumer.
2143///
2144/// A consumer layout that carries lane_layout and lane_data is required.
2145/// `maxChunkSize` is not read yet: the consumer's lane_data already fixes the
2146/// per-lane chunk.
2147///
2148/// TODO: derive lane_layout/lane_data here when the consumer has none, capped
2149/// by `maxChunkSize`, the way `setupGenericStoreAnchorLayout` does via
2150/// `computeScatterIOLaneLayoutAndData`. That path is missing today, so the
2151/// assert below stands in for it.
2152static xegpu::DistributeLayoutAttr setupGenericLoadAnchorLayout(
2153 xegpu::LayoutKind layoutKind, mlir::MLIRContext *context,
2154 xegpu::DistributeLayoutAttr consumerLayout, int maxChunkSize,
2155 ArrayRef<int64_t> resShape, int subgroupSize) {
2156
2157 if (layoutKind == xegpu::LayoutKind::Subgroup)
2158 return consumerLayout;
2159
2160 SmallVector<int64_t> consumerInstData =
2161 consumerLayout.getEffectiveInstDataAsInt();
2162 SmallVector<int64_t> consumerLaneLayout =
2163 consumerLayout.getEffectiveLaneLayoutAsInt();
2164 SmallVector<int64_t> consumerLaneData =
2165 consumerLayout.getEffectiveLaneDataAsInt();
2166
2167 assert(!consumerLaneLayout.empty() && !consumerLaneData.empty() &&
2168 "Expected consumer layout to have lane_layout and lane_data");
2169 SmallVector<int64_t> laneLayout(consumerLaneLayout);
2170
2171 // A matrix/gather load reads one contiguous span per lane, so a lane can pack
2172 // only along the innermost dim, and only up to maxChunkSize.
2173 SmallVector<int64_t> laneData(consumerLaneData.size(), 1);
2174 laneData.back() =
2175 std::min(a: consumerLaneData.back(), b: static_cast<int64_t>(maxChunkSize));
2176
2177 if (layoutKind == xegpu::LayoutKind::InstData) {
2178 SmallVector<int64_t> instData;
2179 instData.resize(N: resShape.size());
2180 for (size_t i = 0; i < resShape.size(); ++i)
2181 instData[i] = laneLayout[i] * laneData[i];
2182 return buildInstDataLayoutWithLane(context, instData, laneLayout, laneData);
2183 }
2184 if (layoutKind == xegpu::LayoutKind::Lane)
2185 return buildLaneLayout(context, laneLayout, laneData);
2186 return nullptr;
2187}
2188
2189/// Sets up the anchor layout for a load gather operation.
2190xegpu::DistributeLayoutAttr xegpu::setupLoadGatherAnchorLayout(
2191 xegpu::LayoutKind layoutKind, VectorType resVecTy, int contigChunkSize,
2192 xegpu::DistributeLayoutAttr consumerLayout, const uArch::uArch *uArch) {
2193
2194 const int subgroupSize = uArch->getSubgroupSize();
2195 ArrayRef<int64_t> resShape = resVecTy.getShape();
2196 auto context = resVecTy.getContext();
2197
2198 // The per-lane chunk is bounded by what the offsets prove contiguous
2199 // (`contigChunkSize`) and by what one lane can access in one instruction.
2200 const auto *uArchInstruction = dyn_cast<xegpu::uArch::LoadGatherInstruction>(
2201 Val: uArch->getInstruction(instKind: xegpu::uArch::InstructionKind::LoadGather));
2202 int maxChunkSize =
2203 std::min(a: uArchInstruction->getMaxLaneAccessSizeBytes(), b: contigChunkSize);
2204
2205 return setupGenericLoadAnchorLayout(layoutKind, context, consumerLayout,
2206 maxChunkSize, resShape, subgroupSize);
2207}
2208
2209/// Sets up the anchor layout for load matrix operation.
2210/// TODO: enhance load matrix to indicate lowering to chunked load or not.
2211xegpu::DistributeLayoutAttr
2212xegpu::setupLoadMatrixAnchorLayout(xegpu::LayoutKind layoutKind,
2213 VectorType resVecTy, int contigChunkSize,
2214 xegpu::DistributeLayoutAttr consumerLayout,
2215 const xegpu::uArch::uArch *uArch) {
2216
2217 const int subgroupSize = uArch->getSubgroupSize();
2218 ArrayRef<int64_t> resShape = resVecTy.getShape();
2219 auto context = resVecTy.getContext();
2220
2221 const auto *uArchInstruction = dyn_cast<xegpu::uArch::LoadGatherInstruction>(
2222 Val: uArch->getInstruction(instKind: xegpu::uArch::InstructionKind::LoadGather));
2223 int maxChunkSize =
2224 std::min(a: uArchInstruction->getMaxLaneAccessSizeBytes(), b: contigChunkSize);
2225 return setupGenericLoadAnchorLayout(layoutKind, context, consumerLayout,
2226 maxChunkSize, resShape, subgroupSize);
2227}
2228
2229/// Picks the subgroup layout for a scatter-style store (store_scatter /
2230/// store_matrix): the most balanced `numSg` factorization that divides
2231/// `wgShape` with sg_data a multiple of `instData`. A store has no consumer.
2232static xegpu::DistributeLayoutAttr
2233getStoreSubgroupLayouts(mlir::MLIRContext *context, ArrayRef<int64_t> wgShape,
2234 ArrayRef<int64_t> instData, int numSg) {
2235 auto candidates = getSgLayoutCandidates(wgShape, instData, sgCount: numSg);
2236 if (candidates.empty())
2237 return nullptr;
2238 // Candidates are ordered most-balanced first.
2239 return buildSgLayout(context, wgTileShape: wgShape, sgLayout: candidates.front(), /*dimK=*/-1);
2240}
2241
2242/// Sets up the anchor layout for store scatter and store matrix operation,
2243/// which share the same logic. Lane layout comes from
2244/// `computeScatterIOLaneLayoutAndData`; inst_data is lane_layout * lane_data.
2245static xegpu::DistributeLayoutAttr setupGenericStoreAnchorLayout(
2246 xegpu::LayoutKind layoutKind, mlir::MLIRContext *context, int maxChunkSize,
2247 ArrayRef<int64_t> srcShape, int subgroupSize, int numSg) {
2248
2249 auto [laneLayout, laneData] =
2250 computeScatterIOLaneLayoutAndData(instShape: srcShape, subgroupSize, maxChunkSize);
2251
2252 SmallVector<int64_t> instData(srcShape.size());
2253 for (size_t i = 0; i < srcShape.size(); ++i)
2254 instData[i] = laneLayout[i] * laneData[i];
2255
2256 if (layoutKind == xegpu::LayoutKind::Subgroup) {
2257 assert(numSg > 0 &&
2258 "Number of subgroups must be provided for sg layout creation.");
2259 return getStoreSubgroupLayouts(context, wgShape: srcShape, instData, numSg);
2260 }
2261 if (layoutKind == xegpu::LayoutKind::InstData) {
2262 return buildInstDataLayoutWithLane(context, instData, laneLayout, laneData);
2263 }
2264 if (layoutKind == xegpu::LayoutKind::Lane) {
2265 return buildLaneLayout(context, laneLayout, laneData);
2266 }
2267 return nullptr;
2268}
2269
2270/// Sets up the anchor layout for a store scatter operation.
2271xegpu::DistributeLayoutAttr
2272xegpu::setupStoreScatterAnchorLayout(xegpu::LayoutKind layoutKind,
2273 VectorType srcVecTy, int contigChunkSize,
2274 int numSg, const uArch::uArch *uArch) {
2275
2276 const int subgroupSize = uArch->getSubgroupSize();
2277 ArrayRef<int64_t> srcShape = srcVecTy.getShape();
2278 auto context = srcVecTy.getContext();
2279
2280 // The per-lane chunk is bounded by what the offsets prove contiguous
2281 // (`contigChunkSize`) and by what one lane can access in one instruction.
2282 const auto *uArchInstruction =
2283 dyn_cast<xegpu::uArch::StoreScatterInstruction>(
2284 Val: uArch->getInstruction(instKind: xegpu::uArch::InstructionKind::StoreScatter));
2285 int maxChunkSize =
2286 std::min(a: uArchInstruction->getMaxLaneAccessSizeBytes(), b: contigChunkSize);
2287 return setupGenericStoreAnchorLayout(layoutKind, context, maxChunkSize,
2288 srcShape, subgroupSize, numSg);
2289}
2290
2291/// Sets up the anchor layout for a store matrix operation.
2292xegpu::DistributeLayoutAttr xegpu::setupStoreMatrixAnchorLayout(
2293 xegpu::LayoutKind layoutKind, VectorType srcVecTy, int contigChunkSize,
2294 int numSg, const xegpu::uArch::uArch *uArch) {
2295
2296 const int subgroupSize = uArch->getSubgroupSize();
2297 ArrayRef<int64_t> srcShape = srcVecTy.getShape();
2298 auto context = srcVecTy.getContext();
2299
2300 const auto *uArchInstruction =
2301 dyn_cast<xegpu::uArch::StoreScatterInstruction>(
2302 Val: uArch->getInstruction(instKind: xegpu::uArch::InstructionKind::StoreScatter));
2303 int maxChunkSize =
2304 std::min(a: uArchInstruction->getMaxLaneAccessSizeBytes(), b: contigChunkSize);
2305
2306 return setupGenericStoreAnchorLayout(layoutKind, context, maxChunkSize,
2307 srcShape, subgroupSize, numSg);
2308}
2309
2310/// Completes a scatter IO layout by deriving lane_layout and lane_data from
2311/// `specifiedLayout`'s inst_data when they are missing. The layout is returned
2312/// unchanged if `specifiedLayout` is null, carries no inst_data, or already has
2313/// both lane_layout and lane_data.
2314///
2315/// When lane info is absent, inst_data is treated as the effective shape and
2316/// the lane factorization is filled in as follows:
2317/// - If `consumerLayout` is present and its lane_layout / lane_data are a
2318/// valid factorization of inst_data, that consumer lane info is reused so
2319/// the completed layout matches the consumer (avoiding a relayout).
2320/// - Otherwise a standard scatter-style factorization is computed via
2321/// `computeScatterIOLaneLayoutAndData`, bounded by `maxChunkSize` — the
2322/// per-lane load width reported by the uArch's LoadGather instruction
2323/// (`getMaxLaneAccessSizeBytes`).
2324///
2325std::optional<xegpu::DistributeLayoutAttr>
2326xegpu::completeScatterLoadLaneLayoutFromInstData(
2327 xegpu::DistributeLayoutAttr specifiedLayout,
2328 xegpu::DistributeLayoutAttr consumerLayout, Type elemTy,
2329 const xegpu::uArch::LoadGatherInstruction *uArchInstruction,
2330 const int subgroupSize) {
2331 if (!specifiedLayout)
2332 return specifiedLayout;
2333 SmallVector<int64_t> specifiedInstData =
2334 specifiedLayout.getEffectiveInstDataAsInt();
2335 if (specifiedInstData.empty())
2336 return specifiedLayout;
2337 if (!specifiedLayout.getEffectiveLaneLayoutAsInt().empty() &&
2338 !specifiedLayout.getEffectiveLaneDataAsInt().empty())
2339 return specifiedLayout;
2340
2341 // Reuse the load-side setup with inst_data as the destination shape.
2342 auto *context = specifiedLayout.getContext();
2343 int maxChunkSize = uArchInstruction->getMaxLaneAccessSizeBytes();
2344 if (consumerLayout) {
2345 auto consumerLaneLayout = consumerLayout.getEffectiveLaneLayoutAsInt();
2346 auto consumerLaneData = consumerLayout.getEffectiveLaneDataAsInt();
2347 if (!consumerLaneLayout.empty() && !consumerLaneData.empty() &&
2348 isValidLaneLayout(dataShape: specifiedInstData, laneLayout: consumerLaneLayout,
2349 laneData: consumerLaneData))
2350 return buildInstDataLayoutWithLane(context, instData: specifiedInstData,
2351 laneLayout: consumerLaneLayout, laneData: consumerLaneData);
2352 }
2353 auto [defLaneLayout, defLaneData] = computeScatterIOLaneLayoutAndData(
2354 instShape: specifiedInstData, subgroupSize, maxChunkSize);
2355 if (!isValidLaneLayout(dataShape: specifiedInstData, laneLayout: defLaneLayout, laneData: defLaneData))
2356 return std::nullopt;
2357 return buildInstDataLayoutWithLane(context, instData: specifiedInstData, laneLayout: defLaneLayout,
2358 laneData: defLaneData);
2359}
2360
2361/// Like completeScatterLoadLaneLayoutFromInstData, but for scatter stores. A
2362/// store is a data sink, so lane info is derived purely from inst_data (bounded
2363/// by the uArch's per-lane store width); there is no consumer layout to reuse.
2364std::optional<xegpu::DistributeLayoutAttr>
2365xegpu::completeScatterStoreLaneLayoutFromInstData(
2366 xegpu::DistributeLayoutAttr specifiedLayout, Type elemTy,
2367 const xegpu::uArch::StoreScatterInstruction *uArchInstruction,
2368 const int subgroupSize) {
2369 if (!specifiedLayout)
2370 return specifiedLayout;
2371 SmallVector<int64_t> specifiedInstData =
2372 specifiedLayout.getEffectiveInstDataAsInt();
2373 if (specifiedInstData.empty())
2374 return specifiedLayout;
2375 if (!specifiedLayout.getEffectiveLaneLayoutAsInt().empty() &&
2376 !specifiedLayout.getEffectiveLaneDataAsInt().empty())
2377 return specifiedLayout;
2378
2379 // Reuse the store-side setup with inst_data as the source shape.
2380 auto *context = specifiedLayout.getContext();
2381 int maxChunkSize = uArchInstruction->getMaxLaneAccessSizeBytes();
2382 auto [defLaneLayout, defLaneData] = computeScatterIOLaneLayoutAndData(
2383 instShape: specifiedInstData, subgroupSize, maxChunkSize);
2384 if (!isValidLaneLayout(dataShape: specifiedInstData, laneLayout: defLaneLayout, laneData: defLaneData))
2385 return std::nullopt;
2386 return buildInstDataLayoutWithLane(context, instData: specifiedInstData, laneLayout: defLaneLayout,
2387 laneData: defLaneData);
2388}
2389
2390/// Completes a 2D-block store/prefetch layout from its inst_data. store_nd and
2391/// prefetch_nd are data sinks, so lane info is derived purely from inst_data
2392/// (no consumer to reuse). One helper serves both via
2393/// BlockIOInstructionInterface.
2394std::optional<xegpu::DistributeLayoutAttr>
2395xegpu::completeBlockStoreLaneLayoutFromInstData(
2396 xegpu::DistributeLayoutAttr specifiedLayout, Type elemTy,
2397 const xegpu::uArch::BlockIOInstructionInterface *uArchInstruction,
2398 const int subgroupSize) {
2399 if (!specifiedLayout)
2400 return specifiedLayout;
2401 SmallVector<int64_t> specifiedInstData =
2402 specifiedLayout.getEffectiveInstDataAsInt();
2403 if (specifiedInstData.empty())
2404 return specifiedLayout;
2405 if (!specifiedLayout.getEffectiveLaneLayoutAsInt().empty() &&
2406 !specifiedLayout.getEffectiveLaneDataAsInt().empty())
2407 return specifiedLayout;
2408
2409 auto *context = specifiedLayout.getContext();
2410 auto [laneLayout, laneData, order] = compute2DBlockIOLaneLayout(
2411 instShape: specifiedInstData, subgroupSize, bitwidth: elemTy.getIntOrFloatBitWidth(),
2412 packingSize: uArchInstruction->getPackedFormatBitSize());
2413 if (!isValidLaneLayout(dataShape: specifiedInstData, laneLayout, laneData))
2414 return std::nullopt;
2415 return buildInstDataLayoutWithLane(context, instData: specifiedInstData, laneLayout,
2416 laneData);
2417}
2418
2419/// Like completeBlockStoreLaneLayoutFromInstData, but for load_nd. The
2420/// consumer's lane_data and order are reused as-is; lane_layout is rebuilt from
2421/// the consumer's lane_layout, bumping every non-unit dim up to the subgroup
2422/// size. The user-provided inst_data is preserved.
2423std::optional<xegpu::DistributeLayoutAttr>
2424xegpu::completeBlockLoadLaneLayoutFromInstData(
2425 xegpu::DistributeLayoutAttr specifiedLayout,
2426 xegpu::DistributeLayoutAttr consumerLayout, Type elemTy,
2427 const xegpu::uArch::BlockIOInstructionInterface *uArchInstruction,
2428 const int subgroupSize) {
2429 if (!specifiedLayout)
2430 return specifiedLayout;
2431 SmallVector<int64_t> specifiedInstData =
2432 specifiedLayout.getEffectiveInstDataAsInt();
2433 if (specifiedInstData.empty())
2434 return specifiedLayout;
2435 if (!specifiedLayout.getEffectiveLaneLayoutAsInt().empty() &&
2436 !specifiedLayout.getEffectiveLaneDataAsInt().empty())
2437 return specifiedLayout;
2438 if (!consumerLayout)
2439 return specifiedLayout;
2440 SmallVector<int64_t> consumerLaneLayout =
2441 consumerLayout.getEffectiveLaneLayoutAsInt();
2442 SmallVector<int64_t> consumerLaneData =
2443 consumerLayout.getEffectiveLaneDataAsInt();
2444 if (consumerLaneLayout.empty() || consumerLaneData.empty())
2445 return specifiedLayout;
2446
2447 auto *context = specifiedLayout.getContext();
2448 int rank = specifiedInstData.size();
2449
2450 SmallVector<int64_t> laneLayout;
2451 // set the laneLayout to use consumer's LaneLayout as base, but adjust its
2452 // size to match the subgroupsize in case its original value is larger than 1
2453 for (int i = 0; i < rank; i++) {
2454 if (consumerLaneLayout[i] > 1) {
2455 laneLayout.push_back(
2456 Elt: std::max(a: static_cast<int64_t>(subgroupSize), b: consumerLaneLayout[i]));
2457 } else {
2458 laneLayout.push_back(Elt: 1);
2459 }
2460 }
2461
2462 if (!isValidLaneLayout(dataShape: specifiedInstData, laneLayout, laneData: consumerLaneData))
2463 return std::nullopt;
2464 return buildInstDataLayoutWithLane(context, instData: specifiedInstData, laneLayout,
2465 laneData: consumerLaneData,
2466 orderAttr: consumerLayout.getOrder());
2467}
2468
2469/// Completes user-provided DPAS A/B/C-D anchors that carry only inst_data by
2470/// filling in lane_layout / lane_data. The lane factorization mirrors the
2471/// InstData branch of `setupDpasLayout` (derived from each operand's shape and
2472/// matmul role, B using VNNI packing); the user's inst_data is preserved.
2473std::optional<
2474 std::tuple<xegpu::DistributeLayoutAttr, xegpu::DistributeLayoutAttr,
2475 xegpu::DistributeLayoutAttr>>
2476xegpu::completeDpasLaneLayoutFromInstData(xegpu::DistributeLayoutAttr aLayout,
2477 xegpu::DistributeLayoutAttr bLayout,
2478 xegpu::DistributeLayoutAttr cdLayout,
2479 VectorType aTy, VectorType bTy,
2480 VectorType cdTy,
2481 const xegpu::uArch::uArch *uArch) {
2482 auto context = aTy.getContext();
2483 const auto *uArchInstruction =
2484 dyn_cast<xegpu::uArch::SubgroupMatrixMultiplyAcc>(Val: uArch->getInstruction(
2485 instKind: xegpu::uArch::InstructionKind::SubgroupMatrixMultiplyAcc));
2486 if (!uArchInstruction)
2487 return std::nullopt;
2488 auto subgroupSize = uArch->getSubgroupSize();
2489 llvm::SmallVector<int64_t> laneLayoutA, laneDataA, laneLayoutB, laneDataB,
2490 laneLayoutCD, laneDataCD;
2491 // DPAS operands are never transposed, so the order comes back empty.
2492 llvm::SmallVector<int64_t> orderA, orderB, orderCD;
2493 SmallVector<int64_t> instDataA = aLayout.getEffectiveInstDataAsInt();
2494 SmallVector<int64_t> instDataB = bLayout.getEffectiveInstDataAsInt();
2495 SmallVector<int64_t> instDataCD = cdLayout.getEffectiveInstDataAsInt();
2496
2497 if (isa<xegpu::uArch::Xe2, xegpu::uArch::Xe3p>(Val: uArch)) {
2498 std::tie(args&: laneLayoutA, args&: laneDataA, args&: orderA) =
2499 compute2DBlockIOLaneLayout(instShape: aTy.getShape(), subgroupSize,
2500 bitwidth: aTy.getElementType().getIntOrFloatBitWidth(),
2501 packingSize: uArchInstruction->getPackedFormatBitSizeA());
2502 std::tie(args&: laneLayoutB, args&: laneDataB, args&: orderB) = compute2DBlockIOLaneLayout(
2503 instShape: bTy.getShape(), subgroupSize,
2504 bitwidth: bTy.getElementType().getIntOrFloatBitWidth(),
2505 packingSize: uArchInstruction->getPackedFormatBitSizeB(), /*vnni=*/transform: true);
2506 std::tie(args&: laneLayoutCD, args&: laneDataCD, args&: orderCD) = compute2DBlockIOLaneLayout(
2507 instShape: cdTy.getShape(), subgroupSize,
2508 bitwidth: cdTy.getElementType().getIntOrFloatBitWidth(),
2509 packingSize: cdTy.getElementType().getIntOrFloatBitWidth());
2510 } else {
2511 assert(false && "Unsupported uArch for DPAS lane layout completion");
2512 }
2513
2514 if (!isValidLaneLayout(dataShape: instDataA, laneLayout: laneLayoutA, laneData: laneDataA) ||
2515 !isValidLaneLayout(dataShape: instDataB, laneLayout: laneLayoutB, laneData: laneDataB) ||
2516 !isValidLaneLayout(dataShape: instDataCD, laneLayout: laneLayoutCD, laneData: laneDataCD))
2517 return std::nullopt;
2518 return std::make_tuple(
2519 args: buildInstDataLayoutWithLane(context, instData: instDataA, laneLayout: laneLayoutA, laneData: laneDataA,
2520 orderAttr: aLayout.getOrder()),
2521 args: buildInstDataLayoutWithLane(context, instData: instDataB, laneLayout: laneLayoutB, laneData: laneDataB,
2522 orderAttr: bLayout.getOrder()),
2523 args: buildInstDataLayoutWithLane(context, instData: instDataCD, laneLayout: laneLayoutCD, laneData: laneDataCD,
2524 orderAttr: cdLayout.getOrder()));
2525}
2526
2527/// Like completeDpasLaneLayoutFromInstData, but for dpas_mx: also re-derives
2528/// the A_scale / B_scale layouts from the completed A / B layouts via
2529/// `createScaleLayout`, matching the default path of `setupDpasMxLayout`.
2530std::optional<
2531 std::tuple<xegpu::DistributeLayoutAttr, xegpu::DistributeLayoutAttr,
2532 xegpu::DistributeLayoutAttr, xegpu::DistributeLayoutAttr,
2533 xegpu::DistributeLayoutAttr>>
2534xegpu::completeDpasMxLaneLayoutFromInstData(
2535 xegpu::DistributeLayoutAttr aLayout, xegpu::DistributeLayoutAttr bLayout,
2536 xegpu::DistributeLayoutAttr cdLayout, VectorType aTy, VectorType bTy,
2537 VectorType cdTy, VectorType aScaleTy, VectorType bScaleTy,
2538 const xegpu::uArch::uArch *uArch) {
2539 auto completed = completeDpasLaneLayoutFromInstData(
2540 aLayout, bLayout, cdLayout, aTy, bTy, cdTy, uArch);
2541 if (!completed)
2542 return std::nullopt;
2543 auto context = aTy.getContext();
2544 auto [completedA, completedB, completedCD] = *completed;
2545
2546 auto aScaleLayout =
2547 createScaleLayout(context, matrixTy: aTy, scaleTy: aScaleTy, matrixLayout: completedA, isBScale: false, uArch);
2548 auto bScaleLayout =
2549 createScaleLayout(context, matrixTy: bTy, scaleTy: bScaleTy, matrixLayout: completedB, isBScale: true, uArch);
2550
2551 return std::make_tuple(args&: completedA, args&: completedB, args&: completedCD, args&: aScaleLayout,
2552 args&: bScaleLayout);
2553}
2554
2555/// Sets up layout for reduction operations by creating a SliceAttr for the
2556/// result.
2557///
2558/// Algorithm Overview:
2559/// This function attempts to construct a source layout that, when sliced along
2560/// reduction dimensions, produces a result layout compatible with the
2561/// consumer layout.
2562///
2563/// For subgroup layouts, it first tries to align the source layout's subgroup
2564/// layout and data with the consumer's layout on non-reduction dimensions.
2565/// Then, it distributes remaining subgroups across reduction dimensions. When
2566/// the consumer slices exactly `reductionDims`, it reuses the slice's parent
2567/// for the source and adjusts sg_data on the reduction dimensions to fit the
2568/// source extent. This is a best-effort alignment: the goal is to pick a source
2569/// layout that minimizes redistribution against the consumer layout.
2570///
2571/// If the resulting source layout still differs from what some consumer
2572/// expects (e.g. a second, inconsistent consumer), that mismatch is reconciled
2573/// later by layout conflict resolution (`ResolveLayoutConflicts`), which
2574/// inserts a `convert_layout` op.
2575///
2576/// InstData kind: `computeReductionLaneLayoutAndData` derives lane_layout,
2577/// lane_data, and inst_data. It can preserve a reused parent's larger inst_data
2578/// on non-reduced dimensions.
2579///
2580/// Lane kind: a consumer sliced over exactly `reductionDims` is reused.
2581/// This preserves the lane layout chosen during instruction-data propagation.
2582/// For other consumer layouts, `computeReductionLaneLayoutAndData` derives
2583/// lane_layout and lane_data. See that helper for inst_data and lane layout
2584/// examples.
2585///
2586/// The function returns the *result* layout (the SliceAttr). The *source*
2587/// layout it decides on is the parent of that slice. The subgroup examples
2588/// below show both layouts.
2589///
2590/// Subgroup layout examples:
2591/// 1. Row reduction on a 2D tensor:
2592/// srcShape=[32, 128], reductionDims=[1], resShape=[32], subgroupSize=16,
2593/// numSg=32
2594/// * Consumer Layout:
2595/// #xegpu.slice<#xegpu.layout<sg_layout=[4, 8], sg_data=[8, 8]>, dims =
2596/// [1]>
2597/// * Source Layout (decided by this function):
2598/// #xegpu.layout<sg_layout=[4, 8], sg_data=[8, 16]>
2599/// * Result Layout (returned):
2600/// #xegpu.slice<#xegpu.layout<sg_layout=[4, 8], sg_data=[8, 16]>, dims =
2601/// [1]>
2602/// The consumer slices exactly the reduction dim, so its parent layout is
2603/// reused for the source: sg_layout is kept, but the source's sg_data on
2604/// the reduction dim is grown from 8 to 16 (= srcShape[1] / sg_layout[1] =
2605/// 128 / 8) so the source tile is evenly distributed over the reduction
2606/// dim. The non-reduction dimensions keep the consumer's distribution.
2607///
2608/// 2. Same shapes as above, but the consumer has no reusable slice layout.
2609/// Copy its subgroup distribution onto non-reduction dims and assign the
2610/// remaining subgroups to reduction dims.
2611/// 2a. * Consumer Layout:
2612/// #xegpu.layout<sg_layout=[32], sg_data=[1]>
2613/// * Source Layout (decided by this function):
2614/// #xegpu.layout<sg_layout=[32, 1], sg_data=[1, 128]>
2615/// * Result Layout (returned):
2616/// #xegpu.slice<#xegpu.layout<sg_layout=[32, 1], sg_data=[1, 128]>,
2617/// dims = [1]>
2618/// All 32 subgroups are assigned to non-reduction dim 0. Dim 1 stays
2619/// within each subgroup (sg_layout=1), so its sg_data is 128.
2620/// 2b. * Consumer Layout:
2621/// #xegpu.slice<#xegpu.layout<sg_layout=[8, 2, 2], sg_data=[4, 64,
2622/// 32]>, dims = [1, 2]>
2623/// * Source Layout (decided by this function):
2624/// #xegpu.layout<sg_layout=[8, 4], sg_data=[4, 32]>
2625/// * Result Layout (returned):
2626/// #xegpu.slice<#xegpu.layout<sg_layout=[8, 4], sg_data=[4, 32]>,
2627/// dims = [1]>
2628/// The consumer slices dims [1, 2], but the reduction uses dim 1.
2629/// Copy its effective sg_layout=[8] onto dim 0 and assign the
2630/// remaining four subgroups to dim 1.
2631
2632xegpu::SliceAttr xegpu::setupMultiReductionResultLayout(
2633 xegpu::LayoutKind layoutKind, VectorType srcVecTy,
2634 DistributeLayoutAttr consumerLayout, SmallVector<int64_t> reductionDims,
2635 int numSg, const xegpu::uArch::uArch *uArch) {
2636
2637 auto srcShape = srcVecTy.getShape();
2638 int srcRank = srcShape.size();
2639 auto context = srcVecTy.getContext();
2640
2641 const int subgroupSize = uArch->getSubgroupSize();
2642 int64_t maxReduceVectorSize = 16; // extend to spirv vector Size
2643 xegpu::DistributeLayoutAttr srcLayout;
2644 if (layoutKind == xegpu::LayoutKind::Subgroup) {
2645 xegpu::SliceAttr consumerSliceLayout =
2646 dyn_cast_if_present<xegpu::SliceAttr>(Val&: consumerLayout);
2647 if (consumerSliceLayout &&
2648 consumerSliceLayout.getDims().asArrayRef().equals(RHS: reductionDims)) {
2649 srcLayout = consumerSliceLayout.getParent();
2650 SmallVector<int64_t> sgLayoutFromConsumer =
2651 srcLayout.getEffectiveSgLayoutAsInt();
2652 auto srcSgData = computeShapeRatio(shape: srcShape, subShape: sgLayoutFromConsumer);
2653 if (srcSgData)
2654 for (int dim = 0; dim < srcRank; dim++) {
2655 if (llvm::is_contained(Range&: reductionDims, Element: dim))
2656 srcLayout =
2657 srcLayout.setDimData(dim, sgData: srcSgData.value()[dim], instData: -1, laneData: -1);
2658 }
2659 } else {
2660 SmallVector<int64_t> consumerSgLayout =
2661 consumerLayout ? consumerLayout.getEffectiveSgLayoutAsInt()
2662 : SmallVector<int64_t>();
2663 SmallVector<int64_t> consumerSgData =
2664 consumerLayout ? consumerLayout.getEffectiveSgDataAsInt()
2665 : SmallVector<int64_t>();
2666 SmallVector<int64_t> consumerOrder =
2667 consumerLayout ? consumerLayout.getEffectiveOrderAsInt()
2668 : SmallVector<int64_t>();
2669 DenseI32ArrayAttr orderAttr =
2670 consumerLayout ? consumerLayout.getOrder() : nullptr;
2671 SmallVector<int64_t> sgLayout(srcRank), sgData(srcRank), order(srcRank);
2672 int remainingSgCount =
2673 consumerLayout ? consumerLayout.getNumSubgroups() : numSg;
2674 int consumerIdx = 0;
2675
2676 // First pass: match the consumer's layout on non-reduction dims.
2677 for (int i = 0; i < srcRank; i++) {
2678 if (!llvm::is_contained(Range&: reductionDims, Element: i) &&
2679 consumerIdx < static_cast<int>(consumerSgLayout.size())) {
2680 sgLayout[i] = consumerSgLayout[consumerIdx];
2681 sgData[i] = consumerSgData[consumerIdx];
2682 remainingSgCount /= sgLayout[i];
2683 consumerIdx++;
2684 }
2685 }
2686
2687 // Second pass: distribute remaining subgroups across the reduction dims.
2688 // The reduction-to-scalar case is handled only by this loop.
2689 for (int i = 0; i < srcRank; i++) {
2690 if (llvm::is_contained(Range&: reductionDims, Element: i)) {
2691 sgLayout[i] =
2692 std::min(a: srcShape[i], b: static_cast<int64_t>(remainingSgCount));
2693 assert((srcShape[i] % sgLayout[i] == 0) &&
2694 "source shape not divisible by sg_layout");
2695 sgData[i] = srcShape[i] / sgLayout[i];
2696 remainingSgCount /= sgLayout[i];
2697 }
2698 }
2699 // The reduction dims are assumed to be row-major contiguous with the
2700 // left-hand neighbor.
2701 DenseI32ArrayAttr resOrderAttr = nullptr;
2702 int numRetainedDims = srcRank - static_cast<int>(reductionDims.size());
2703 if (orderAttr && !orderAttr.empty() &&
2704 static_cast<int>(consumerOrder.size()) == numRetainedDims) {
2705 // Fill known order
2706 int retainedRank = 0;
2707 for (int dim = 0; dim < srcRank; dim++)
2708 if (!llvm::is_contained(Range&: reductionDims, Element: dim))
2709 order[dim] = consumerOrder[retainedRank++];
2710 // Each new order is placed as close as possible to its neighbors,
2711 // row-major. Re-index the order of higher walk orders.
2712 for (int reducedDim = 0; reducedDim < srcRank; reducedDim++) {
2713 if (!llvm::is_contained(Range&: reductionDims, Element: reducedDim))
2714 continue;
2715 int64_t insertRank = 0;
2716 // Add as an inner dim to the outer neighbor
2717 if (reducedDim)
2718 insertRank = order[reducedDim - 1];
2719 // Add as an outer dim to the inner neighbor if no left neighbor
2720 // exists.
2721 else if (srcRank > 1)
2722 insertRank = order[reducedDim + 1] + 1;
2723 for (int otherDim = 0; otherDim < srcRank; otherDim++)
2724 if (order[otherDim] >= insertRank)
2725 order[otherDim]++;
2726 order[reducedDim] = insertRank;
2727 }
2728 resOrderAttr = DenseI32ArrayAttr::get(
2729 context, content: SmallVector<int32_t>(order.begin(), order.end()));
2730 }
2731 assert(remainingSgCount == 1 && "not all subgroups distributed");
2732 srcLayout = buildLayout(context, sgLayout, sgData,
2733 /*instData=*/{}, /*laneLayout=*/{},
2734 /*laneData=*/{}, orderAttr: resOrderAttr);
2735 }
2736 } else if (layoutKind == xegpu::LayoutKind::InstData) {
2737 auto [laneLayout, laneData, instData] =
2738 computeReductionLaneLayoutAndData(srcShape, reductionDims, subgroupSize,
2739 maxReduceVectorSize, consumerLayout);
2740 srcLayout =
2741 buildInstDataLayoutWithLane(context, instData, laneLayout, laneData);
2742 } else if (layoutKind == xegpu::LayoutKind::Lane) {
2743 xegpu::SliceAttr consumerSliceLayout =
2744 dyn_cast_if_present<xegpu::SliceAttr>(Val&: consumerLayout);
2745 if (consumerSliceLayout &&
2746 consumerSliceLayout.getDims().asArrayRef().equals(RHS: reductionDims)) {
2747 // at the lane level, the consumerSliceLayout can be directly reused
2748 // since the inst_data propagation already insert convert_layout if
2749 // the layout is not consistent
2750 srcLayout = consumerSliceLayout.getParent();
2751 } else {
2752 auto [laneLayout, laneData, instData] = computeReductionLaneLayoutAndData(
2753 srcShape, reductionDims, subgroupSize, maxReduceVectorSize,
2754 consumerLayout);
2755 (void)instData;
2756 srcLayout = buildLaneLayout(context, laneLayout, laneData);
2757 }
2758 }
2759
2760 return xegpu::SliceAttr::get(context, parent: srcLayout,
2761 dims: DenseI64ArrayAttr::get(context, content: reductionDims));
2762}
2763
2764/// Sets up layout for Reduction operations by creating a SliceAttr for the
2765/// result.
2766xegpu::SliceAttr
2767xegpu::setupReductionResultLayout(xegpu::LayoutKind layoutKind,
2768 VectorType srcVecTy,
2769 const xegpu::uArch::uArch *uArch) {
2770
2771 auto srcShape = srcVecTy.getShape();
2772 auto context = srcVecTy.getContext();
2773 auto subgroupSize = uArch->getSubgroupSize();
2774 xegpu::LayoutAttr srcLayout;
2775
2776 if (layoutKind == xegpu::LayoutKind::Subgroup) {
2777 assert(false &&
2778 "subgroup layout assignment not supported for reduction (op "
2779 "is not expected at this level).");
2780 } else if (layoutKind == xegpu::LayoutKind::InstData) {
2781 assert(false &&
2782 "instData layout assignment not supported for reduction (op "
2783 "is not expected at this level).");
2784 } else if (layoutKind == xegpu::LayoutKind::Lane) {
2785 SmallVector<int64_t> laneLayout(1), laneData(1);
2786 laneLayout[0] = std::min(a: static_cast<int64_t>(subgroupSize), b: srcShape[0]);
2787 laneData[0] = 1;
2788 srcLayout = buildLaneLayout(context, laneLayout, laneData);
2789 }
2790
2791 auto result = xegpu::SliceAttr::get(context, parent: srcLayout,
2792 dims: DenseI64ArrayAttr::get(context, content: 0));
2793 return result;
2794}
2795
2796/// Adjusts `consumerLayout`'s innermost-dim data field selected by
2797/// `layoutKind` so that the source layout can be safely inferred by dividing
2798/// that value by `ratio`. Doubles the value until the divisibility constraint
2799/// is met, bounded above by `bound` like result-shape.
2800///
2801/// Used by ops whose source relates to the result by a fixed factor along the
2802/// innermost dim (e.g., bitcast: bitwidth ratio; interleave: 2x).
2803///
2804/// Divisibility constraints per LayoutKind:
2805/// - Subgroup: sgData[innermost] % ratio == 0
2806/// - InstData: instData[innermost] % (laneLayout[innermost] * ratio) == 0
2807/// (laneLayout falls back to subgroupSize if absent)
2808/// - Lane: laneData[innermost] % ratio == 0
2809static xegpu::DistributeLayoutAttr
2810adjustInnermostDimForDivisibility(xegpu::DistributeLayoutAttr consumerLayout,
2811 xegpu::LayoutKind layoutKind,
2812 size_t innerMostDim, int ratio, int64_t bound,
2813 const xegpu::uArch::uArch *uArch) {
2814 SmallVector<int64_t> sgData = consumerLayout.getEffectiveSgDataAsInt();
2815 SmallVector<int64_t> instData = consumerLayout.getEffectiveInstDataAsInt();
2816 SmallVector<int64_t> laneData = consumerLayout.getEffectiveLaneDataAsInt();
2817 SmallVector<int64_t> laneLayout =
2818 consumerLayout.getEffectiveLaneLayoutAsInt();
2819
2820 int64_t sgDataValue = -1;
2821 int64_t instDataValue = -1;
2822 int64_t laneDataValue = -1;
2823
2824 if (layoutKind == xegpu::LayoutKind::Subgroup) {
2825 sgDataValue = sgData[innerMostDim];
2826 while ((sgDataValue <= bound) && (sgDataValue % ratio) != 0)
2827 sgDataValue *= 2;
2828 } else if (layoutKind == xegpu::LayoutKind::InstData) {
2829 instDataValue = instData[innerMostDim];
2830 const int innermostDimLaneLayout = laneLayout.empty()
2831 ? uArch->getSubgroupSize()
2832 : laneLayout[innerMostDim];
2833 while ((instDataValue <= bound) &&
2834 (instDataValue % (innermostDimLaneLayout * ratio) != 0))
2835 instDataValue *= 2;
2836 assert((bound % instDataValue) == 0 &&
2837 "bound, instData, and laneLayout for innermost must be 2^n!");
2838 } else if (layoutKind == xegpu::LayoutKind::Lane) {
2839 laneDataValue = laneData[innerMostDim];
2840 while ((laneDataValue <= bound) && (laneDataValue % ratio) != 0)
2841 laneDataValue *= 2;
2842 }
2843
2844 return consumerLayout.setDimData(dim: innerMostDim, sgData: sgDataValue, instData: instDataValue,
2845 laneData: laneDataValue);
2846}
2847
2848/// Sets up the result layout for a bitcast operation.
2849/// When casting to a smaller bitwidth, adjusts the layout dimensions (sgData,
2850/// instData, or laneData) by multiplying by the bitwidth ratio to ensure the
2851/// result layout can be correctly divided back to the source layout during
2852/// inference.
2853///
2854/// Examples:
2855/// 1. Casting f32 -> f16 (32-bit to 16-bit, bitWidthRatio = 2):
2856/// Consumer layout: instData=[1, 16], subgroupSize=16
2857/// Source shape: [8, 32]
2858/// Result layout: instData=[1, 32] (16 * 2)
2859/// The innermost dimension is multiplied by 2 to maintain consistency.
2860///
2861/// 2. Casting f32 -> i8 (32-bit to 8-bit, bitWidthRatio = 4):
2862/// Consumer instData=[1, 16], subgroupSize=16
2863/// Source shape: [4, 128]
2864/// adjust the instData from [1, 16] to [1, 16 * 4 = 64]
2865///
2866/// 3. Casting i8 -> i32 (8-bit to 32-bit, bitWidthRatio = 1/4):
2867/// Consumer layout: laneLayout=[1, 16], laneData=[1, 4]
2868/// No adjustment needed - returns consumer layout directly.
2869///
2870xegpu::DistributeLayoutAttr xegpu::setupBitCastResultLayout(
2871 xegpu::LayoutKind layoutKind, VectorType srcVecTy, VectorType resVecTy,
2872 DistributeLayoutAttr consumerLayout, const xegpu::uArch::uArch *uArch) {
2873
2874 int srcElemTyBitWidth = srcVecTy.getElementType().getIntOrFloatBitWidth();
2875 int resElemTyBitWidth = resVecTy.getElementType().getIntOrFloatBitWidth();
2876
2877 ArrayRef<int64_t> srcShape = srcVecTy.getShape();
2878 ArrayRef<int64_t> resShape = resVecTy.getShape();
2879
2880 assert(consumerLayout.getRank() == static_cast<int64_t>(srcShape.size()) &&
2881 "laneData must be available for all dimensions");
2882
2883 // Casting to same/larger element type: result has fewer (or equal) elements
2884 // along the innermost dim, no adjustment needed.
2885 if (srcElemTyBitWidth <= resElemTyBitWidth)
2886 return consumerLayout;
2887
2888 // Casting to smaller element type: result has more elements along innermost
2889 // dim. Adjust the innermost data field upward so the source layout can be
2890 // recovered by dividing by bitWidthRatio.
2891 size_t innerMostDim = srcShape.size() - 1;
2892 int bitWidthRatio = srcElemTyBitWidth / resElemTyBitWidth;
2893 return adjustInnermostDimForDivisibility(consumerLayout, layoutKind,
2894 innerMostDim, ratio: bitWidthRatio,
2895 bound: resShape[innerMostDim], uArch);
2896}
2897
2898/// Sets up the result layout for an interleave operation to ensure the source
2899/// layout can be safely derived. Interleave doubles the innermost dimension,
2900/// so the result layout must ensure that laneData is a multiple
2901/// of 2, and instData must be divisible by innermostDimLaneLayout * 2.
2902///
2903/// Example:
2904/// Interleave: vector<128x256xf4> -> vector<128x512xf4>
2905/// Consumer layout: laneLayout=[1, 16], laneData=[1, 4], instData=[1, 64]
2906/// Result layout adjustment to ensure source can be safely inferred:
2907/// - laneData must be >= 2 and multiple of 2 (so source = laneData/2 is
2908/// valid)
2909/// - instData must be divisible by (16 * 2 = 32) (so source = instData/2 is
2910/// valid)
2911/// - Adjusted instData: ensure (instData % 32 == 0)
2912///
2913xegpu::DistributeLayoutAttr xegpu::setupInterleaveResultLayout(
2914 xegpu::LayoutKind layoutKind, VectorType srcVecTy, VectorType resVecTy,
2915 DistributeLayoutAttr consumerLayout, const xegpu::uArch::uArch *uArch) {
2916
2917 ArrayRef<int64_t> resShape = resVecTy.getShape();
2918 assert(consumerLayout.getRank() == static_cast<int64_t>(resShape.size()) &&
2919 "consumer layout rank must match source shape rank");
2920
2921 // Interleave doubles the innermost dimension (ratio = 2). Adjust the
2922 // innermost data field so the source layout can be recovered by dividing
2923 // by 2.
2924 const size_t innerMostDim = resShape.size() - 1;
2925 constexpr int ratio = 2;
2926 return adjustInnermostDimForDivisibility(consumerLayout, layoutKind,
2927 innerMostDim, ratio,
2928 bound: resShape[innerMostDim], uArch);
2929}
2930
2931/// Checks that the tiles a split group is cut into walk the collapsed source
2932/// dim with one constant stride, which is what `collapseDims` assumes when it
2933/// multiplies a layout field across the group.
2934///
2935/// Along each dim the group is cut into `counts[dim]` tiles of `tileShape[dim]`
2936/// elements (empty for single-element tiles) taken from `shape[dim]`, which
2937/// gives the row-major strides. Walking from the innermost dim out, a dim cut
2938/// into several tiles steps by `tileShape[dim] * stride(dim)` and has to pick
2939/// up where the inner dims left off, `expectedStride`; a dim cut into one tile
2940/// does not step at all, so it only contributes its size.
2941static bool isContiguousTiling(ArrayRef<int64_t> dimGroup,
2942 ArrayRef<int64_t> shape,
2943 ArrayRef<int64_t> counts,
2944 ArrayRef<int64_t> tileShape,
2945 int64_t expectedStride) {
2946 int64_t stride = 1;
2947 for (int64_t dim : llvm::reverse(C&: dimGroup)) {
2948 int64_t tileSize = tileShape.empty() ? 1 : tileShape[dim];
2949 if (counts[dim] != 1) {
2950 if (stride * tileSize != expectedStride)
2951 return false;
2952 expectedStride = stride * tileSize * counts[dim];
2953 }
2954 stride *= shape[dim];
2955 }
2956 return true;
2957}
2958
2959xegpu::DistributeLayoutAttr
2960xegpu::setupShapeCastResultLayout(xegpu::LayoutKind layoutKind,
2961 VectorType srcVecTy, VectorType resVecTy,
2962 DistributeLayoutAttr consumerLayout) {
2963 // TODO: work out the subgroup level rule; leave such layouts alone for now.
2964 if (!consumerLayout || layoutKind == xegpu::LayoutKind::Subgroup)
2965 return consumerLayout;
2966
2967 ArrayRef<int64_t> resShape = resVecTy.getShape();
2968 SmallVector<SmallVector<int64_t>> splitDimGroups;
2969 if (!xegpu::matchSplitDimExpansion(src: srcVecTy.getShape(), dst: resShape,
2970 splitDimGroups))
2971 return consumerLayout;
2972
2973 SmallVector<int64_t> laneLayout =
2974 consumerLayout.getEffectiveLaneLayoutAsInt();
2975 SmallVector<int64_t> instData = consumerLayout.getEffectiveInstDataAsInt();
2976
2977 xegpu::DistributeLayoutAttr resLayout = [&]() -> DistributeLayoutAttr {
2978 DistributeLayoutAttr layout = consumerLayout;
2979 for (const SmallVector<int64_t> &dimGroup : splitDimGroups) {
2980 size_t i = 0;
2981 while (i < dimGroup.size() && laneLayout[dimGroup[i]] == 1)
2982 ++i;
2983
2984 for (++i; i < dimGroup.size(); ++i) {
2985 int64_t dim = dimGroup[i];
2986 int64_t dimSize = resShape[dim];
2987 int64_t lanes = laneLayout[dim];
2988 if (lanes <= 0 || dimSize % lanes != 0)
2989 return consumerLayout;
2990 layout =
2991 layout.setDimData(dim, /*sgData=*/-1,
2992 instData: instData.empty() ? -1 : dimSize, laneData: dimSize / lanes);
2993 }
2994 }
2995 return layout;
2996 }();
2997
2998 // Stretching does not make every layout collapsible; reject the rest.
2999 SmallVector<int64_t> resInstData = resLayout.getEffectiveInstDataAsInt();
3000 SmallVector<int64_t> resLaneLayout = resLayout.getEffectiveLaneLayoutAsInt();
3001 SmallVector<int64_t> resLaneData = resLayout.getEffectiveLaneDataAsInt();
3002
3003 for (const SmallVector<int64_t> &dimGroup : splitDimGroups) {
3004 // One lane's elements must be a single contiguous run of the collapsed dim.
3005 if (!isContiguousTiling(dimGroup, shape: resShape, counts: resLaneData, /*tileShape=*/{},
3006 /*expectedStride=*/1))
3007 return nullptr;
3008
3009 // Consecutive lanes must be exactly one such run apart.
3010 int64_t collapsedLaneData = 1;
3011 for (int64_t dim : dimGroup)
3012 collapsedLaneData *= resLaneData[dim];
3013 if (!isContiguousTiling(dimGroup, shape: resShape, counts: resLaneLayout, tileShape: resLaneData,
3014 /*expectedStride=*/collapsedLaneData))
3015 return nullptr;
3016
3017 // The inst tile must walk the result shape the same way.
3018 if (!resInstData.empty() &&
3019 !isContiguousTiling(dimGroup, shape: resShape, counts: resInstData, /*tileShape=*/{},
3020 /*expectedStride=*/1))
3021 return nullptr;
3022 }
3023 return resLayout;
3024}
3025
3026/// Sets up the result layout for an insert strided slice operation.
3027/// Creates a result layout based on the specified layout kind (InstData or
3028/// Lane).
3029xegpu::DistributeLayoutAttr xegpu::setupInsertStridedSliceResultLayout(
3030 xegpu::LayoutKind layoutKind, VectorType srcVectorTy,
3031 VectorType resVectorTy, xegpu::DistributeLayoutAttr consumerLayout,
3032 const xegpu::uArch::uArch *uArch) {
3033
3034 xegpu::DistributeLayoutAttr requiredResLayout;
3035 SmallVector<int64_t> consumerInstData =
3036 consumerLayout.getEffectiveInstDataAsInt();
3037 SmallVector<int64_t> consumerLaneData =
3038 consumerLayout.getEffectiveLaneDataAsInt();
3039 SmallVector<int64_t> consumerLaneLayout =
3040 consumerLayout.getEffectiveLaneLayoutAsInt();
3041 ArrayRef<int64_t> srcShape = srcVectorTy.getShape();
3042 int64_t laneDataValue = -1;
3043
3044 requiredResLayout = consumerLayout;
3045 int srcRank = srcShape.size();
3046
3047 if (layoutKind == xegpu::LayoutKind::Subgroup ||
3048 layoutKind == xegpu::LayoutKind::InstData) {
3049 assert(false && "subgroup/instData layout assignment not supported for "
3050 "insertStridedSlice.");
3051 } else if (layoutKind == xegpu::LayoutKind::Lane) {
3052 for (int dim = 0; dim < srcRank; dim++) {
3053 // A size-1 source dim is broadcast across the lanes of that dim.
3054 if (srcShape[dim] == 1) {
3055 laneDataValue = 1;
3056 } else {
3057 assert(srcShape[dim] % consumerLaneLayout[dim] == 0 &&
3058 "srcShape must be divisible by laneLayout for all dimensions");
3059 laneDataValue = std::min(a: srcShape[dim] / consumerLaneLayout[dim],
3060 b: consumerLaneData[dim]);
3061 }
3062 requiredResLayout =
3063 requiredResLayout.setDimData(dim, sgData: -1, instData: -1, laneData: laneDataValue);
3064 }
3065 }
3066 return requiredResLayout;
3067}
3068
3069/// Back-propagates a known result layout to the layout required on `operand`
3070/// for a non-anchor (layout-propagating) vector op. Dispatches on the op kind —
3071/// broadcast, (multi)reduction, bitcast, shape/transpose, insert/extract,
3072/// interleave, etc. — applying the shape/permutation/bitwidth transform to
3073/// derive the source layout; elementwise and pass-through ops reuse resLayout
3074/// as-is. Returns nullptr for unknown ops or an absent result layout.
3075xegpu::DistributeLayoutAttr xegpu::inferSourceLayoutFromResultForNonAnchorOp(
3076 OpOperand &operand, xegpu::DistributeLayoutAttr resLayout) {
3077 if (!resLayout)
3078 return nullptr;
3079 Operation *op = operand.getOwner();
3080 unsigned idx = operand.getOperandNumber();
3081
3082 // For vector::BroadcastOp, infer the source layout from the result layout.
3083 if (auto broadcast = dyn_cast<vector::BroadcastOp>(Val: op)) {
3084 auto srcTy = dyn_cast<VectorType>(Val: broadcast.getSourceType());
3085 if (!srcTy)
3086 return nullptr;
3087 return xegpu::inferBroadcastSourceLayout(
3088 resLayout, resShape: broadcast.getResultVectorType().getShape(),
3089 srcShape: srcTy.getShape());
3090 }
3091
3092 // For vector::MultiDimReductionOp, infer source layout from result layout
3093 // using reduction dims. Acc operand is expected to have the same layout as
3094 // the result.
3095 if (auto reduction = dyn_cast<vector::MultiDimReductionOp>(Val: op)) {
3096 if (idx == 0) {
3097 SmallVector<int64_t> reductionDims(reduction.getReductionDims());
3098 return xegpu::inferMultiReductionSourceLayout(resLayout, reduceDims: reductionDims);
3099 }
3100 if (idx == 1)
3101 return resLayout;
3102 }
3103
3104 if (auto reduction = dyn_cast<vector::ReductionOp>(Val: op))
3105 return xegpu::inferReductionSourceLayout(resLayout);
3106
3107 // For vector::BitCastOp, infer source layout from result layout using
3108 // element type bitwidths.
3109 if (auto bitcast = dyn_cast<vector::BitCastOp>(Val: op)) {
3110 int resElemBitWidth =
3111 bitcast.getResultVectorType().getElementType().getIntOrFloatBitWidth();
3112 int srcElemBitWidth =
3113 bitcast.getSourceVectorType().getElementType().getIntOrFloatBitWidth();
3114 return xegpu::inferBitCastSourceLayout(resLayout, resElemTyBitWidth: resElemBitWidth,
3115 srcElemTyBitWidth: srcElemBitWidth);
3116 }
3117
3118 // For vector::ShapeCastOp, infer source layout from result layout using
3119 // shapes.
3120 if (auto shapeCast = dyn_cast<vector::ShapeCastOp>(Val: op)) {
3121 return xegpu::inferShapeCastSourceLayout(
3122 resLayout, resShape: shapeCast.getResultVectorType().getShape(),
3123 srcShape: shapeCast.getSourceVectorType().getShape());
3124 }
3125
3126 // For vector::InsertStridedSliceOp, infer source layout from result
3127 // layout. Dest vector must have the same layout as the result.
3128 if (auto insertSlice = dyn_cast<vector::InsertStridedSliceOp>(Val: op)) {
3129 if (idx == 0) {
3130 return xegpu::inferInsertStridedSliceSourceLayout(
3131 resLayout, resShape: insertSlice.getDestVectorType().getShape(),
3132 srcShape: insertSlice.getSourceVectorType().getShape());
3133 }
3134 if (idx == 1)
3135 return resLayout;
3136 }
3137
3138 // For vector::Insert Op, infer source layout from result layout using
3139 // shapes.
3140 if (auto insert = dyn_cast<vector::InsertOp>(Val: op)) {
3141 VectorType resVecTy = dyn_cast<VectorType>(Val: insert.getResult().getType());
3142 VectorType valueToStoreTy =
3143 dyn_cast<VectorType>(Val: insert.getValueToStore().getType());
3144
3145 if ((idx == 0) && valueToStoreTy) {
3146 return xegpu::inferInsertSourceLayout(resLayout, resShape: resVecTy.getShape(),
3147 srcShape: valueToStoreTy.getShape());
3148 }
3149 if (idx == 1)
3150 return resLayout;
3151 }
3152
3153 // For vector::Extract Op, infer source layout from result layout using
3154 // shapes.
3155 if (auto extract = dyn_cast<vector::ExtractOp>(Val: op)) {
3156 VectorType srcVecTy = dyn_cast<VectorType>(Val: extract.getSource().getType());
3157 VectorType resVecTy = dyn_cast<VectorType>(Val: extract.getResult().getType());
3158 if (!srcVecTy || !resVecTy)
3159 return nullptr;
3160 return xegpu::inferExtractSourceLayout(resLayout, resShape: resVecTy.getShape(),
3161 srcShape: srcVecTy.getShape());
3162 }
3163
3164 // For vector::TransposeOp, infer source layout from result layout using
3165 // permutation.
3166 if (auto transpose = dyn_cast<vector::TransposeOp>(Val: op)) {
3167 return xegpu::inferTransposeSourceLayout(resLayout,
3168 permutation: transpose.getPermutation());
3169 }
3170
3171 // For vector::BitCastOp, infer source layout from result layout using
3172 // element type bitwidths.
3173 if (auto bitcast = dyn_cast<vector::BitCastOp>(Val: op)) {
3174 int resElemBitWidth =
3175 bitcast.getResultVectorType().getElementType().getIntOrFloatBitWidth();
3176 int srcElemBitWidth =
3177 bitcast.getSourceVectorType().getElementType().getIntOrFloatBitWidth();
3178 return xegpu::inferBitCastSourceLayout(resLayout, resElemTyBitWidth: resElemBitWidth,
3179 srcElemTyBitWidth: srcElemBitWidth);
3180 }
3181
3182 // for vector::interleave
3183 if (auto interleave = dyn_cast<vector::InterleaveOp>(Val: op)) {
3184 return xegpu::inferInterleaveSourceLayout(resLayout);
3185 }
3186
3187 // for vector::deinterleave
3188 if (auto deinterleave = dyn_cast<vector::DeinterleaveOp>(Val: op)) {
3189 return xegpu::inferDeinterleaveSourceLayout(resLayout);
3190 }
3191
3192 // For vector::ExtractStridedSliceOp, simply return result layout
3193 if (dyn_cast<vector::ExtractStridedSliceOp>(Val: op))
3194 return resLayout;
3195
3196 // For elementwise operations, all operands must have the same layout as
3197 // the result.
3198 if (OpTrait::hasElementwiseMappableTraits(op) && op->getNumResults() == 1)
3199 return resLayout;
3200
3201 return nullptr;
3202}
3203
3204// For a loop terminator operand (scf.for's scf.yield, scf.while's
3205// scf.condition), returns the layout of the region iter_arg it forwards into,
3206// which is the authoritative loop-carried layout, or nullptr when that position
3207// was never assigned a layout.
3208static xegpu::DistributeLayoutAttr getLoopCarriedLayoutForYieldOperand(
3209 RegionBranchTerminatorOpInterface terminator, OpOperand &operand) {
3210 auto branch = dyn_cast<RegionBranchOpInterface>(Val: terminator->getParentOp());
3211 if (!branch)
3212 return nullptr;
3213 RegionBranchSuccessorMapping mapping;
3214 branch.getSuccessorOperandInputMapping(mapping,
3215 src: RegionBranchPoint(terminator));
3216 auto it = mapping.find(Val: &operand);
3217 if (it == mapping.end())
3218 return nullptr;
3219 xegpu::DistributeLayoutAttr iterArgLayout;
3220 for (auto arg : llvm::make_isa_range<BlockArgument>(Range&: it->second)) {
3221 xegpu::DistributeLayoutAttr layout = xegpu::getDistributeLayoutAttr(value: arg);
3222 assert((!iterArgLayout || !layout || iterArgLayout.isEqualTo(layout)) &&
3223 "region inputs fed by one terminator operand disagree on layout");
3224 if (!iterArgLayout)
3225 iterArgLayout = layout;
3226 }
3227 return iterArgLayout;
3228}
3229
3230// For the terminator of a region op that carries nothing back into its regions
3231// (scf.if), returns the layout of the parent result the operand feeds.
3232static xegpu::DistributeLayoutAttr getParentResultLayoutForYieldOperand(
3233 RegionBranchTerminatorOpInterface terminator, OpOperand &operand) {
3234 auto branch = dyn_cast<RegionBranchOpInterface>(Val: terminator->getParentOp());
3235 if (!branch)
3236 return nullptr;
3237 RegionBranchSuccessorMapping mapping;
3238 branch.getSuccessorOperandInputMapping(mapping,
3239 src: RegionBranchPoint(terminator));
3240 auto it = mapping.find(Val: &operand);
3241 if (it == mapping.end())
3242 return nullptr;
3243 for (Value input : it->second)
3244 if (auto result = dyn_cast<OpResult>(Val&: input))
3245 return xegpu::getDistributeLayoutAttr(value: result);
3246 return nullptr;
3247}
3248
3249/// Returns the layout required on `operand`: anchor ops report their declared
3250/// per-operand layout directly; non-anchor ops back-derive it from their result
3251/// layout via inferSourceLayoutFromResultForNonAnchorOp.
3252xegpu::DistributeLayoutAttr xegpu::getConsumerLayoutAt(OpOperand &operand) {
3253 Operation *op = operand.getOwner();
3254 // Anchor ops declare the layout they
3255 // require on each operand. Trust that declaration directly so that
3256 // ResolveLayoutConflicts compares producer-vs-declared
3257 if (isa<xegpu::AnchorLayoutInterface>(Val: op))
3258 return xegpu::getDistributeLayoutAttr(opr: operand);
3259 // Region ops with forwarded operands (scf.for's and scf.while's inits) carry
3260 // the required operand layout as the layout_operand_N that
3261 // propagateRegionArgsToInits back-propagated from the region argument. Do not
3262 // re-derive it from that argument here: conflict resolution inserts
3263 // convert_layout ops as it walks, rewriting the argument's uses, so what
3264 // those uses require depends on how far the walk has progressed.
3265 if (isa<RegionBranchOpInterface>(Val: op))
3266 return xegpu::getDistributeLayoutAttr(opr: operand);
3267 // A region terminator requires the layout of the successor input its operand
3268 // feeds: the region iter_arg for a loop, and the parent result for a region
3269 // op with no loop-carried values (scf.if).
3270 if (auto terminator = dyn_cast<RegionBranchTerminatorOpInterface>(Val: op)) {
3271 if (isa<LoopLikeOpInterface>(Val: op->getParentOp()))
3272 return getLoopCarriedLayoutForYieldOperand(terminator, operand);
3273 return getParentResultLayoutForYieldOperand(terminator, operand);
3274 }
3275 // For non-anchor ops, derive the operand layout from the op's result
3276 // layout via op-specific semantics.
3277 xegpu::DistributeLayoutAttr resLayout;
3278 if (op->getNumResults() == 1 || isa<vector::DeinterleaveOp>(Val: op))
3279 resLayout = xegpu::getDistributeLayoutAttr(value: op->getResult(idx: 0));
3280 return inferSourceLayoutFromResultForNonAnchorOp(operand, resLayout);
3281}
3282