1//===- XeGPUPropagateLayout.cpp - XeGPU Layout Propagation ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h"
10#include "mlir/Analysis/DataFlow/SparseAnalysis.h"
11#include "mlir/Analysis/DataFlow/Utils.h"
12#include "mlir/Analysis/DataFlowFramework.h"
13#include "mlir/Dialect/Func/IR/FuncOps.h"
14#include "mlir/Dialect/GPU/IR/GPUDialect.h"
15#include "mlir/Dialect/MemRef/IR/MemRef.h"
16#include "mlir/Dialect/Utils/IndexingUtils.h"
17#include "mlir/Dialect/Vector/IR/VectorOps.h"
18#include "mlir/Dialect/XeGPU/IR/XeGPU.h"
19#include "mlir/Dialect/XeGPU/Transforms/Passes.h"
20#include "mlir/Dialect/XeGPU/Transforms/XeGPULayoutImpl.h"
21#include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
22#include "mlir/Dialect/XeGPU/uArch/uArchCommon.h"
23#include "mlir/IR/Attributes.h"
24#include "mlir/IR/Builders.h"
25#include "mlir/IR/BuiltinAttributes.h"
26#include "mlir/IR/BuiltinTypes.h"
27#include "mlir/IR/Operation.h"
28#include "mlir/IR/Value.h"
29#include "mlir/IR/Visitors.h"
30#include "mlir/Interfaces/ControlFlowInterfaces.h"
31#include "mlir/Interfaces/FunctionInterfaces.h"
32#include "mlir/Interfaces/LoopLikeInterface.h"
33#include "mlir/Support/LLVM.h"
34#include "llvm/ADT/ArrayRef.h"
35#include "llvm/ADT/DenseMap.h"
36#include "llvm/ADT/STLExtras.h"
37#include "llvm/ADT/SetVector.h"
38#include "llvm/ADT/SmallSet.h"
39#include "llvm/ADT/SmallVector.h"
40#include "llvm/ADT/TypeSwitch.h"
41#include "llvm/Support/Casting.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Support/LogicalResult.h"
44#include "llvm/Support/MathExtras.h"
45#include "llvm/Support/raw_ostream.h"
46#include <limits>
47
48namespace mlir {
49namespace xegpu {
50#define GEN_PASS_DEF_XEGPUPROPAGATELAYOUT
51#include "mlir/Dialect/XeGPU/Transforms/Passes.h.inc"
52} // namespace xegpu
53} // namespace mlir
54
55#define DEBUG_TYPE "xegpu-propagate-layout"
56#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE "]: ")
57
58using namespace mlir;
59using namespace mlir::dataflow;
60
61namespace {
62//===----------------------------------------------------------------------===//
63// LayoutInfo
64//===----------------------------------------------------------------------===//
65
66/// Helper class for tracking the analysis state of an mlir value. For layout
67/// propagation, the analysis state is simply the distribution layout of
68/// each value. The distribution layout information is encapsulated using
69/// xegpu::DistributeLayoutAttr class which can hold information about any type
70/// of distribution layout that XeGPU dialect supports. Purpose of this analysis
71/// to propagate some unique distribution layout for each value in the program
72/// starting from a set of anchor operations (like DPAS, StoreNd, etc.). Note
73/// that analysis will reach a fixed point when all values are reached some
74/// layout and, analysis does not try to modify any already assigned layouts.
75///
76/// Given this, LayoutInfo satisifies the following properties:
77/// 1) A LayoutInfo value can be in one of two states - `assigned` or `not
78/// assigned`.
79/// 2) Two LayoutInfo values are equal if they are both not assigned, or both
80/// assigned with the same layout.
81/// 3) The meet operator works as follows:
82/// - If only one side is assigned, return that side.
83/// - If both sides are assigned, prefer the layout demanded by the user
84/// that is nearer to the producer in program order (smaller
85/// `programOrder`); on a tie keep lhs.
86///
87/// The `programOrder` field records the program-order index of the consumer op
88/// that demanded the layout (stamped via
89/// `LayoutInfoPropagation::makeLayoutInfo` from
90/// `LayoutInfoPropagation::currentProgramOrder`). During this backward analysis
91/// a value can be demanded by several users; keeping the nearest one tends to
92/// preserve a consumer's layout as far up the def chain as possible, minimizing
93/// layout conversions. This is a hint, not an optimum. `programOrder` is never
94/// propagated up the chain - each visited op stamps its own index - so it is
95/// excluded from `operator==`.
96
97struct LayoutInfo {
98private:
99 xegpu::DistributeLayoutAttr storage = nullptr;
100 // Program-order index of the consumer op that demanded this layout. Smaller
101 // means nearer to the producer. Unassigned/unknown demands sort last.
102 int64_t programOrder = std::numeric_limits<int64_t>::max();
103
104public:
105 LayoutInfo() = default;
106 LayoutInfo(const xegpu::DistributeLayoutAttr &layout, int64_t programOrder)
107 : storage(layout), programOrder(programOrder) {}
108
109 // Equality by assignment state and, when both assigned, by the layout:
110 // - one assigned, the other not -> not equal;
111 // - both unassigned -> equal;
112 // - both assigned -> equal iff the layouts match.
113 bool operator==(const LayoutInfo &other) const {
114 if (isAssigned() != other.isAssigned())
115 return false;
116 if (!isAssigned())
117 return true;
118 return storage.isEqualTo(other: other.storage);
119 }
120
121 static LayoutInfo meet(const LayoutInfo &lhs, const LayoutInfo &rhs);
122
123 static LayoutInfo join(const LayoutInfo &lhs, const LayoutInfo &rhs);
124
125 void print(raw_ostream &os) const;
126
127 bool isAssigned() const { return storage != nullptr; }
128
129 SmallVector<int> getLaneLayout() const;
130
131 SmallVector<int> getLaneData() const;
132
133 SmallVector<int> getInstData() const;
134
135 SmallVector<int> getSgLayout() const;
136
137 SmallVector<int> getSgData() const;
138
139 SmallVector<int> getOrder() const;
140
141 bool isSliceLayout() const {
142 if (!isAssigned())
143 return false;
144 return isa<xegpu::SliceAttr>(Val: storage);
145 }
146
147 int64_t getRank() const {
148 if (!isAssigned())
149 return -1;
150 return storage.getRank();
151 }
152
153 Attribute get() { return storage; }
154 void set(const xegpu::DistributeLayoutAttr &layout) { storage = layout; }
155};
156
157void LayoutInfo::print(raw_ostream &os) const {
158 if (isAssigned()) {
159 os << storage;
160 } else {
161 os << "Not assigned.";
162 }
163}
164
165LayoutInfo LayoutInfo::meet(const LayoutInfo &lhs, const LayoutInfo &rhs) {
166 if (!lhs.isAssigned())
167 return rhs;
168 if (!rhs.isAssigned())
169 return lhs;
170 // Prefer the demand from the user nearer to the producer in program order.
171 // Distinct users always have distinct indices, so this decides every
172 // real conflict; on a tie (same op, or both unknown) keep lhs.
173 if (rhs.programOrder < lhs.programOrder)
174 return rhs;
175 return lhs;
176}
177
178/// Since this is a backward analysis, join method is not used.
179LayoutInfo LayoutInfo::join(const LayoutInfo &lhs, const LayoutInfo &rhs) {
180 llvm_unreachable("Join should not be triggered by layout propagation.");
181}
182
183//===----------------------------------------------------------------------===//
184// LayoutInfoLattice
185//===----------------------------------------------------------------------===//
186
187/// Lattice holding the LayoutInfo for each value.
188struct LayoutInfoLattice : public Lattice<LayoutInfo> {
189 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LayoutInfoLattice)
190 using Lattice::Lattice;
191};
192
193//===----------------------------------------------------------------------===//
194// LayoutInfoPropagation
195//===----------------------------------------------------------------------===//
196
197/// Backward data flow analysis to propagate the lane_layout and lane_data of
198/// each value in the program. Currently, the layouts for operands DPAS,
199/// StoreNd, and StoreScatter are fixed (known before propagation). Purpose of
200/// this analysis is to propagate those known layouts to all their producers and
201/// (other) consumers.
202class LayoutInfoPropagation
203 : public SparseBackwardDataFlowAnalysis<LayoutInfoLattice> {
204public:
205 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LayoutInfoPropagation)
206
207private:
208 xegpu::LayoutKind layoutKind;
209 unsigned indexBitWidth;
210
211 // The op this analysis runs on; program order is numbered within this scope
212 // only (not the enclosing module, which may be mutated concurrently by the
213 // parallel pass manager running this pass on sibling gpu.modules).
214 Operation *scopeRoot = nullptr;
215
216 // Program-order index of every op, built lazily on first use via a pre-order
217 // walk of `scopeRoot` (matching printed-IR order). Used to tell which
218 // consumer of a value is nearer to its producer.
219 DenseMap<Operation *, int64_t> programOrder;
220 // Returns the program-order index of `op`, populating `programOrder` from
221 // `scopeRoot` on first call.
222 int64_t getProgramOrder(Operation *op);
223
224 int64_t currentProgramOrder = std::numeric_limits<int64_t>::max();
225 LayoutInfo makeLayoutInfo(const xegpu::DistributeLayoutAttr &layout) {
226 return LayoutInfo(layout, currentProgramOrder);
227 }
228
229 void visitDpasOp(xegpu::DpasOp dpas, ArrayRef<LayoutInfoLattice *> operands,
230 ArrayRef<const LayoutInfoLattice *> results);
231
232 void visitDpasMxOp(xegpu::DpasMxOp dpasMx,
233 ArrayRef<LayoutInfoLattice *> operands,
234 ArrayRef<const LayoutInfoLattice *> results);
235
236 void visitStoreNdOp(xegpu::StoreNdOp store,
237 ArrayRef<LayoutInfoLattice *> operands,
238 ArrayRef<const LayoutInfoLattice *> results);
239
240 void visitStoreScatterOp(xegpu::StoreScatterOp storeScatter,
241 ArrayRef<LayoutInfoLattice *> operands,
242 ArrayRef<const LayoutInfoLattice *> results);
243
244 void visitLoadNdOp(xegpu::LoadNdOp load,
245 ArrayRef<LayoutInfoLattice *> operands,
246 ArrayRef<const LayoutInfoLattice *> results);
247
248 void visitLoadGatherOp(xegpu::LoadGatherOp load,
249 ArrayRef<LayoutInfoLattice *> operands,
250 ArrayRef<const LayoutInfoLattice *> results);
251
252 void visitTransposeOp(vector::TransposeOp transpose,
253 ArrayRef<LayoutInfoLattice *> operands,
254 ArrayRef<const LayoutInfoLattice *> results);
255
256 void visitVectorBitcastOp(vector::BitCastOp bitcast,
257 ArrayRef<LayoutInfoLattice *> operands,
258 ArrayRef<const LayoutInfoLattice *> results);
259
260 void visitVectorInterleaveOp(vector::InterleaveOp interleave,
261 ArrayRef<LayoutInfoLattice *> operands,
262 ArrayRef<const LayoutInfoLattice *> results);
263
264 void visitVectorDeinterleaveOp(vector::DeinterleaveOp deinterleave,
265 ArrayRef<LayoutInfoLattice *> operands,
266 ArrayRef<const LayoutInfoLattice *> results);
267
268 void visitPrefetchNdOp(xegpu::PrefetchNdOp prefetch,
269 ArrayRef<LayoutInfoLattice *> operands,
270 ArrayRef<const LayoutInfoLattice *> results);
271
272 void visitVectorMultiReductionOp(vector::MultiDimReductionOp reduction,
273 ArrayRef<LayoutInfoLattice *> operands,
274 ArrayRef<const LayoutInfoLattice *> results);
275
276 void visitVectorReductionOp(vector::ReductionOp reduction,
277 ArrayRef<LayoutInfoLattice *> operands,
278 ArrayRef<const LayoutInfoLattice *> results);
279
280 void visitVectorBroadCastOp(vector::BroadcastOp broadcast,
281 ArrayRef<LayoutInfoLattice *> operands,
282 ArrayRef<const LayoutInfoLattice *> results);
283 void visitShapeCastOp(vector::ShapeCastOp shapeCast,
284 ArrayRef<LayoutInfoLattice *> operands,
285 ArrayRef<const LayoutInfoLattice *> results);
286 void
287 visitInsertStridedSliceOp(vector::InsertStridedSliceOp insertStridedSlice,
288 ArrayRef<LayoutInfoLattice *> operands,
289 ArrayRef<const LayoutInfoLattice *> results);
290
291 void visitLoadMatrixOp(xegpu::LoadMatrixOp load,
292 ArrayRef<LayoutInfoLattice *> operands,
293 ArrayRef<const LayoutInfoLattice *> results);
294
295 void visitStoreMatrixOp(xegpu::StoreMatrixOp store,
296 ArrayRef<LayoutInfoLattice *> operands,
297 ArrayRef<const LayoutInfoLattice *> results);
298
299 void visitLoadGatherOp(xegpu::LoadMatrixOp load,
300 ArrayRef<LayoutInfoLattice *> operands,
301 ArrayRef<const LayoutInfoLattice *> results);
302
303 void visitStoreScatterOp(xegpu::StoreMatrixOp store,
304 ArrayRef<LayoutInfoLattice *> operands,
305 ArrayRef<const LayoutInfoLattice *> results);
306
307 void visitConvertLayoutOp(xegpu::ConvertLayoutOp convertLayout,
308 ArrayRef<LayoutInfoLattice *> operands,
309 ArrayRef<const LayoutInfoLattice *> results);
310
311 bool hasParamsOfLayoutKind(xegpu::DistributeLayoutAttr anchorLayout);
312
313 // Number of subgroups for `op`. Missing count is fatal in subgroup mode.
314 FailureOr<int64_t> getNumSgOrFail(Operation *op, int sgSize,
315 xegpu::DistributeLayoutAttr consumerLayout);
316
317 // Channel to surface hard failures out of the void visit callbacks.
318 bool propagationFailed = false;
319
320 // Reserved for the anchor ops that are the sources of the propagation
321 // (store/dpas), whose layout must be correct.
322 void markFailure(Operation *op, const llvm::Twine &message) {
323 op->emitError(message);
324 propagationFailed = true;
325 }
326
327public:
328 bool hasFailed() const { return propagationFailed; }
329
330 LayoutInfoPropagation(DataFlowSolver &solver,
331 SymbolTableCollection &symbolTable,
332 xegpu::LayoutKind layoutKind, unsigned indexBitWidth,
333 Operation *scopeRoot)
334 : SparseBackwardDataFlowAnalysis(solver, symbolTable),
335 layoutKind(layoutKind), indexBitWidth(indexBitWidth),
336 scopeRoot(scopeRoot) {}
337 using SparseBackwardDataFlowAnalysis::SparseBackwardDataFlowAnalysis;
338
339 LogicalResult
340 visitOperation(Operation *op, ArrayRef<LayoutInfoLattice *> operands,
341 ArrayRef<const LayoutInfoLattice *> results) override;
342
343 void visitBranchOperand(OpOperand &operand) override {};
344
345 void visitCallOperand(OpOperand &operand) override {};
346
347 void
348 visitNonControlFlowArguments(RegionSuccessor &successor,
349 ArrayRef<BlockArgument> arguments) override {};
350
351 void visitExternalCall(CallOpInterface call,
352 ArrayRef<LayoutInfoLattice *> operands,
353 ArrayRef<const LayoutInfoLattice *> results) override {
354 };
355
356 void setToExitState(LayoutInfoLattice *lattice) override {
357 (void)lattice->meet(rhs: LayoutInfo());
358 }
359};
360} // namespace
361
362int64_t LayoutInfoPropagation::getProgramOrder(Operation *op) {
363 auto it = programOrder.find(Val: op);
364 if (it != programOrder.end())
365 return it->second;
366 // First time we number the tree: number every op under the analysis scope in
367 // pre-order (i.e. printed-IR order). Nested ops (e.g. inside an scf.for body)
368 // get an index between their parent and the parent's next sibling, so a use
369 // inside a loop is "nearer" than a use after it. Numbering is confined to
370 // `scopeRoot` (the op this pass runs on) rather than the enclosing module,
371 // which may be mutated concurrently by the parallel pass manager.
372 int64_t counter = 0;
373 scopeRoot->walk<WalkOrder::PreOrder>(
374 callback: [&](Operation *o) { programOrder[o] = counter++; });
375 return programOrder.lookup(Val: op);
376}
377
378LogicalResult LayoutInfoPropagation::visitOperation(
379 Operation *op, ArrayRef<LayoutInfoLattice *> operands,
380 ArrayRef<const LayoutInfoLattice *> results) {
381 // Stamp demands pushed by this op with its program-order index so `meet` can
382 // prefer the nearest consumer.
383 currentProgramOrder = getProgramOrder(op);
384 TypeSwitch<Operation *>(op)
385 .Case(
386 caseFn: [&](xegpu::DpasOp dpasOp) { visitDpasOp(dpas: dpasOp, operands, results); })
387 .Case(caseFn: [&](xegpu::DpasMxOp dpasMxOp) {
388 visitDpasMxOp(dpasMx: dpasMxOp, operands, results);
389 })
390 .Case(caseFn: [&](xegpu::StoreNdOp storeNdOp) {
391 visitStoreNdOp(store: storeNdOp, operands, results);
392 })
393 .Case(caseFn: [&](xegpu::StoreScatterOp storeScatterOp) {
394 visitStoreScatterOp(storeScatter: storeScatterOp, operands, results);
395 })
396 .Case(caseFn: [&](xegpu::LoadNdOp loadNdOp) {
397 visitLoadNdOp(load: loadNdOp, operands, results);
398 })
399 .Case(caseFn: [&](xegpu::LoadGatherOp loadGatherOp) {
400 visitLoadGatherOp(load: loadGatherOp, operands, results);
401 })
402 .Case(caseFn: [&](xegpu::PrefetchNdOp prefetchNdOp) {
403 visitPrefetchNdOp(prefetch: prefetchNdOp, operands, results);
404 })
405 .Case(caseFn: [&](vector::TransposeOp transposeOp) {
406 visitTransposeOp(transpose: transposeOp, operands, results);
407 })
408 .Case(caseFn: [&](vector::BitCastOp bitcastOp) {
409 visitVectorBitcastOp(bitcast: bitcastOp, operands, results);
410 })
411 .Case(caseFn: [&](vector::InterleaveOp interleaveOp) {
412 visitVectorInterleaveOp(interleave: interleaveOp, operands, results);
413 })
414 .Case(caseFn: [&](vector::DeinterleaveOp deinterleaveOp) {
415 visitVectorDeinterleaveOp(deinterleave: deinterleaveOp, operands, results);
416 })
417 .Case(caseFn: [&](vector::MultiDimReductionOp reductionOp) {
418 visitVectorMultiReductionOp(reduction: reductionOp, operands, results);
419 })
420 .Case(caseFn: [&](vector::ReductionOp reductionOp) {
421 visitVectorReductionOp(reduction: reductionOp, operands, results);
422 })
423 .Case(caseFn: [&](vector::BroadcastOp broadcastOp) {
424 visitVectorBroadCastOp(broadcast: broadcastOp, operands, results);
425 })
426 .Case(caseFn: [&](vector::ShapeCastOp shapeCastOp) {
427 visitShapeCastOp(shapeCast: shapeCastOp, operands, results);
428 })
429 .Case(caseFn: [&](vector::InsertStridedSliceOp insertStridedSliceOp) {
430 visitInsertStridedSliceOp(insertStridedSlice: insertStridedSliceOp, operands, results);
431 })
432 .Case(caseFn: [&](xegpu::LoadMatrixOp loadMatrixOp) {
433 visitLoadMatrixOp(load: loadMatrixOp, operands, results);
434 })
435 .Case(caseFn: [&](xegpu::StoreMatrixOp storeMatrixOp) {
436 visitStoreMatrixOp(store: storeMatrixOp, operands, results);
437 })
438 .Case(caseFn: [&](xegpu::ConvertLayoutOp convertLayoutOp) {
439 visitConvertLayoutOp(convertLayout: convertLayoutOp, operands, results);
440 })
441 // All other ops.
442 .Default(defaultFn: [&](Operation *op) {
443 for (const LayoutInfoLattice *resultInfo : results) {
444 if (!resultInfo->getValue().isAssigned())
445 continue;
446 for (auto [operandInfo, operand] :
447 llvm::zip(t&: operands, u: op->getOpOperands())) {
448 // If the operand type is not a vector or tensor descriptor, skip
449 // it.
450 if (!isa<xegpu::TensorDescType, VectorType>(
451 Val: operand.get().getType()))
452 continue;
453 // Propagate the result layout to the operand.
454 meet(lhs: operandInfo, rhs: *resultInfo);
455 }
456 }
457 });
458
459 return success();
460}
461
462bool LayoutInfoPropagation::hasParamsOfLayoutKind(
463 xegpu::DistributeLayoutAttr anchorLayout) {
464 if (anchorLayout == nullptr) {
465 return false;
466 }
467 if (layoutKind == xegpu::LayoutKind::InstData) {
468 return !(anchorLayout.getEffectiveInstDataAsInt().empty());
469 }
470 if (layoutKind == xegpu::LayoutKind::Lane) {
471 return !(anchorLayout.getEffectiveLaneLayoutAsInt().empty() ||
472 anchorLayout.getEffectiveLaneDataAsInt().empty());
473 }
474 if (layoutKind == xegpu::LayoutKind::Subgroup) {
475 return !(anchorLayout.getEffectiveSgLayoutAsInt().empty() ||
476 anchorLayout.getEffectiveSgDataAsInt().empty());
477 }
478 return false;
479}
480
481FailureOr<int64_t> LayoutInfoPropagation::getNumSgOrFail(
482 Operation *op, int sgSize, xegpu::DistributeLayoutAttr consumerLayout) {
483 // The consumer's sg_layout, when present, dictates the count.
484 if (consumerLayout) {
485 auto sgLayout = consumerLayout.getEffectiveSgLayoutAsInt();
486 if (!sgLayout.empty())
487 return llvm::product_of(Range&: sgLayout);
488 }
489 // Otherwise fall back to the kernel's known_block_size.
490 if (FailureOr<int64_t> numSg =
491 xegpu::getNumSubgroupsFromBlockSize(op, subgroupSize: sgSize);
492 succeeded(Result: numSg))
493 return *numSg;
494 // Only subgroup mode needs the count; elsewhere a missing one is benign.
495 if (layoutKind == xegpu::LayoutKind::Subgroup) {
496 markFailure(op, message: "Unable to determine the number of subgroups for the "
497 "operation. Please check @known_block_size is properly "
498 "attached as kernel attributes, with power-of-two "
499 "dimensions covering at least one subgroup.");
500 return failure();
501 }
502 return int64_t{0};
503}
504
505void LayoutInfoPropagation::visitPrefetchNdOp(
506 xegpu::PrefetchNdOp prefetch, ArrayRef<LayoutInfoLattice *> operands,
507 ArrayRef<const LayoutInfoLattice *> results) {
508
509 LayoutInfo prefetchLayout;
510 const auto *uArch = xegpu::uArch::getUArch(archName: getChipStr(op: prefetch).value_or(u: ""));
511 if (!uArch)
512 return;
513 xegpu::DistributeLayoutAttr anchorLayout = prefetch.getLayoutAttr();
514 if (hasParamsOfLayoutKind(anchorLayout)) {
515 prefetchLayout = makeLayoutInfo(layout: anchorLayout);
516 if (layoutKind == xegpu::LayoutKind::InstData) {
517 const auto *uArchInstruction =
518 dyn_cast<xegpu::uArch::Subgroup2DBlockPrefetchInstruction>(
519 Val: uArch->getInstruction(
520 instKind: xegpu::uArch::InstructionKind::Subgroup2DBlockPrefetch));
521 if (!uArchInstruction)
522 return;
523 auto completed = xegpu::completeBlockStoreLaneLayoutFromInstData(
524 specifiedLayout: anchorLayout, elemTy: prefetch.getTensorDescType().getElementType(),
525 uArchInstruction, subgroupSize: uArch->getSubgroupSize());
526 if (!completed) {
527 prefetch.emitWarning(
528 message: "Failed to identify lane layouts for the specified inst_data.");
529 return;
530 }
531 prefetch.setLayoutAttr(*completed);
532 prefetchLayout = makeLayoutInfo(layout: *completed);
533 }
534 } else {
535 auto tdescTy = prefetch.getTensorDescType();
536 auto numSgOrErr =
537 getNumSgOrFail(op: prefetch, sgSize: uArch->getSubgroupSize(), consumerLayout: nullptr);
538 if (failed(Result: numSgOrErr))
539 return;
540
541 auto layoutAttr = xegpu::setupPrefetchNdAnchorLayout(
542 layoutKind, tdescTy, numSg: numSgOrErr.value_or(u: 0), uArch);
543 if (!layoutAttr) {
544 prefetch.emitWarning(
545 message: "Failed to determine required layout for prefetch_nd.");
546 return;
547 }
548 prefetchLayout = makeLayoutInfo(layout: layoutAttr);
549 prefetch.setLayoutAttr(layoutAttr);
550 }
551 // Propagate the layout to the source tensor descriptor.
552 propagateIfChanged(state: operands[0], changed: operands[0]->meet(rhs: prefetchLayout));
553}
554
555void LayoutInfoPropagation::visitVectorMultiReductionOp(
556 vector::MultiDimReductionOp reduction,
557 ArrayRef<LayoutInfoLattice *> operands,
558 ArrayRef<const LayoutInfoLattice *> results) {
559 Type resultTy = reduction.getDestType();
560 // The layout of the result must be present.
561 LayoutInfo resLayoutInfo = results[0]->getValue();
562
563 xegpu::DistributeLayoutAttr consumerLayoutAttr;
564 if (!resultTy.isIntOrFloat()) {
565 if (!resLayoutInfo.isAssigned())
566 return;
567 consumerLayoutAttr =
568 dyn_cast<xegpu::DistributeLayoutAttr>(Val: resLayoutInfo.get());
569 }
570
571 VectorType sourceTy = reduction.getSourceVectorType();
572 SmallVector<int64_t> reductionDims(reduction.getReductionDims());
573
574 const auto *uArch =
575 xegpu::uArch::getUArch(archName: xegpu::getChipStr(op: reduction).value_or(u: ""));
576 if (!uArch)
577 return;
578
579 auto numSgOrErr =
580 getNumSgOrFail(op: reduction, sgSize: uArch->getSubgroupSize(), consumerLayout: consumerLayoutAttr);
581 if (failed(Result: numSgOrErr))
582 return;
583
584 // The result layout represents the layout requirements of the operation.
585 // it is recorded to anchor layout or temporary layout.
586 // it must be honored for current op and may conflict with the layout
587 // propagated from consumer op, the conflict is resolved in later phase by
588 // converting the required result layout to the consumer layout
589 auto requiredResLayoutAttr = xegpu::setupMultiReductionResultLayout(
590 layoutKind, srcVectorTy: sourceTy, consumerLayout: consumerLayoutAttr, reductionDims,
591 numSg: numSgOrErr.value_or(u: 0), uArch);
592
593 xegpu::setTemporaryLayout(operandOrResult: reduction->getResult(idx: 0), layout: requiredResLayoutAttr);
594
595 // derive the source layout from the dominant layout and reduction dims
596 auto srcLayoutAttr = xegpu::inferMultiReductionSourceLayout(
597 resLayout: requiredResLayoutAttr, reduceDims: reductionDims);
598
599 propagateIfChanged(state: operands[0],
600 changed: operands[0]->meet(rhs: makeLayoutInfo(layout: srcLayoutAttr)));
601 // Accumulator should have the same layout as the result.
602 propagateIfChanged(state: operands[1],
603 changed: operands[1]->meet(rhs: makeLayoutInfo(layout: requiredResLayoutAttr)));
604}
605
606void LayoutInfoPropagation::visitVectorReductionOp(
607 vector::ReductionOp reduction, ArrayRef<LayoutInfoLattice *> operands,
608 ArrayRef<const LayoutInfoLattice *> results) {
609
610 VectorType sourceTy = reduction.getSourceVectorType();
611 const auto *uArch =
612 xegpu::uArch::getUArch(archName: xegpu::getChipStr(op: reduction).value_or(u: ""));
613 if (!uArch)
614 return;
615
616 auto requiredResLayoutAttr =
617 xegpu::setupReductionResultLayout(layoutKind, srcVectorTy: sourceTy, uArch);
618 xegpu::setTemporaryLayout(operandOrResult: reduction->getResult(idx: 0), layout: requiredResLayoutAttr);
619
620 auto srcLayoutAttr = xegpu::inferReductionSourceLayout(resLayout: requiredResLayoutAttr);
621 propagateIfChanged(state: operands[0],
622 changed: operands[0]->meet(rhs: makeLayoutInfo(layout: srcLayoutAttr)));
623 if (reduction.getAcc())
624 propagateIfChanged(
625 state: operands[1], changed: operands[1]->meet(rhs: makeLayoutInfo(layout: requiredResLayoutAttr)));
626}
627
628void LayoutInfoPropagation::visitVectorBroadCastOp(
629 vector::BroadcastOp broadcast, ArrayRef<LayoutInfoLattice *> operands,
630 ArrayRef<const LayoutInfoLattice *> results) {
631 // The layout of the result must be present.
632 LayoutInfo resLayoutInfo = results[0]->getValue();
633 if (!resLayoutInfo.isAssigned())
634 return;
635
636 // Only consider vector to vector broadcasts for now.
637 VectorType resultTy = broadcast.getResultVectorType();
638 VectorType sourceTy = dyn_cast<VectorType>(Val: broadcast.getSourceType());
639 // skip layout propagation for non-vector source operand.
640 if (!sourceTy)
641 return;
642
643 auto srcShape = sourceTy.getShape();
644 auto resShape = resultTy.getShape();
645
646 auto resultLayoutAttr =
647 dyn_cast<xegpu::DistributeLayoutAttr>(Val: resLayoutInfo.get());
648
649 xegpu::DistributeLayoutAttr srcLayoutAttr =
650 xegpu::inferBroadcastSourceLayout(resLayout: resultLayoutAttr, resShape, srcShape);
651
652 propagateIfChanged(state: operands[0],
653 changed: operands[0]->meet(rhs: makeLayoutInfo(layout: srcLayoutAttr)));
654}
655
656void LayoutInfoPropagation::visitShapeCastOp(
657 vector::ShapeCastOp shapeCast, ArrayRef<LayoutInfoLattice *> operands,
658 ArrayRef<const LayoutInfoLattice *> results) {
659 // The layout of the result must be present.
660 LayoutInfo resLayoutInfo = results[0]->getValue();
661 if (!resLayoutInfo.isAssigned())
662 return;
663 ArrayRef<int64_t> resShape = shapeCast.getResultVectorType().getShape();
664 ArrayRef<int64_t> srcShape = shapeCast.getSourceVectorType().getShape();
665 auto resultLayoutAttr =
666 dyn_cast<xegpu::DistributeLayoutAttr>(Val: resLayoutInfo.get());
667
668 auto requiredResLayoutAttr = xegpu::setupShapeCastResultLayout(
669 layoutKind, srcVectorTy: shapeCast.getSourceVectorType(),
670 resVectorTy: shapeCast.getResultVectorType(), consumerLayout: resultLayoutAttr);
671 // The consumer layout cannot be expressed on the source: no lane_data makes
672 // each lane's data a contiguous run of the collapsed source dim. shape_cast
673 // is not an anchor op, so warn and leave the value un-laid-out instead of
674 // propagating a layout that would move data between lanes.
675 if (!requiredResLayoutAttr) {
676 shapeCast.emitWarning(message: "Failed to infer source layout for shape_cast; the "
677 "result layout required by its consumers cannot be "
678 "collapsed onto the source shape.");
679 return;
680 }
681 xegpu::setTemporaryLayout(operandOrResult: shapeCast->getResult(idx: 0), layout: requiredResLayoutAttr);
682
683 xegpu::DistributeLayoutAttr srcLayoutAttr = xegpu::inferShapeCastSourceLayout(
684 resLayout: requiredResLayoutAttr, resShape, srcShape);
685 // shape_cast is not an anchor op: another consumer of the source value may
686 // still supply a valid layout, so warn instead of stopping the propagation.
687 if (!srcLayoutAttr) {
688 shapeCast.emitWarning(message: "Failed to infer source layout for shape_cast; "
689 "unsupported shape-cast pattern.");
690 return;
691 }
692
693 propagateIfChanged(state: operands[0],
694 changed: operands[0]->meet(rhs: makeLayoutInfo(layout: srcLayoutAttr)));
695}
696
697/// Set the layouts for DPAS A, B, and C operands.
698void LayoutInfoPropagation::visitDpasOp(
699 xegpu::DpasOp dpas, ArrayRef<LayoutInfoLattice *> operands,
700 ArrayRef<const LayoutInfoLattice *> results) {
701 LayoutInfo dpasALayout;
702 LayoutInfo dpasBLayout;
703 LayoutInfo dpasCDLayout;
704
705 const auto *uArch = xegpu::uArch::getUArch(archName: getChipStr(op: dpas).value_or(u: ""));
706 if (!uArch)
707 return;
708 VectorType aTy = dpas.getLhsType();
709 VectorType bTy = dpas.getRhsType();
710 VectorType cdTy = dpas.getResultType();
711
712 xegpu::DistributeLayoutAttr anchorLayoutCD = dpas.getLayoutCdAttr();
713 if (hasParamsOfLayoutKind(anchorLayout: anchorLayoutCD)) {
714 xegpu::DistributeLayoutAttr anchorLayoutA = dpas.getLayoutAAttr();
715 xegpu::DistributeLayoutAttr anchorLayoutB = dpas.getLayoutBAttr();
716 assert(hasParamsOfLayoutKind(anchorLayoutA) &&
717 "Expected anchor layout for DPAS A operand.");
718 assert(hasParamsOfLayoutKind(anchorLayoutB) &&
719 "Expected anchor layout for DPAS B operand.");
720 dpasALayout = makeLayoutInfo(layout: anchorLayoutA);
721 dpasBLayout = makeLayoutInfo(layout: anchorLayoutB);
722 dpasCDLayout = makeLayoutInfo(layout: anchorLayoutCD);
723 if (layoutKind == xegpu::LayoutKind::InstData) {
724 auto completed = xegpu::completeDpasLaneLayoutFromInstData(
725 aLayout: anchorLayoutA, bLayout: anchorLayoutB, cdLayout: anchorLayoutCD, aTy, bTy, cdTy, uArch);
726 if (!completed) {
727 markFailure(
728 op: dpas,
729 message: "Failed to identify lane layouts for the specified inst_data.");
730 return;
731 }
732 auto [completedA, completedB, completedCD] = *completed;
733 dpas.setLayoutAAttr(completedA);
734 dpas.setLayoutBAttr(completedB);
735 dpas.setLayoutCdAttr(completedCD);
736 dpasALayout = makeLayoutInfo(layout: completedA);
737 dpasBLayout = makeLayoutInfo(layout: completedB);
738 dpasCDLayout = makeLayoutInfo(layout: completedCD);
739 }
740 } else {
741
742 xegpu::DistributeLayoutAttr consumerLayoutAttr = nullptr;
743 xegpu::DistributeLayoutAttr requiredCDLayoutAttr, requiredALayout,
744 requiredBLayout;
745
746 LayoutInfo consumerLayout = results[0]->getValue();
747 if (!consumerLayout.isAssigned())
748 return;
749 consumerLayoutAttr =
750 dyn_cast<xegpu::DistributeLayoutAttr>(Val: consumerLayout.get());
751
752 auto numSgOrErr =
753 getNumSgOrFail(op: dpas, sgSize: uArch->getSubgroupSize(), consumerLayout: consumerLayoutAttr);
754 if (failed(Result: numSgOrErr))
755 return;
756
757 auto layouts =
758 xegpu::setupDpasLayout(layoutKind, aTy, bTy, cdTy, consumerLayout: consumerLayoutAttr,
759 numSg: numSgOrErr.value_or(u: 0), uArch);
760 if (!layouts.has_value()) {
761 markFailure(op: dpas,
762 message: "Failed to determine required layouts for DPAS operands.");
763 return;
764 }
765
766 std::tie(args&: requiredALayout, args&: requiredBLayout, args&: requiredCDLayoutAttr) = *layouts;
767
768 dpas.setLayoutAAttr(requiredALayout);
769 dpas.setLayoutBAttr(requiredBLayout);
770 dpas.setLayoutCdAttr(requiredCDLayoutAttr);
771 dpasALayout = makeLayoutInfo(layout: requiredALayout);
772 dpasBLayout = makeLayoutInfo(layout: requiredBLayout);
773 dpasCDLayout = makeLayoutInfo(layout: requiredCDLayoutAttr);
774 }
775 propagateIfChanged(state: operands[0], changed: operands[0]->meet(rhs: dpasALayout));
776 propagateIfChanged(state: operands[1], changed: operands[1]->meet(rhs: dpasBLayout));
777 if (operands.size() > 2)
778 propagateIfChanged(state: operands[2], changed: operands[2]->meet(rhs: dpasCDLayout));
779}
780
781/// Propagate layout for DpasMxOp operands using the layout attributes.
782/// DpasMxOp has operands: a, b, acc (optional), scale_a (optional), scale_b
783/// (optional)
784void LayoutInfoPropagation::visitDpasMxOp(
785 xegpu::DpasMxOp dpasMx, ArrayRef<LayoutInfoLattice *> operands,
786 ArrayRef<const LayoutInfoLattice *> results) {
787
788 // Initialize layout variables
789 LayoutInfo dpasMxALayout, dpasMxBLayout, dpasMxCDLayout;
790 LayoutInfo dpasMxAScaleLayout, dpasMxBScaleLayout;
791
792 // Get existing layout attributes from the operation
793 xegpu::DistributeLayoutAttr anchorLayoutA = dpasMx.getLayoutAAttr();
794 xegpu::DistributeLayoutAttr anchorLayoutB = dpasMx.getLayoutBAttr();
795 xegpu::DistributeLayoutAttr anchorLayoutCD = dpasMx.getLayoutCdAttr();
796
797 const auto *uArch = xegpu::uArch::getUArch(archName: getChipStr(op: dpasMx).value_or(u: ""));
798 if (!uArch)
799 return;
800
801 VectorType aTy = dpasMx.getAType();
802 VectorType bTy = dpasMx.getBType();
803 VectorType cdTy = dpasMx.getResultType();
804
805 // Get scale types if present
806 VectorType aScaleTy;
807 VectorType bScaleTy;
808 Value scaleA = dpasMx.getScaleA();
809 Value scaleB = dpasMx.getScaleB();
810 if (scaleA)
811 aScaleTy = dyn_cast<VectorType>(Val: scaleA.getType());
812 if (scaleB)
813 bScaleTy = dyn_cast<VectorType>(Val: scaleB.getType());
814
815 // Check if all layouts are already set
816 if (anchorLayoutA && anchorLayoutB && anchorLayoutCD &&
817 hasParamsOfLayoutKind(anchorLayout: anchorLayoutA) &&
818 hasParamsOfLayoutKind(anchorLayout: anchorLayoutB) &&
819 hasParamsOfLayoutKind(anchorLayout: anchorLayoutCD)) {
820 dpasMxALayout = makeLayoutInfo(layout: anchorLayoutA);
821 dpasMxBLayout = makeLayoutInfo(layout: anchorLayoutB);
822 dpasMxCDLayout = makeLayoutInfo(layout: anchorLayoutCD);
823
824 // Get scale layouts if available
825 xegpu::DistributeLayoutAttr anchorLayoutAScale =
826 dpasMx.getLayoutAScaleAttr();
827 xegpu::DistributeLayoutAttr anchorLayoutBScale =
828 dpasMx.getLayoutBScaleAttr();
829 if (anchorLayoutAScale)
830 dpasMxAScaleLayout = makeLayoutInfo(layout: anchorLayoutAScale);
831 if (anchorLayoutBScale)
832 dpasMxBScaleLayout = makeLayoutInfo(layout: anchorLayoutBScale);
833
834 if (layoutKind == xegpu::LayoutKind::InstData) {
835 auto completed = xegpu::completeDpasMxLaneLayoutFromInstData(
836 aLayout: anchorLayoutA, bLayout: anchorLayoutB, cdLayout: anchorLayoutCD, aTy, bTy, cdTy,
837 aScaleTy, bScaleTy, uArch);
838 if (!completed) {
839 markFailure(
840 op: dpasMx,
841 message: "Failed to identify lane layouts for the specified inst_data.");
842 return;
843 }
844 auto [completedA, completedB, completedCD, completedAScale,
845 completedBScale] = *completed;
846 dpasMx.setLayoutAAttr(completedA);
847 dpasMx.setLayoutBAttr(completedB);
848 dpasMx.setLayoutCdAttr(completedCD);
849 dpasMxALayout = makeLayoutInfo(layout: completedA);
850 dpasMxBLayout = makeLayoutInfo(layout: completedB);
851 dpasMxCDLayout = makeLayoutInfo(layout: completedCD);
852 if (completedAScale) {
853 dpasMx.setLayoutAScaleAttr(completedAScale);
854 dpasMxAScaleLayout = makeLayoutInfo(layout: completedAScale);
855 }
856 if (completedBScale) {
857 dpasMx.setLayoutBScaleAttr(completedBScale);
858 dpasMxBScaleLayout = makeLayoutInfo(layout: completedBScale);
859 }
860 }
861 } else {
862 xegpu::DistributeLayoutAttr consumerLayoutAttr = nullptr;
863 xegpu::DistributeLayoutAttr requiredCDLayoutAttr, requiredALayout,
864 requiredBLayout, requiredAScaleLayout, requiredBScaleLayout;
865
866 LayoutInfo consumerLayout = results[0]->getValue();
867 if (!consumerLayout.isAssigned())
868 return;
869 consumerLayoutAttr =
870 dyn_cast<xegpu::DistributeLayoutAttr>(Val: consumerLayout.get());
871
872 auto numSgOrErr =
873 getNumSgOrFail(op: dpasMx, sgSize: uArch->getSubgroupSize(), consumerLayout: consumerLayoutAttr);
874 if (failed(Result: numSgOrErr))
875 return;
876
877 auto layouts = xegpu::setupDpasMxLayout(
878 layoutKind, aTy, bTy, cdTy, aScaleTy, bScaleTy, consumerLayout: consumerLayoutAttr,
879 numSg: numSgOrErr.value_or(u: 0), uArch);
880 if (!layouts.has_value()) {
881 markFailure(op: dpasMx,
882 message: "Failed to determine required layouts for DPAS_MX operands.");
883 return;
884 }
885
886 std::tie(args&: requiredALayout, args&: requiredBLayout, args&: requiredCDLayoutAttr,
887 args&: requiredAScaleLayout, args&: requiredBScaleLayout) = *layouts;
888
889 dpasMx.setLayoutAAttr(requiredALayout);
890 dpasMx.setLayoutBAttr(requiredBLayout);
891 dpasMx.setLayoutCdAttr(requiredCDLayoutAttr);
892 if (requiredAScaleLayout)
893 dpasMx.setLayoutAScaleAttr(requiredAScaleLayout);
894 if (requiredBScaleLayout)
895 dpasMx.setLayoutBScaleAttr(requiredBScaleLayout);
896
897 dpasMxALayout = makeLayoutInfo(layout: requiredALayout);
898 dpasMxBLayout = makeLayoutInfo(layout: requiredBLayout);
899 dpasMxCDLayout = makeLayoutInfo(layout: requiredCDLayoutAttr);
900 if (requiredAScaleLayout)
901 dpasMxAScaleLayout = makeLayoutInfo(layout: requiredAScaleLayout);
902 if (requiredBScaleLayout)
903 dpasMxBScaleLayout = makeLayoutInfo(layout: requiredBScaleLayout);
904 }
905
906 // Propagate layouts to operands. Because acc, scale_a, scale_b are all
907 // optional (AttrSizedOperandSegments), the index of each present operand in
908 // `operands` depends on which optionals are actually supplied. Use the
909 // op's accessors to determine the correct positional index.
910 propagateIfChanged(state: operands[0], changed: operands[0]->meet(rhs: dpasMxALayout));
911 propagateIfChanged(state: operands[1], changed: operands[1]->meet(rhs: dpasMxBLayout));
912 unsigned idx = 2;
913 if (dpasMx.getAcc()) {
914 propagateIfChanged(state: operands[idx], changed: operands[idx]->meet(rhs: dpasMxCDLayout));
915 ++idx;
916 }
917 if (dpasMx.getScaleA()) {
918 if (dpasMxAScaleLayout.isAssigned())
919 propagateIfChanged(state: operands[idx],
920 changed: operands[idx]->meet(rhs: dpasMxAScaleLayout));
921 ++idx;
922 }
923 if (dpasMx.getScaleB()) {
924 if (dpasMxBScaleLayout.isAssigned())
925 propagateIfChanged(state: operands[idx],
926 changed: operands[idx]->meet(rhs: dpasMxBScaleLayout));
927 ++idx;
928 }
929}
930
931/// Set the layout for the value and tensor descriptor operands in StoreNdOp.
932void LayoutInfoPropagation::visitStoreNdOp(
933 xegpu::StoreNdOp store, ArrayRef<LayoutInfoLattice *> operands,
934 ArrayRef<const LayoutInfoLattice *> results) {
935 LayoutInfo storeLayout;
936 const auto *uArch = xegpu::uArch::getUArch(archName: getChipStr(op: store).value_or(u: ""));
937 if (!uArch)
938 return;
939 xegpu::DistributeLayoutAttr anchorLayout = store.getLayoutAttr();
940 if (hasParamsOfLayoutKind(anchorLayout)) {
941 storeLayout = makeLayoutInfo(layout: anchorLayout);
942 if (layoutKind == xegpu::LayoutKind::InstData) {
943
944 const auto *uArchInstruction =
945 dyn_cast<xegpu::uArch::Subgroup2DBlockStoreInstruction>(
946 Val: uArch->getInstruction(
947 instKind: xegpu::uArch::InstructionKind::Subgroup2DBlockStore));
948 if (!uArchInstruction)
949 return;
950 auto completed = xegpu::completeBlockStoreLaneLayoutFromInstData(
951 specifiedLayout: anchorLayout, elemTy: store.getValueType().getElementType(), uArchInstruction,
952 subgroupSize: uArch->getSubgroupSize());
953 if (!completed) {
954 markFailure(
955 op: store,
956 message: "Failed to identify lane layouts for the specified inst_data.");
957 return;
958 }
959 store.setLayoutAttr(*completed);
960 storeLayout = makeLayoutInfo(layout: *completed);
961 }
962 } else {
963 auto numSgOrErr = getNumSgOrFail(op: store, sgSize: uArch->getSubgroupSize(), consumerLayout: nullptr);
964 if (failed(Result: numSgOrErr))
965 return;
966
967 auto layoutAttr = xegpu::setupStoreNdAnchorLayout(
968 layoutKind, vectorTy: store.getValueType(), numSg: numSgOrErr.value_or(u: 0), uArch);
969 if (!layoutAttr) {
970 markFailure(op: store, message: "Failed to determine required layout for store_nd.");
971 return;
972 }
973 storeLayout = makeLayoutInfo(layout: layoutAttr);
974 store.setLayoutAttr(layoutAttr);
975 }
976 // Propagate the layout to the value operand.
977 // Both operands should have the same layout
978 for (LayoutInfoLattice *operand : operands)
979 propagateIfChanged(state: operand, changed: operand->meet(rhs: storeLayout));
980}
981
982/// Propagate the layout of the value to the tensor descriptor operand in
983/// LoadNdOp.
984void LayoutInfoPropagation::visitLoadNdOp(
985 xegpu::LoadNdOp load, ArrayRef<LayoutInfoLattice *> operands,
986 ArrayRef<const LayoutInfoLattice *> results) {
987 LayoutInfo loadLayout;
988
989 const auto *uArch = xegpu::uArch::getUArch(archName: getChipStr(op: load).value_or(u: ""));
990 if (!uArch)
991 return;
992 LayoutInfo valueLayout = results[0]->getValue();
993 if (!valueLayout.isAssigned())
994 return;
995 auto consumerLayoutAttr =
996 dyn_cast<xegpu::DistributeLayoutAttr>(Val: valueLayout.get());
997 xegpu::DistributeLayoutAttr anchorLayout = load.getLayoutAttr();
998 if (hasParamsOfLayoutKind(anchorLayout)) {
999 loadLayout = makeLayoutInfo(layout: anchorLayout);
1000 if (layoutKind == xegpu::LayoutKind::InstData &&
1001 !consumerLayoutAttr.getEffectiveLaneLayoutAsInt().empty()) {
1002 const auto *uArchInstruction =
1003 dyn_cast<xegpu::uArch::Subgroup2DBlockLoadInstruction>(
1004 Val: uArch->getInstruction(
1005 instKind: xegpu::uArch::InstructionKind::Subgroup2DBlockLoad));
1006 if (!uArchInstruction)
1007 return;
1008 auto completed = xegpu::completeBlockLoadLaneLayoutFromInstData(
1009 specifiedLayout: anchorLayout, consumerLayout: consumerLayoutAttr, elemTy: load.getType().getElementType(),
1010 uArchInstruction, subgroupSize: uArch->getSubgroupSize());
1011 if (!completed) {
1012 load.emitWarning(
1013 message: "Failed to identify lane layouts for the specified inst_data.");
1014 return;
1015 }
1016 load.setLayoutAttr(*completed);
1017 loadLayout = makeLayoutInfo(layout: *completed);
1018 }
1019 } else {
1020 auto numSgOrErr =
1021 getNumSgOrFail(op: load, sgSize: uArch->getSubgroupSize(), consumerLayout: consumerLayoutAttr);
1022 if (failed(Result: numSgOrErr))
1023 return;
1024 auto layoutAttr = xegpu::setupLoadNdAnchorLayout(
1025 layoutKind, vectorTy: load.getType(), consumerLayout: consumerLayoutAttr, numSg: numSgOrErr.value_or(u: 0),
1026 uArch);
1027 if (!layoutAttr) {
1028 load.emitWarning(message: "Failed to determine required layout for load_nd.");
1029 return;
1030 }
1031 loadLayout = makeLayoutInfo(layout: layoutAttr);
1032 load.setLayoutAttr(layoutAttr);
1033 }
1034 // Propagate the new layout to the tensor descriptor operand.
1035 propagateIfChanged(state: operands[0], changed: operands[0]->meet(rhs: loadLayout));
1036}
1037
1038/// Propagate the layout of the value to the tensor descriptor operand in
1039/// ConvertLayoutOp.
1040void LayoutInfoPropagation::visitConvertLayoutOp(
1041 xegpu::ConvertLayoutOp convert, ArrayRef<LayoutInfoLattice *> operands,
1042 ArrayRef<const LayoutInfoLattice *> results) {
1043
1044 LayoutInfo resultLayout = results[0]->getValue();
1045
1046 // TODO: fix if one of the layouts is a slice layout
1047 auto targetLayoutAttr =
1048 dyn_cast<xegpu::LayoutAttr>(Val: convert.getTargetLayoutAttr());
1049 // input_layout is optional, so it may be null.
1050 auto inputLayoutAttr =
1051 dyn_cast_if_present<xegpu::LayoutAttr>(Val: convert.getInputLayoutAttr());
1052
1053 // The result's propagated layout is authoritative for the converted value.
1054 // Fill the lane_layout / lane_data / order parameters the target_layout is
1055 // missing from it (sg_layout / sg_data / inst_data are left as-is), so the
1056 // target stays consistent with what is actually propagated downstream.
1057 auto resultLayoutAttr = resultLayout.isAssigned()
1058 ? dyn_cast<xegpu::LayoutAttr>(Val: resultLayout.get())
1059 : nullptr;
1060 if (resultLayoutAttr && targetLayoutAttr) {
1061 if (layoutKind == xegpu::LayoutKind::InstData &&
1062 !targetLayoutAttr.getLaneLayout()) {
1063 targetLayoutAttr = xegpu::LayoutAttr::get(
1064 context: convert.getContext(), sg_layout: targetLayoutAttr.getSgLayout(),
1065 sg_data: targetLayoutAttr.getSgData(), inst_data: targetLayoutAttr.getInstData(),
1066 lane_layout: resultLayoutAttr.getLaneLayout(), lane_data: resultLayoutAttr.getLaneData(),
1067 order: resultLayoutAttr.getOrder());
1068 convert.setTargetLayoutAttr(targetLayoutAttr);
1069 }
1070 }
1071
1072 // Fill only the lane_layout / lane_data / order parameters the input_layout
1073 // is missing from the target_layout (sg_layout / sg_data / inst_data are left
1074 // as-is), so the producer side receives a fully-populated lane layout.
1075 if (inputLayoutAttr && targetLayoutAttr) {
1076 if (layoutKind == xegpu::LayoutKind::InstData &&
1077 !inputLayoutAttr.getLaneLayout()) {
1078 auto merged = xegpu::LayoutAttr::get(
1079 context: convert.getContext(), sg_layout: inputLayoutAttr.getSgLayout(),
1080 sg_data: inputLayoutAttr.getSgData(), inst_data: inputLayoutAttr.getInstData(),
1081 lane_layout: targetLayoutAttr.getLaneLayout(), lane_data: targetLayoutAttr.getLaneData(),
1082 order: targetLayoutAttr.getOrder());
1083 convert.setInputLayoutAttr(merged);
1084 }
1085 }
1086
1087 xegpu::DistributeLayoutAttr anchorLayout = convert.getEffectiveInputLayout();
1088 LayoutInfo convertLayout = makeLayoutInfo(layout: anchorLayout);
1089 // Propagate the new layout to the tensor descriptor operand.
1090 propagateIfChanged(state: operands[0], changed: operands[0]->meet(rhs: convertLayout));
1091}
1092
1093/// For vector::TransposeOp, the layout of the result is transposed and
1094/// propagated to the operand.
1095void LayoutInfoPropagation::visitTransposeOp(
1096 vector::TransposeOp transpose, ArrayRef<LayoutInfoLattice *> operands,
1097 ArrayRef<const LayoutInfoLattice *> results) {
1098 // Need the layout of transpose result to propagate to the operands.
1099 LayoutInfo resultLayout = results[0]->getValue();
1100 if (!resultLayout.isAssigned())
1101 return;
1102
1103 auto consumerLayoutAttr =
1104 dyn_cast<xegpu::DistributeLayoutAttr>(Val: resultLayout.get());
1105 auto srcLayoutAttr = xegpu::inferTransposeSourceLayout(
1106 resLayout: consumerLayoutAttr, permutation: transpose.getPermutation());
1107
1108 // Propagate the new layout to the vector operand.
1109 propagateIfChanged(state: operands[0],
1110 changed: operands[0]->meet(rhs: makeLayoutInfo(layout: srcLayoutAttr)));
1111}
1112
1113/// For vector::BitCastOp, the lane_data of the source layout is changed based
1114/// on the bit width of the source and result types.
1115void LayoutInfoPropagation::visitVectorBitcastOp(
1116 vector::BitCastOp bitcast, ArrayRef<LayoutInfoLattice *> operands,
1117 ArrayRef<const LayoutInfoLattice *> results) {
1118 // Need the layout of bitcast result to propagate to the operands.
1119 LayoutInfo resLayoutInfo = results[0]->getValue();
1120 if (!resLayoutInfo.isAssigned())
1121 return;
1122
1123 auto srcVecType = bitcast.getSourceVectorType();
1124 auto resVecType = bitcast.getResultVectorType();
1125
1126 auto consumerLayoutAttr =
1127 dyn_cast<xegpu::DistributeLayoutAttr>(Val: resLayoutInfo.get());
1128 const auto *uArch =
1129 xegpu::uArch::getUArch(archName: xegpu::getChipStr(op: bitcast).value_or(u: ""));
1130 if (!uArch)
1131 return;
1132 auto requiredResLayoutAttr = setupBitCastResultLayout(
1133 layoutKind, srcVectorTy: srcVecType, resVectorTy: resVecType, consumerLayout: consumerLayoutAttr, uArch);
1134
1135 xegpu::setTemporaryLayout(operandOrResult: bitcast->getResult(idx: 0), layout: requiredResLayoutAttr);
1136
1137 int inElemTyBitWidth = srcVecType.getElementType().getIntOrFloatBitWidth();
1138 int outElemTyBitWidth = resVecType.getElementType().getIntOrFloatBitWidth();
1139
1140 // derive the source layout from the dominant layout and reduction dims
1141 auto srcLayoutAttr = xegpu::inferBitCastSourceLayout(
1142 resLayout: requiredResLayoutAttr, resElemTyBitWidth: outElemTyBitWidth, srcElemTyBitWidth: inElemTyBitWidth);
1143
1144 propagateIfChanged(state: operands[0],
1145 changed: operands[0]->meet(rhs: makeLayoutInfo(layout: srcLayoutAttr)));
1146}
1147
1148/// For vector::InterleaveOp, the result has double the innermost dimension
1149/// size compared to each source operand. The layout is propagated from result
1150/// to sources, adjusting for the 2x size increase.
1151void LayoutInfoPropagation::visitVectorInterleaveOp(
1152 vector::InterleaveOp interleave, ArrayRef<LayoutInfoLattice *> operands,
1153 ArrayRef<const LayoutInfoLattice *> results) {
1154 // Need the layout of interleave result to propagate to the operands.
1155 LayoutInfo resLayoutInfo = results[0]->getValue();
1156 if (!resLayoutInfo.isAssigned())
1157 return;
1158
1159 auto srcVecType = interleave.getSourceVectorType();
1160 auto resVecType = interleave.getResultVectorType();
1161
1162 auto consumerLayoutAttr =
1163 dyn_cast<xegpu::DistributeLayoutAttr>(Val: resLayoutInfo.get());
1164 const auto *uArch =
1165 xegpu::uArch::getUArch(archName: xegpu::getChipStr(op: interleave).value_or(u: ""));
1166 if (!uArch)
1167 return;
1168
1169 // Setup the result layout to ensure the source layout can be safely derived
1170 auto requiredResLayoutAttr = setupInterleaveResultLayout(
1171 layoutKind, srcVectorTy: srcVecType, resVectorTy: resVecType, consumerLayout: consumerLayoutAttr, uArch);
1172
1173 xegpu::setTemporaryLayout(operandOrResult: interleave->getResult(idx: 0), layout: requiredResLayoutAttr);
1174
1175 // Derive the source layout from the result layout (halve the innermost dim)
1176 auto srcLayoutAttr =
1177 xegpu::inferInterleaveSourceLayout(resLayout: requiredResLayoutAttr);
1178
1179 // Both operands (lhs and rhs) get the same source layout
1180 propagateIfChanged(state: operands[0],
1181 changed: operands[0]->meet(rhs: makeLayoutInfo(layout: srcLayoutAttr)));
1182 propagateIfChanged(state: operands[1],
1183 changed: operands[1]->meet(rhs: makeLayoutInfo(layout: srcLayoutAttr)));
1184}
1185
1186/// For vector::DeinterleaveOp, the source has double the innermost dimension
1187/// size compared to each result. The layout is propagated from results to
1188/// source, adjusting for the 2x size decrease in results.
1189void LayoutInfoPropagation::visitVectorDeinterleaveOp(
1190 vector::DeinterleaveOp deinterleave, ArrayRef<LayoutInfoLattice *> operands,
1191 ArrayRef<const LayoutInfoLattice *> results) {
1192 // Need the layout of deinterleave results to propagate to the operand.
1193 // Use the first result's layout (both results should have the same layout)
1194 LayoutInfo resLayoutInfo = results[0]->getValue();
1195 if (!resLayoutInfo.isAssigned())
1196 return;
1197
1198 auto consumerLayoutAttr =
1199 dyn_cast<xegpu::DistributeLayoutAttr>(Val: resLayoutInfo.get());
1200
1201 // Derive the source layout from the result layout (double the innermost
1202 // dim) No setup function needed - just infer directly
1203 auto srcLayoutAttr = xegpu::inferDeinterleaveSourceLayout(resLayout: consumerLayoutAttr);
1204
1205 propagateIfChanged(state: operands[0],
1206 changed: operands[0]->meet(rhs: makeLayoutInfo(layout: srcLayoutAttr)));
1207}
1208
1209void LayoutInfoPropagation::visitInsertStridedSliceOp(
1210 vector::InsertStridedSliceOp insertStridedSlice,
1211 ArrayRef<LayoutInfoLattice *> operands,
1212 ArrayRef<const LayoutInfoLattice *> results) {
1213 // The layout of the result must be present.
1214 LayoutInfo resLayoutInfo = results[0]->getValue();
1215 if (!resLayoutInfo.isAssigned())
1216 return;
1217
1218 auto srcVecType = insertStridedSlice.getSourceVectorType();
1219 auto resVecType = insertStridedSlice.getDestVectorType();
1220
1221 auto consumerLayoutAttr =
1222 dyn_cast<xegpu::DistributeLayoutAttr>(Val: resLayoutInfo.get());
1223 const auto *uArch = xegpu::uArch::getUArch(
1224 archName: xegpu::getChipStr(op: insertStridedSlice).value_or(u: ""));
1225 if (!uArch)
1226 return;
1227
1228 auto requiredResLayoutAttr = xegpu::setupInsertStridedSliceResultLayout(
1229 layoutKind, srcVectorTy: srcVecType, resVectorTy: resVecType, consumerLayout: consumerLayoutAttr, uArch);
1230 xegpu::setTemporaryLayout(operandOrResult: insertStridedSlice->getResult(idx: 0),
1231 layout: requiredResLayoutAttr);
1232
1233 auto srcLayoutAttr = xegpu::inferInsertStridedSliceSourceLayout(
1234 resLayout: requiredResLayoutAttr, resShape: resVecType.getShape(), srcShape: srcVecType.getShape());
1235 propagateIfChanged(state: operands[0],
1236 changed: operands[0]->meet(rhs: makeLayoutInfo(layout: srcLayoutAttr)));
1237 propagateIfChanged(state: operands[1],
1238 changed: operands[1]->meet(rhs: makeLayoutInfo(layout: requiredResLayoutAttr)));
1239}
1240
1241/// Propagate the layout of the result to the tensor descriptor, mask and
1242/// offset operands in LoadGatherOp.
1243void LayoutInfoPropagation::visitLoadGatherOp(
1244 xegpu::LoadGatherOp load, ArrayRef<LayoutInfoLattice *> operands,
1245 ArrayRef<const LayoutInfoLattice *> results) {
1246 xegpu::DistributeLayoutAttr requiredAnchorLayoutAttr;
1247 xegpu::DistributeLayoutAttr anchorLayoutAttr = load.getLayoutAttr();
1248 const auto *uArch = xegpu::uArch::getUArch(archName: getChipStr(op: load).value_or(u: ""));
1249 if (!uArch)
1250 return;
1251 VectorType resVecTy = load.getValueType();
1252 // `contiguity` says how many neighbouring elements one lane may take in a
1253 // single access. Absent, only one element per lane is safe.
1254 int contigChunkSize = load.getContiguity().value_or(u: 1);
1255
1256 LayoutInfo resLayoutInfo = results[0]->getValue();
1257 if (!resLayoutInfo.isAssigned())
1258 return;
1259 auto consumerLayoutAttr =
1260 dyn_cast<xegpu::DistributeLayoutAttr>(Val: resLayoutInfo.get());
1261
1262 if (hasParamsOfLayoutKind(anchorLayout: anchorLayoutAttr)) {
1263 requiredAnchorLayoutAttr = anchorLayoutAttr;
1264 if (layoutKind == xegpu::LayoutKind::InstData &&
1265 !consumerLayoutAttr.getEffectiveLaneLayoutAsInt().empty()) {
1266 const auto uArchInstruction =
1267 dyn_cast<xegpu::uArch::LoadGatherInstruction>(
1268 Val: uArch->getInstruction(instKind: xegpu::uArch::InstructionKind::LoadGather));
1269 if (!uArchInstruction)
1270 return;
1271 auto completed = xegpu::completeScatterLoadLaneLayoutFromInstData(
1272 userSpecifiedLayout: anchorLayoutAttr, consumerLayout: consumerLayoutAttr, elemTy: resVecTy.getElementType(),
1273 uArchInstruction, subgroupSize: uArch->getSubgroupSize());
1274 if (!completed) {
1275 load.emitWarning(
1276 message: "Failed to identify lane layouts for the specified inst_data.");
1277 return;
1278 }
1279 requiredAnchorLayoutAttr = *completed;
1280 load.setLayoutAttr(requiredAnchorLayoutAttr);
1281 }
1282 } else {
1283 if (!resVecTy) {
1284 load.emitWarning(message: "Not propagating, non-vector payload supplied.");
1285 return;
1286 }
1287 requiredAnchorLayoutAttr = xegpu::setupLoadGatherAnchorLayout(
1288 layoutKind, vectorTy: resVecTy, contigChunkSize, consumerLayout: consumerLayoutAttr, uArch);
1289 load.setLayoutAttr(requiredAnchorLayoutAttr);
1290 }
1291
1292 // The mask and offset operands share the value's anchor layout.
1293 LayoutInfo maskLayoutInfo = makeLayoutInfo(layout: requiredAnchorLayoutAttr);
1294 auto loadLayoutInfo = makeLayoutInfo(layout: requiredAnchorLayoutAttr);
1295
1296 // Propagate the new layout to the tensor descriptor operand.
1297 if (isa<xegpu::TensorDescType>(Val: load.getSourceType()))
1298 propagateIfChanged(state: operands[0], changed: operands[0]->meet(rhs: loadLayoutInfo));
1299 // Propagate the new layout to the offset and mask operands.
1300 propagateIfChanged(state: operands[1], changed: operands[1]->meet(rhs: maskLayoutInfo));
1301 propagateIfChanged(state: operands[2], changed: operands[2]->meet(rhs: maskLayoutInfo));
1302}
1303
1304/// Set the layout for the value, tensor descriptor, offset and mask operands
1305/// in the StoreScatterOp.
1306void LayoutInfoPropagation::visitStoreScatterOp(
1307 xegpu::StoreScatterOp storeScatter, ArrayRef<LayoutInfoLattice *> operands,
1308 ArrayRef<const LayoutInfoLattice *> results) {
1309
1310 xegpu::DistributeLayoutAttr requiredAnchorLayoutAttr;
1311 xegpu::DistributeLayoutAttr anchorLayoutAttr = storeScatter.getLayoutAttr();
1312 const auto *uArch =
1313 xegpu::uArch::getUArch(archName: getChipStr(op: storeScatter).value_or(u: ""));
1314 if (!uArch)
1315 return;
1316 VectorType srcVecTy = storeScatter.getValueType();
1317 // `contiguity` says how many neighbouring elements one lane may write in a
1318 // single access. Absent, only one element per lane is safe.
1319 int contigChunkSize = storeScatter.getContiguity().value_or(u: 1);
1320
1321 if (hasParamsOfLayoutKind(anchorLayout: anchorLayoutAttr)) {
1322 requiredAnchorLayoutAttr = anchorLayoutAttr;
1323 if (layoutKind == xegpu::LayoutKind::InstData) {
1324 const auto uArchInstruction =
1325 dyn_cast<xegpu::uArch::StoreScatterInstruction>(Val: uArch->getInstruction(
1326 instKind: xegpu::uArch::InstructionKind::StoreScatter));
1327 if (!uArchInstruction)
1328 return;
1329 auto completed = xegpu::completeScatterStoreLaneLayoutFromInstData(
1330 specifiedLayout: anchorLayoutAttr, elemTy: srcVecTy.getElementType(), uArchInstruction,
1331 subgroupSize: uArch->getSubgroupSize());
1332 if (!completed) {
1333 markFailure(
1334 op: storeScatter,
1335 message: "Failed to identify lane layouts for the specified inst_data.");
1336 return;
1337 }
1338 requiredAnchorLayoutAttr = *completed;
1339 storeScatter.setLayoutAttr(requiredAnchorLayoutAttr);
1340 }
1341 } else {
1342 if (!srcVecTy) {
1343 storeScatter.emitWarning(message: "Not propagating, non-vector payload supplied.");
1344 return;
1345 }
1346 auto numSgOrErr =
1347 getNumSgOrFail(op: storeScatter, sgSize: uArch->getSubgroupSize(), consumerLayout: nullptr);
1348 if (failed(Result: numSgOrErr))
1349 return;
1350 requiredAnchorLayoutAttr = xegpu::setupStoreScatterAnchorLayout(
1351 layoutKind, vectorTy: srcVecTy, contigChunkSize, numSg: numSgOrErr.value_or(u: 0), uArch);
1352 if (!requiredAnchorLayoutAttr) {
1353 markFailure(op: storeScatter,
1354 message: "Failed to determine required layout for store scatter.");
1355 return;
1356 }
1357 storeScatter.setLayoutAttr(requiredAnchorLayoutAttr);
1358 }
1359
1360 LayoutInfo srcLayoutInfo = makeLayoutInfo(layout: requiredAnchorLayoutAttr);
1361 // The mask and offset operands share the value's anchor layout.
1362 LayoutInfo maskLayoutInfo = makeLayoutInfo(layout: requiredAnchorLayoutAttr);
1363
1364 // Propagate the payload operand layout
1365 propagateIfChanged(state: operands[0], changed: operands[0]->meet(rhs: srcLayoutInfo));
1366 // Propagate the destination (if tdesc) operand layout
1367 if (isa<xegpu::TensorDescType>(Val: storeScatter.getDestType()))
1368 propagateIfChanged(state: operands[1], changed: operands[1]->meet(rhs: srcLayoutInfo));
1369 // Propagate the new layout to the offset and mask operands.
1370 propagateIfChanged(state: operands[2], changed: operands[2]->meet(rhs: maskLayoutInfo));
1371 propagateIfChanged(state: operands[3], changed: operands[3]->meet(rhs: maskLayoutInfo));
1372}
1373
1374void LayoutInfoPropagation::visitLoadMatrixOp(
1375 xegpu::LoadMatrixOp loadMatrixOp, ArrayRef<LayoutInfoLattice *> operands,
1376 ArrayRef<const LayoutInfoLattice *> results) {
1377
1378 LayoutInfo resLayoutInfo = results[0]->getValue();
1379 if (!resLayoutInfo.isAssigned())
1380 return;
1381
1382 auto consumerLayoutAttr =
1383 dyn_cast<xegpu::DistributeLayoutAttr>(Val: resLayoutInfo.get());
1384
1385 xegpu::DistributeLayoutAttr anchorLayout = loadMatrixOp.getLayoutAttr();
1386
1387 // only need to set anchor layout, no need to porpagate to memdesc and
1388 // offset
1389 if (!hasParamsOfLayoutKind(anchorLayout)) {
1390 VectorType resVecTy =
1391 llvm::cast<VectorType>(Val: loadMatrixOp.getRes().getType());
1392 const auto *uArch =
1393 xegpu::uArch::getUArch(archName: getChipStr(op: loadMatrixOp).value_or(u: ""));
1394 if (!uArch)
1395 return;
1396 int chunkSize =
1397 1; // placeHolder for future use when LoadMatrix supports coalescing
1398 auto requiredAnchorLayoutAttr = xegpu::setupLoadMatrixAnchorLayout(
1399 layoutKind, vectorTy: resVecTy, contigChunkSize: chunkSize, consumerLayout: consumerLayoutAttr, uArch);
1400 loadMatrixOp.setLayoutAttr(requiredAnchorLayoutAttr);
1401 }
1402}
1403
1404void LayoutInfoPropagation::visitStoreMatrixOp(
1405 xegpu::StoreMatrixOp storeMatrix, ArrayRef<LayoutInfoLattice *> operands,
1406 ArrayRef<const LayoutInfoLattice *> results) {
1407 xegpu::DistributeLayoutAttr requiredAnchorLayoutAttr;
1408 xegpu::DistributeLayoutAttr anchorLayoutAttr = storeMatrix.getLayoutAttr();
1409 LayoutInfo layout;
1410 VectorType srcVecTy = llvm::cast<VectorType>(Val: storeMatrix.getData().getType());
1411 const auto *uArch =
1412 xegpu::uArch::getUArch(archName: getChipStr(op: storeMatrix).value_or(u: ""));
1413 if (!uArch)
1414 return;
1415 if (hasParamsOfLayoutKind(anchorLayout: anchorLayoutAttr)) {
1416 requiredAnchorLayoutAttr = anchorLayoutAttr;
1417 if (layoutKind == xegpu::LayoutKind::InstData) {
1418 const auto uArchInstruction =
1419 dyn_cast<xegpu::uArch::StoreScatterInstruction>(Val: uArch->getInstruction(
1420 instKind: xegpu::uArch::InstructionKind::StoreScatter));
1421 if (!uArchInstruction)
1422 return;
1423 auto completed = xegpu::completeScatterStoreLaneLayoutFromInstData(
1424 specifiedLayout: anchorLayoutAttr, elemTy: srcVecTy.getElementType(), uArchInstruction,
1425 subgroupSize: uArch->getSubgroupSize());
1426 if (!completed) {
1427 markFailure(
1428 op: storeMatrix,
1429 message: "Failed to identify lane layouts for the specified inst_data.");
1430 return;
1431 }
1432 requiredAnchorLayoutAttr = *completed;
1433 storeMatrix.setLayoutAttr(requiredAnchorLayoutAttr);
1434 }
1435 } else {
1436 int chunkSize =
1437 1; // placeHolder for future use when StoreMatrix supports coalescing
1438 auto numSgOrErr =
1439 getNumSgOrFail(op: storeMatrix, sgSize: uArch->getSubgroupSize(), consumerLayout: nullptr);
1440 if (failed(Result: numSgOrErr))
1441 return;
1442 requiredAnchorLayoutAttr = xegpu::setupStoreMatrixAnchorLayout(
1443 layoutKind, vectorTy: srcVecTy, contigChunkSize: chunkSize, numSg: numSgOrErr.value_or(u: 0), uArch);
1444 if (!requiredAnchorLayoutAttr) {
1445 markFailure(op: storeMatrix,
1446 message: "Failed to determine required layout for store matrix.");
1447 return;
1448 }
1449 storeMatrix.setLayoutAttr(requiredAnchorLayoutAttr);
1450 }
1451 layout = makeLayoutInfo(layout: requiredAnchorLayoutAttr);
1452 propagateIfChanged(state: operands[0], changed: operands[0]->meet(rhs: layout));
1453}
1454
1455namespace {
1456//===----------------------------------------------------------------------===//
1457// RunLayoutInfoPropagation
1458//===----------------------------------------------------------------------===//
1459
1460/// Driver class for running the LayoutInfoPropagation analysis.
1461class RunLayoutInfoPropagation {
1462public:
1463 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(RunLayoutInfoPropagation)
1464
1465 RunLayoutInfoPropagation(Operation *op, xegpu::LayoutKind layoutKind,
1466 unsigned indexBitWidth)
1467 : target(op) {
1468 SymbolTableCollection symbolTable;
1469 loadBaselineAnalyses(solver);
1470 analysis = solver.load<LayoutInfoPropagation>(args&: symbolTable, args&: layoutKind,
1471 args&: indexBitWidth, args&: op);
1472 (void)solver.initializeAndRun(top: op);
1473 }
1474
1475 LayoutInfo getLayoutInfo(Value val);
1476
1477 void printAnalysisResult(llvm::raw_ostream &os);
1478
1479 bool hasFailed() const { return analysis && analysis->hasFailed(); }
1480
1481private:
1482 DataFlowSolver solver;
1483 const Operation *target;
1484 LayoutInfoPropagation *analysis = nullptr;
1485};
1486} // namespace
1487
1488LayoutInfo RunLayoutInfoPropagation::getLayoutInfo(Value val) {
1489 auto *state = solver.lookupState<LayoutInfoLattice>(anchor: val);
1490 if (!state)
1491 return {};
1492 return state->getValue();
1493}
1494
1495// Print the analysis result for debugging purposes.
1496void RunLayoutInfoPropagation::printAnalysisResult(llvm::raw_ostream &os) {
1497 auto printFunctionResult = [&](FunctionOpInterface funcOp) {
1498 os << "function: " << funcOp.getName() << ":\n";
1499 // Function arguments
1500 for (BlockArgument arg : funcOp.getArguments()) {
1501 LayoutInfo layout = getLayoutInfo(val: arg);
1502 os << "argument: " << arg << "\n";
1503 os << "layout : ";
1504 layout.print(os);
1505 os << "\n";
1506 }
1507 // Function ops
1508 funcOp.walk(callback: [&](Operation *op) {
1509 // Skip ops that do not have results
1510 if (op->getResults().empty())
1511 return;
1512 os << "op : ";
1513 // For control-flow ops, print the op name only.
1514 if (isa<BranchOpInterface>(Val: op) || isa<RegionBranchOpInterface>(Val: op))
1515 os << op->getName();
1516 else
1517 op->print(os);
1518 os << "\n";
1519 // Print the layout for each result.
1520 for (auto [i, r] : llvm::enumerate(First: op->getResults())) {
1521 LayoutInfo layout = getLayoutInfo(val: r);
1522 os << "layout for result #" << i << ": ";
1523 layout.print(os);
1524 os << "\n";
1525 }
1526 });
1527 };
1528
1529 SmallVector<FunctionOpInterface> funcOps;
1530 if (auto modOp = dyn_cast<ModuleOp>(Val: target)) {
1531 for (auto funcOp : modOp.getOps<FunctionOpInterface>())
1532 funcOps.push_back(Elt: funcOp);
1533
1534 // Collect all GpuFuncOps in the module.
1535 for (auto gpuModOp : modOp.getOps<gpu::GPUModuleOp>()) {
1536 for (auto gpuFuncOp : gpuModOp.getOps<FunctionOpInterface>())
1537 funcOps.push_back(Elt: gpuFuncOp);
1538 }
1539 }
1540 // Print the analysis result for each function.
1541 for (FunctionOpInterface funcOp : funcOps)
1542 printFunctionResult(funcOp);
1543}
1544
1545namespace {
1546//===----------------------------------------------------------------------===//
1547// ResolveLayoutConflicts
1548//===----------------------------------------------------------------------===//
1549
1550/// Helper to get the defining CreateNdDescOp of a tensor descriptor value.
1551/// This function tries to find the defining CreateNdDescOp recursively
1552/// accross control-flow boundaries.
1553static xegpu::CreateNdDescOp getDefiningCreateNdDescOp(Value tdescValue) {
1554 // Try to get the defining CreateNdDescOp of the tensor descriptor.
1555 auto definingOp = tdescValue.getDefiningOp<xegpu::CreateNdDescOp>();
1556 if (definingOp)
1557 return definingOp;
1558 // If tdescValue is an argument, try to get the tied init value from the
1559 // parent loop-like op.
1560 if (auto arg = dyn_cast<BlockArgument>(Val&: tdescValue)) {
1561 auto *parentOp = arg.getOwner()->getParentOp();
1562 if (auto loop = dyn_cast<LoopLikeOpInterface>(Val: parentOp)) {
1563 OpOperand *tiedInit = loop.getTiedLoopInit(bbArg: arg);
1564 if (tiedInit)
1565 return getDefiningCreateNdDescOp(tdescValue: tiedInit->get());
1566 }
1567 }
1568 // If not found, return null.
1569 return nullptr;
1570}
1571
1572struct ResolveLayoutConflicts {
1573 ResolveLayoutConflicts(Operation *parentOp)
1574 : parentOp(parentOp), builder(parentOp->getContext()) {}
1575 LogicalResult run();
1576
1577private:
1578 Operation *parentOp;
1579 OpBuilder builder;
1580 LogicalResult resolveTensorDescConsumer(OpOperand &operand);
1581 LogicalResult resolveVectorConsumer(OpOperand &operand);
1582 LogicalResult assignResultLayout(OpResult &result);
1583};
1584} // namespace
1585
1586LogicalResult ResolveLayoutConflicts::run() {
1587 // Scan all operations in the parent op and resolve layout conflicts at
1588 // tensor descriptor and vector use points.
1589 auto r = parentOp->walk(callback: [&](Operation *op) -> WalkResult {
1590 for (OpResult result : op->getResults()) {
1591 // if the operation inputs vector and output scalar, like multi-reduction
1592 // we need to check if the result has layout and add a convert_layout to
1593 // serve as anchor op for the reduction op's layout.
1594 if (result.getType().isIntOrFloat() &&
1595 (isa<vector::MultiDimReductionOp>(Val: op) ||
1596 isa<vector::ReductionOp>(Val: op))) {
1597 auto res = assignResultLayout(result);
1598 if (failed(Result: res)) {
1599 DBGS() << "Failed to assign layout for scalar consumer of reduction "
1600 << *op << "\n";
1601 return WalkResult::interrupt();
1602 }
1603 }
1604 // If the op is a region branch op with a vector result that has no uses,
1605 // we need to add a convert_layout to serve as an anchor op for the
1606 // result's layout.
1607 if (isa<VectorType>(Val: result.getType()) && result.use_empty() &&
1608 isa<RegionBranchOpInterface>(Val: op)) {
1609 auto res = assignResultLayout(result);
1610 if (failed(Result: res)) {
1611 DBGS() << "Failed to assign layout for vector consumer of region op "
1612 << *op << "\n";
1613 return WalkResult::interrupt();
1614 }
1615 }
1616 }
1617 for (OpOperand &operand : op->getOpOperands()) {
1618 // Handle conflicts in tensor descriptor operands.
1619 Type operandType = operand.get().getType();
1620 if (isa<xegpu::AnchorLayoutInterface>(Val: op) &&
1621 isa<xegpu::TensorDescType>(Val: operandType)) {
1622 auto res = resolveTensorDescConsumer(operand);
1623 if (failed(Result: res)) {
1624 DBGS() << "Failed to resolve tensor descriptor consumer: " << *op
1625 << "\n";
1626 return WalkResult::interrupt();
1627 }
1628 }
1629 // Handle conflicts in vector operands.
1630 if (isa<VectorType>(Val: operandType)) {
1631 auto res = resolveVectorConsumer(operand);
1632 if (failed(Result: res)) {
1633 DBGS() << "Failed to resolve vector consumer: " << *op << "\n";
1634 return WalkResult::interrupt();
1635 }
1636 }
1637 }
1638 return WalkResult::advance();
1639 });
1640
1641 if (r.wasInterrupted())
1642 return failure();
1643
1644 LLVM_DEBUG({
1645 DBGS() << "IR after resolving layout conflicts:\n";
1646 parentOp->dump();
1647 });
1648
1649 return success();
1650}
1651
1652LogicalResult ResolveLayoutConflicts::assignResultLayout(OpResult &result) {
1653 Operation *producerOp = result.getDefiningOp();
1654 auto producerLayout = xegpu::getDistributeLayoutAttr(value: result);
1655 // Insert a convert_layout op to assign the layout.
1656 builder.setInsertionPointAfterValue(result);
1657 auto convertOp = xegpu::ConvertLayoutOp::create(
1658 builder, location: producerOp->getLoc(), result: result.getType(), source: result, input_layout: producerLayout,
1659 target_layout: producerLayout);
1660 result.replaceAllUsesExcept(newValue: convertOp.getResult(), exceptedUser: convertOp);
1661 return success();
1662}
1663
1664LogicalResult
1665ResolveLayoutConflicts::resolveVectorConsumer(OpOperand &operand) {
1666 Value vectorValue = operand.get();
1667 Operation *consumerOp = operand.getOwner();
1668 // Get the current layout of the vector value.
1669 auto producerLayout = xegpu::getDistributeLayoutAttr(value: vectorValue);
1670 if (!producerLayout) {
1671 if (auto vectorTy = dyn_cast<VectorType>(Val: vectorValue.getType());
1672 vectorTy && vectorTy.getRank() > 1)
1673 consumerOp->emitWarning(message: "Expected layout for non-1D vectors.");
1674 return success(); // uniform non-tensor-data vector does not require
1675 // layout
1676 }
1677 // getConsumerLayoutAt also covers region-carried operands (loop init and
1678 // yield operands), so a layout conflict there is reconciled below rather than
1679 // silently trusted to region forwarding.
1680 auto consumerLayout = xegpu::getConsumerLayoutAt(operand);
1681 if (!consumerLayout) {
1682 if (isa<func::ReturnOp>(Val: consumerOp) || isa<gpu::ReturnOp>(Val: consumerOp))
1683 return success();
1684 return consumerOp->emitError(
1685 message: "No consumer layout found for vector operand.");
1686 }
1687
1688 // If layouts are same, no conflict exists, return success.
1689 if (consumerLayout.isEqualTo(other: producerLayout))
1690 return success();
1691
1692 // Consumer is a convert_layout: retarget its input_layout to the producer
1693 // instead of chaining a second convert. Always safe (single source
1694 // operand).
1695 if (auto consumerConvert = dyn_cast<xegpu::ConvertLayoutOp>(Val: consumerOp)) {
1696 consumerConvert.setInputLayoutAttr(producerLayout);
1697 return success();
1698 }
1699
1700 // Producer is a convert_layout feeding only this use: retarget its
1701 // target_layout to the consumer instead of appending another convert.
1702 if (auto producerConvert =
1703 vectorValue.getDefiningOp<xegpu::ConvertLayoutOp>();
1704 producerConvert && vectorValue.hasOneUse()) {
1705 // Pin the effective input before retargeting target, else an omitted
1706 // input_layout would follow target and make the conversion a no-op.
1707 producerConvert.setInputLayoutAttr(
1708 producerConvert.getEffectiveInputLayout());
1709 producerConvert.setTargetLayoutAttr(consumerLayout);
1710 return success();
1711 }
1712
1713 // Insert a convert_layout op to resolve the conflict.
1714 builder.setInsertionPointAfterValue(vectorValue);
1715 auto convertOp = xegpu::ConvertLayoutOp::create(
1716 builder, location: consumerOp->getLoc(), result: vectorValue.getType(), source: vectorValue,
1717 input_layout: producerLayout, target_layout: consumerLayout);
1718
1719 // Update the operand to use the converted value.
1720 operand.set(convertOp.getResult());
1721 return success();
1722}
1723
1724LogicalResult
1725ResolveLayoutConflicts::resolveTensorDescConsumer(OpOperand &operand) {
1726 Operation *consumerOp = operand.getOwner();
1727 Value tdescValue = operand.get();
1728 auto anchorOp = dyn_cast<xegpu::AnchorLayoutInterface>(Val: consumerOp);
1729 auto currTDescType = dyn_cast<xegpu::TensorDescType>(Val: tdescValue.getType());
1730 assert(anchorOp && currTDescType &&
1731 "Expected anchor layout op and tensor descriptor consumer.");
1732 Attribute currLayout = currTDescType.getLayout();
1733 Attribute expectedLayout = anchorOp.getAnchorLayout();
1734 // A conflict exists in tensor descriptor operand if tensor descriptor's
1735 // layout is different from the anchor layout expected by the consumer.
1736 if (expectedLayout && currLayout && expectedLayout != currLayout) {
1737 // Try to get the defining CreateNdDescOp of the tensor descriptor.
1738 auto conflictingCreateNdOp = getDefiningCreateNdDescOp(tdescValue);
1739 if (!conflictingCreateNdOp) {
1740 DBGS() << "Unable to find defining CreateNdDescOp for tensor descriptor: "
1741 << tdescValue << "\n";
1742 return failure();
1743 }
1744 // Duplicate the CreateNdDescOp with the expected layout.
1745 builder.setInsertionPointAfter(conflictingCreateNdOp);
1746 auto newTensorDescType = xegpu::TensorDescType::get(
1747 context: conflictingCreateNdOp.getContext(), shape: currTDescType.getShape(),
1748 elementType: currTDescType.getElementType(), encoding: currTDescType.getEncoding(),
1749 layout: expectedLayout);
1750 auto newOp = xegpu::CreateNdDescOp::create(
1751 builder, location: consumerOp->getLoc(), resultTypes: TypeRange{newTensorDescType},
1752 operands: conflictingCreateNdOp->getOperands(),
1753 properties: conflictingCreateNdOp.getProperties(),
1754 discardableAttributes: conflictingCreateNdOp->getDiscardableAttrDictionary().getValue());
1755 // Replace the tensor descriptor operand in the consumer op with the new
1756 // tensor descriptor.
1757 consumerOp->replaceUsesOfWith(from: tdescValue, to: newOp.getResult());
1758 }
1759 return success();
1760}
1761
1762using GetLayoutFnTy = function_ref<xegpu::DistributeLayoutAttr(Value)>;
1763
1764/// Update an operation with the layout of its results. For a vector result a
1765/// temporary layout attribute is added to the op; for a tensor descriptor
1766/// result the layout is written into its type.
1767///
1768/// If the global propagation left a result without a layout, forward-fill it
1769/// locally from the operand layouts.
1770static LogicalResult updateOpWithForwardFill(mlir::OpBuilder &builder,
1771 mlir::Operation *op,
1772 GetLayoutFnTy getLayoutOfValue) {
1773 // Iterate over all the results.
1774 for (OpResult result : op->getResults()) {
1775 Type resultType = result.getType();
1776 // Layouts are needed only for vector and tensor descriptor types.
1777 if (!isa<VectorType, xegpu::TensorDescType>(Val: resultType))
1778 continue;
1779 // If the result has no layout but has users, emit a warning and continue.
1780 xegpu::DistributeLayoutAttr layout = getLayoutOfValue(result);
1781 if (!layout) {
1782 // Gather operand layouts, indexed by operand number.
1783 SmallVector<xegpu::DistributeLayoutAttr> srcLayouts;
1784 srcLayouts.reserve(N: op->getNumOperands());
1785 bool anyAssigned = false;
1786 for (Value operand : op->getOperands()) {
1787 auto srclayout = xegpu::getDistributeLayoutAttr(value: operand);
1788 srcLayouts.push_back(Elt: srclayout);
1789 anyAssigned |= (srclayout != nullptr);
1790 }
1791 if (anyAssigned) {
1792 layout =
1793 xegpu::inferResultLayoutFromSourceForNonAnchorOp(op, operandLayouts: srcLayouts);
1794 }
1795 }
1796 if (!layout && result.getNumUses() > 0) {
1797 op->emitWarning(message: "op has users but no layout assigned for its result");
1798 }
1799 // If the result is a tensor descriptor type, update the tensor desc type
1800 // with layout.
1801 if (auto tensorDescTy = dyn_cast<xegpu::TensorDescType>(Val&: resultType)) {
1802 auto typeWithLayout = xegpu::TensorDescType::get(
1803 context: tensorDescTy.getContext(), shape: tensorDescTy.getShape(),
1804 elementType: tensorDescTy.getElementType(), encoding: tensorDescTy.getEncoding(), layout);
1805 result.setType(typeWithLayout);
1806 continue;
1807 }
1808 // If the result is a vector type, add a temporary layout attribute to the
1809 // op.
1810 xegpu::setDistributeLayoutAttr(Result: result, layout);
1811 }
1812 return success();
1813}
1814
1815/// Replace layout conversions of cheap producer chains by recomputation.
1816///
1817/// %step = vector.step {L1}
1818/// %cast = arith.index_castui %step {L1}
1819/// %cvt = xegpu.convert_layout %cast : L1 -> L2
1820/// use(%cast) {L1}
1821/// use(%cvt) {L2}
1822///
1823/// becomes
1824///
1825/// %step = vector.step {L1}
1826/// %step' = vector.step
1827/// %cast = arith.index_castui %step {L1}
1828/// %cast' = arith.index_castui %step'
1829/// use(%cast) {L1}
1830/// use(%cast') {L2}
1831///
1832/// Clone the whole vector producer chain so the original and rematerialized
1833/// values can serve consumers with different layouts. Clones carry no layout
1834/// attributes; layout-dependent passes recover them from the use points. For
1835/// small, layout-preserving chains, recomputation can avoid cross-subgroup data
1836/// movement through SLM.
1837void xegpu::rematerializeConversionSources(OpBuilder &builder,
1838 Operation *parentOp) {
1839 SmallVector<xegpu::ConvertLayoutOp> conversions;
1840 parentOp->walk(
1841 callback: [&](xegpu::ConvertLayoutOp convert) { conversions.push_back(Elt: convert); });
1842 for (xegpu::ConvertLayoutOp convert : conversions) {
1843 Value clone = xegpu::rematerialize(builder, value: convert.getSource());
1844 if (!clone)
1845 continue;
1846 convert.getResult().replaceAllUsesWith(newValue: clone);
1847 convert.erase();
1848 }
1849}
1850
1851/// Optimize elementwise operations by sinking costly layout conversion.
1852///
1853/// %m = vector.create_mask ... {L1}
1854/// %d = arith.mulf ... {L1}
1855/// %cm = xegpu.convert_layout %m : L1 -> L2
1856/// %cd = xegpu.convert_layout %d : L1 -> L2
1857/// %s = arith.select %cm, %cd, %splat {L2}
1858///
1859/// becomes
1860///
1861/// %s' = arith.select %m, %d, %splat {L1}
1862/// %s = xegpu.convert_layout %s' : L1 -> L2
1863///
1864/// We only sink if a coarser layout applies to an elementwise op
1865///
1866void xegpu::sinkElementwiseConversions(OpBuilder &builder,
1867 Operation *parentOp) {
1868 llvm::SmallSetVector<xegpu::ConvertLayoutOp, 8> deadConverts;
1869 parentOp->walk(callback: [&](Operation *op) {
1870 if (!OpTrait::hasElementwiseMappableTraits(op) || op->getNumResults() != 1)
1871 return;
1872 OpResult result = op->getResult(idx: 0);
1873 if (!isa<VectorType>(Val: result.getType()))
1874 return;
1875 xegpu::DistributeLayoutAttr resultLayout =
1876 xegpu::getDistributeLayoutAttr(value: result);
1877 if (!resultLayout)
1878 return;
1879
1880 // Ensure that all feeding conversions have the same layout.
1881 SmallVector<std::pair<OpOperand *, xegpu::ConvertLayoutOp>> conversions;
1882 xegpu::DistributeLayoutAttr uniformConvSrcLayout;
1883 for (OpOperand &operand : op->getOpOperands()) {
1884 Value operandValue = operand.get();
1885 if (!isa<VectorType>(Val: operandValue.getType()))
1886 continue;
1887 auto operandConversionOp =
1888 operandValue.getDefiningOp<xegpu::ConvertLayoutOp>();
1889 // Those that are not a conversion should be easily materializable with
1890 // another layout.
1891 if (!operandConversionOp) {
1892 Operation *definingOp = operandValue.getDefiningOp();
1893 if (!definingOp || !xegpu::isTriviallyRematerializable(op: definingOp))
1894 return;
1895 continue;
1896 }
1897 xegpu::DistributeLayoutAttr input =
1898 operandConversionOp.getEffectiveInputLayout();
1899 if (!input)
1900 return;
1901 if (!uniformConvSrcLayout)
1902 uniformConvSrcLayout = input;
1903 else if (!uniformConvSrcLayout.isEqualTo(other: input))
1904 return;
1905 conversions.emplace_back(Args: &operand, Args&: operandConversionOp);
1906 }
1907
1908 if (!uniformConvSrcLayout)
1909 return;
1910
1911 // Sink only when it leaves the op coarser.
1912 // Conversion's source layout must be larger than the elemwise result
1913 // layout.
1914 SmallVector<int64_t> resultLayoutInstData =
1915 resultLayout.getEffectiveInstDataAsInt();
1916 SmallVector<int64_t> sourceInstData =
1917 uniformConvSrcLayout.getEffectiveInstDataAsInt();
1918 if (resultLayoutInstData.empty() || sourceInstData.empty() ||
1919 computeProduct(basis: sourceInstData) <= computeProduct(basis: resultLayoutInstData))
1920 return;
1921
1922 // Rewire the conversions, then rematerialize the remaining operands in the
1923 // source layout.
1924 llvm::SmallDenseSet<unsigned> rewiredOpIdxs;
1925 for (auto [operand, convert] : conversions) {
1926 operand->set(convert.getSource());
1927 rewiredOpIdxs.insert(V: operand->getOperandNumber());
1928 }
1929 for (OpOperand &operand : op->getOpOperands()) {
1930 Value operandValue = operand.get();
1931 if (!isa<VectorType>(Val: operandValue.getType()))
1932 continue;
1933 if (rewiredOpIdxs.contains(V: operand.getOperandNumber()))
1934 continue;
1935 Operation *definingOp = operandValue.getDefiningOp();
1936 assert(definingOp && xegpu::isTriviallyRematerializable(definingOp) &&
1937 "operand should have been rejected above");
1938 // Rematerialize with uniform source layout.
1939 builder.setInsertionPointAfter(definingOp);
1940 Operation *clone = builder.clone(op&: *definingOp);
1941 OpResult cloneResult =
1942 clone->getResult(idx: cast<OpResult>(Val&: operandValue).getResultNumber());
1943 xegpu::removeLayoutAttr(operandOrResult: cloneResult);
1944 xegpu::setDistributeLayoutAttr(Result: cloneResult, layout: uniformConvSrcLayout);
1945 operand.set(cloneResult);
1946 }
1947
1948 // Run the op in the source layout and bridge its result back, so that
1949 // `getConsumerLayoutAt` now reports the source layout for every operand.
1950 builder.setInsertionPointAfterValue(result);
1951 auto newConvOp = xegpu::ConvertLayoutOp::create(
1952 builder, location: op->getLoc(), result: result.getType(), source: result, input_layout: uniformConvSrcLayout,
1953 target_layout: resultLayout);
1954 result.replaceAllUsesExcept(newValue: newConvOp.getResult(), exceptedUser: newConvOp);
1955 xegpu::removeLayoutAttr(operandOrResult: result);
1956 xegpu::setDistributeLayoutAttr(Result: result, layout: uniformConvSrcLayout);
1957
1958 for (auto [operand, convert] : conversions)
1959 if (convert.getResult().use_empty())
1960 deadConverts.insert(X: convert);
1961 });
1962
1963 // Memory effects block dce
1964 for (xegpu::ConvertLayoutOp convert : deadConverts)
1965 convert.erase();
1966}
1967
1968/// Update the function arguments and results with the layouts.
1969static LogicalResult updateFunctionOpInterface(mlir::OpBuilder &builder,
1970 mlir::FunctionOpInterface funcOp,
1971 GetLayoutFnTy getLayoutOfValue) {
1972 // Only process functions whose type is a standard MLIR FunctionType.
1973 // Functions using a different type representation (e.g. llvm.func with
1974 // LLVMFunctionType) are not targets for XeGPU layout propagation, and
1975 // calling setType(FunctionType{}) on them would corrupt their type.
1976 if (!isa<FunctionType>(Val: funcOp.getFunctionType()))
1977 return success();
1978 SmallVector<Type> newArgTypes;
1979 // Update the function arguments.
1980 for (BlockArgument arg : funcOp.getArguments()) {
1981 Type argType = arg.getType();
1982 newArgTypes.push_back(Elt: argType);
1983 if (!isa<VectorType, xegpu::TensorDescType>(Val: argType))
1984 continue;
1985 xegpu::DistributeLayoutAttr layout = getLayoutOfValue(arg);
1986 if (!layout) {
1987 LLVM_DEBUG(DBGS() << "Expecting layout for function argument: " << arg
1988 << " but got none.\n");
1989 return failure();
1990 }
1991 if (auto tensorDescTy = dyn_cast<xegpu::TensorDescType>(Val&: argType)) {
1992 auto newTdescTy = xegpu::TensorDescType::get(
1993 context: tensorDescTy.getContext(), shape: tensorDescTy.getShape(),
1994 elementType: tensorDescTy.getElementType(), encoding: tensorDescTy.getEncoding(), layout);
1995 arg.setType(newTdescTy);
1996 newArgTypes.back() = newTdescTy;
1997 }
1998 }
1999 // Update the function type with the new argument types.
2000 // NOTE: We assume that function results are not expected to have layouts.
2001 funcOp.setType(FunctionType::get(context: funcOp.getContext(), inputs: newArgTypes,
2002 results: funcOp.getResultTypes()));
2003 return success();
2004}
2005
2006namespace {
2007struct XeGPUPropagateLayoutPass final
2008 : public xegpu::impl::XeGPUPropagateLayoutBase<XeGPUPropagateLayoutPass> {
2009 XeGPUPropagateLayoutPass() = default;
2010 XeGPUPropagateLayoutPass(const XeGPUPropagateLayoutPass &other) = default;
2011 XeGPUPropagateLayoutPass(xegpu::XeGPUPropagateLayoutOptions options)
2012 : XeGPUPropagateLayoutBase(std::move(options)) {}
2013 void runOnOperation() override;
2014};
2015} // namespace
2016
2017LogicalResult xegpu::propagateLayouts(OpBuilder &builder, Operation *target,
2018 LayoutKind layoutKind,
2019 unsigned indexBitWidth, bool printOnly) {
2020 RunLayoutInfoPropagation analysis(target, layoutKind, indexBitWidth);
2021 // Print the analysis result and exit. (for debugging purposes)
2022 if (printOnly) {
2023 auto &os = llvm::outs();
2024 analysis.printAnalysisResult(os);
2025 return success();
2026 }
2027 // An op with no determinable layout cannot be lowered; stop before the update
2028 // walk fabricates degenerate layouts from the unlabeled values.
2029 if (analysis.hasFailed())
2030 return failure();
2031 // Helper to convert LayoutInfo to xegpu::LayoutAttr.
2032 auto getLayoutFromPropagation =
2033 [&](Value val) -> xegpu::DistributeLayoutAttr {
2034 LayoutInfo layout = analysis.getLayoutInfo(val);
2035 if (auto opResult = dyn_cast<OpResult>(Val&: val)) {
2036 Operation *defOp = opResult.getDefiningOp();
2037 if (auto anchorOp = dyn_cast<xegpu::AnchorLayoutInterface>(Val: defOp)) {
2038 auto anchorLayout = anchorOp.getAnchorLayout();
2039 if (anchorLayout != nullptr)
2040 return anchorLayout;
2041 }
2042 xegpu::DistributeLayoutAttr requiredResLayoutAttr =
2043 xegpu::getTemporaryLayout(operandOrResult: opResult);
2044 if (requiredResLayoutAttr != nullptr)
2045 return requiredResLayoutAttr;
2046 }
2047 if (!layout.isAssigned())
2048 return {};
2049 xegpu::DistributeLayoutAttr layoutAttr =
2050 cast<xegpu::DistributeLayoutAttr>(Val: layout.get());
2051 if (layout.isSliceLayout())
2052 return cast<xegpu::SliceAttr>(Val&: layoutAttr);
2053
2054 return cast<xegpu::LayoutAttr>(Val&: layoutAttr);
2055 };
2056
2057 Operation *op = target;
2058 auto walkResult = op->walk(callback: [&](mlir::Block *block) -> WalkResult {
2059 for (mlir::Operation &op : block->getOperations()) {
2060 LogicalResult r = success();
2061 TypeSwitch<Operation *>(&op)
2062 .Case(caseFn: [&](mlir::RegionBranchTerminatorOpInterface branchTermOp) {
2063 r = xegpu::propagateYieldOperandsToRegionResults(
2064 terminator: branchTermOp, getLayoutOfValue: getLayoutFromPropagation);
2065 })
2066 .Case(caseFn: [&](mlir::RegionBranchOpInterface branchOp) {
2067 r = xegpu::propagateRegionArgsToInits(regionOp: branchOp,
2068 getLayoutOfValue: getLayoutFromPropagation);
2069 })
2070 .Case(caseFn: [&](mlir::FunctionOpInterface funcOp) {
2071 r = updateFunctionOpInterface(builder, funcOp,
2072 getLayoutOfValue: getLayoutFromPropagation);
2073 })
2074 .Default(defaultFn: [&](Operation *op) {
2075 r = updateOpWithForwardFill(builder, op, getLayoutOfValue: getLayoutFromPropagation);
2076 });
2077 if (failed(Result: r)) {
2078 op.emitError(message: "Failed to update operation with the layout.");
2079 return WalkResult::interrupt();
2080 }
2081 }
2082 return WalkResult::advance();
2083 });
2084 if (walkResult.wasInterrupted())
2085 return failure();
2086
2087 return success();
2088}
2089
2090LogicalResult xegpu::resolveLayoutConflicts(Operation *target) {
2091 ResolveLayoutConflicts resolver(target);
2092 return resolver.run();
2093}
2094
2095void XeGPUPropagateLayoutPass::runOnOperation() {
2096
2097 xegpu::removeTemporaryLayoutAttrs(op: getOperation());
2098
2099 xegpu::LayoutKind layoutKind;
2100 if (this->layoutKind == "lane") {
2101 layoutKind = xegpu::LayoutKind::Lane;
2102 } else if (this->layoutKind == "inst") {
2103 layoutKind = xegpu::LayoutKind::InstData;
2104 } else if (this->layoutKind == "subgroup") {
2105 layoutKind = xegpu::LayoutKind::Subgroup;
2106 } else {
2107 getOperation()->emitError(message: "Unsupported layout kind option: " +
2108 this->layoutKind);
2109 signalPassFailure();
2110 return;
2111 }
2112 OpBuilder builder(&getContext());
2113 if (failed(Result: xegpu::propagateLayouts(builder, target: getOperation(), layoutKind,
2114 indexBitWidth: this->indexBitWidth, printOnly: this->printOnly))) {
2115 signalPassFailure();
2116 return;
2117 }
2118 // Resolve layout conflicts if any.
2119 if (failed(Result: xegpu::resolveLayoutConflicts(target: getOperation()))) {
2120 signalPassFailure();
2121 return;
2122 }
2123 xegpu::rematerializeConversionSources(builder, parentOp: getOperation());
2124 if (layoutKind == xegpu::LayoutKind::InstData)
2125 xegpu::sinkElementwiseConversions(builder, parentOp: getOperation());
2126}
2127