1//===- TestXeGPUTransforms.cpp -- Test Vector transforms and lowerings ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Arith/IR/Arith.h"
10#include "mlir/Dialect/GPU/IR/GPUDialect.h"
11#include "mlir/Dialect/Index/IR/IndexDialect.h"
12#include "mlir/Dialect/Math/IR/Math.h"
13#include "mlir/Dialect/SCF/Transforms/Patterns.h"
14#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
15#include "mlir/Dialect/XeGPU/IR/XeGPU.h"
16#include "mlir/Dialect/XeGPU/Transforms/Transforms.h"
17#include "mlir/Dialect/XeGPU/Transforms/XeGPULayoutImpl.h"
18#include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
19#include "mlir/Dialect/XeGPU/uArch/IntelGpuXe2.h"
20#include "mlir/IR/BuiltinTypes.h"
21#include "mlir/IR/Value.h"
22#include "mlir/Pass/Pass.h"
23#include "mlir/Pass/PassManager.h"
24#include "mlir/Transforms/DialectConversion.h"
25#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
26#include "llvm/Support/raw_ostream.h"
27#include <optional>
28
29using namespace mlir;
30using namespace mlir::xegpu;
31
32namespace {
33
34#define DEBUG_TYPE "test-xegpu-unroll"
35
36struct TestXeGPUUnrollingPatterns
37 : public PassWrapper<TestXeGPUUnrollingPatterns,
38 OperationPass<gpu::GPUModuleOp>> {
39 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestXeGPUUnrollingPatterns)
40
41 StringRef getArgument() const final {
42 return "test-xegpu-unrolling-patterns";
43 }
44
45 StringRef getDescription() const final {
46 return "Test lowering patterns to unroll ops in the xegpu dialect";
47 }
48
49 void getDependentDialects(::mlir::DialectRegistry &registry) const override {
50 registry.insert<memref::MemRefDialect>();
51 registry.insert<xegpu::XeGPUDialect>();
52 registry.insert<vector::VectorDialect>();
53 }
54
55 TestXeGPUUnrollingPatterns() = default;
56 TestXeGPUUnrollingPatterns(const TestXeGPUUnrollingPatterns &pass) = default;
57
58 void runOnOperation() override {
59 MLIRContext *ctx = &getContext();
60 xegpu::UnrollOptions options;
61 options.setNativeShapeFn([&](Operation *op)
62 -> std::optional<SmallVector<int64_t>> {
63 if (isa<xegpu::CreateNdDescOp, xegpu::PrefetchNdOp, xegpu::LoadNdOp,
64 xegpu::StoreNdOp, xegpu::PrefetchOp, xegpu::LoadGatherOp,
65 xegpu::StoreScatterOp>(Val: op)) {
66 xegpu::TensorDescType tdescTy;
67 if (auto createNdOp = dyn_cast<xegpu::CreateNdDescOp>(Val: op)) {
68 tdescTy = createNdOp.getType();
69 } else if (auto prefetchNdOp = dyn_cast<xegpu::PrefetchNdOp>(Val: op)) {
70 tdescTy = prefetchNdOp.getTensorDescType();
71 } else if (auto loadNdOp = dyn_cast<xegpu::LoadNdOp>(Val: op)) {
72 tdescTy = loadNdOp.getTensorDescType();
73 } else if (auto storeNdOp = dyn_cast<xegpu::StoreNdOp>(Val: op)) {
74 tdescTy = storeNdOp.getTensorDescType();
75 } else if (isa<xegpu::PrefetchOp, xegpu::LoadGatherOp,
76 xegpu::StoreScatterOp>(Val: op)) {
77 auto anchorOp = cast<xegpu::AnchorLayoutInterface>(Val: op);
78 auto layout =
79 dyn_cast_or_null<xegpu::LayoutAttr>(Val: anchorOp.getAnchorLayout());
80 if (layout && layout.isForSubgroup()) {
81 auto inst_data = layout.getEffectiveInstDataAsInt();
82 if (!inst_data.empty())
83 return SmallVector<int64_t>(inst_data.begin(), inst_data.end());
84 }
85 return std::nullopt;
86 }
87
88 if (auto layout = tdescTy.getLayoutAttr()) {
89 auto inst_data = layout.getEffectiveInstDataAsInt();
90 if (!inst_data.empty() && layout.isForSubgroup())
91 return SmallVector<int64_t>(inst_data.begin(), inst_data.end());
92 }
93 }
94
95 if (isa<xegpu::DpasOp>(Val: op))
96 return SmallVector<int64_t>{8, 16, 16};
97
98 // For vector.multi_reduction, read tile shape from the layout attribute
99 // on the source operand (layout_operand_0).
100 if (isa<vector::MultiDimReductionOp>(Val: op)) {
101 xegpu::DistributeLayoutAttr layout =
102 xegpu::getDistributeLayoutAttr(opr: op->getOpOperand(idx: 0));
103 if (layout) {
104 auto instData = layout.getEffectiveInstDataAsInt();
105 if (!instData.empty())
106 return instData;
107 }
108 return std::nullopt;
109 }
110
111 return std::nullopt;
112 });
113
114 options.setUnrolledTypesFn(
115 [&](ShapedType type, ArrayRef<int64_t> tileShape) -> SmallVector<Type> {
116 Type elemTy = type.getElementType();
117 Type newTy;
118
119 // TensorDescType needs to drop the inst_data field in the layout
120 // attribute
121 if (auto tdescTy = dyn_cast<xegpu::TensorDescType>(Val&: type)) {
122 Attribute encoding = tdescTy.getEncoding();
123 auto layout = tdescTy.getLayoutAttr();
124
125 if (layout) {
126 if (layout.getEffectiveLaneLayoutAsInt().empty())
127 layout = xegpu::LayoutAttr();
128 else
129 layout = layout.dropInstData();
130 }
131
132 newTy = xegpu::TensorDescType::get(context: ctx, shape: tileShape, elementType: elemTy, encoding,
133 layout);
134 // compute the product of batch (higher) dimensions
135 ArrayRef<int64_t> shape = type.getShape();
136 int64_t batchCount =
137 shape.size() > 2 ? computeProduct(basis: shape.drop_back(N: 2)) : 1;
138 return SmallVector<Type>(batchCount, newTy);
139 }
140
141 newTy = type.clone(shape: tileShape, elementType: elemTy);
142 std::optional<SmallVector<int64_t>> ratio =
143 computeShapeRatio(shape: type.getShape(), subShape: tileShape);
144 assert(ratio && "Expecting the ratio to be valid.");
145 return SmallVector<Type>(computeProduct(basis: *ratio), newTy);
146 });
147
148 RewritePatternSet patterns(ctx);
149
150 populateXeGPUUnrollPatterns(patterns, options);
151 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
152 }
153};
154
155#undef DEBUG_TYPE
156#define DEBUG_TYPE "test-xegpu-layout-interface"
157
158// Test pattern for distributing vector::StepOp from workgroup to subgroup.
159// Validates DistributeLayoutAttr interfaces for offset computation
160// abstraction between LayoutAttr and SliceAttr.
161class TestStepOpPattern : public OpConversionPattern<vector::StepOp> {
162 using OpConversionPattern<vector::StepOp>::OpConversionPattern;
163
164 LogicalResult
165 matchAndRewrite(vector::StepOp op, OneToNOpAdaptor adaptor,
166 ConversionPatternRewriter &rewriter) const override {
167
168 auto layoutName = xegpu::getTemporaryLayoutName(result: op->getResult(idx: 0));
169 auto sliceAttr = op->getDiscardableAttrOfType<xegpu::SliceAttr>(name: layoutName);
170 if (!sliceAttr || sliceAttr.getRank() != 1)
171 return failure();
172
173 std::optional<SmallVector<int64_t>> sgShape =
174 sliceAttr.getEffectiveSgDataAsInt();
175 if (!sgShape)
176 return failure();
177
178 Location loc = op.getLoc();
179 VectorType type = op.getResult().getType();
180 auto wgShape = type.getShape();
181
182 Value sgId =
183 gpu::SubgroupIdOp::create(builder&: rewriter, location: loc, /*upper_bound=*/nullptr);
184 auto maybeOffsets =
185 sliceAttr.computeDistributedCoords(builder&: rewriter, loc, linearId: sgId, shape: wgShape);
186 if (failed(Result: maybeOffsets))
187 return failure();
188
189 VectorType newTy = type.cloneWith(shape: *sgShape, elementType: type.getElementType());
190 Value base = vector::StepOp::create(builder&: rewriter, location: loc, result: newTy);
191 SmallVector<Value> newOps;
192 for (auto offsets : *maybeOffsets) {
193 Value bcast =
194 vector::BroadcastOp::create(builder&: rewriter, location: loc, result: newTy, source: offsets[0]);
195 Value add = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: base, rhs: bcast);
196 newOps.push_back(Elt: add);
197 }
198 rewriter.replaceOpWithMultiple(op, newValues: {newOps});
199 return success();
200 }
201};
202
203struct TestXeGPURecoverTemporaryLayouts
204 : public PassWrapper<TestXeGPURecoverTemporaryLayouts,
205 OperationPass<gpu::GPUModuleOp>> {
206 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestXeGPURecoverTemporaryLayouts)
207
208 StringRef getArgument() const final {
209 return "test-xegpu-recover-temporary-layouts";
210 }
211
212 StringRef getDescription() const final {
213 return "Test the implementation of XeGPU temporary layout recovery";
214 }
215
216 void getDependentDialects(::mlir::DialectRegistry &registry) const override {
217 registry.insert<arith::ArithDialect>();
218 registry.insert<memref::MemRefDialect>();
219 registry.insert<xegpu::XeGPUDialect>();
220 registry.insert<vector::VectorDialect>();
221 registry.insert<gpu::GPUDialect>();
222 }
223
224 TestXeGPURecoverTemporaryLayouts() = default;
225 TestXeGPURecoverTemporaryLayouts(const TestXeGPURecoverTemporaryLayouts &pass)
226 : PassWrapper(pass) {}
227
228 void runOnOperation() override {
229 Operation *op = getOperation();
230 if (!xegpu::recoverTemporaryLayouts(rootOp: op))
231 signalPassFailure();
232 }
233};
234
235/// This test pass is intended to test the subgroup to lane distribution of
236/// xegpu/vector/arith operations in isolation, it does not handle any
237/// structural ops like scf.for etc.
238struct TestXeGPUSgToLaneDistribute
239 : public PassWrapper<TestXeGPUSgToLaneDistribute,
240 OperationPass<gpu::GPUModuleOp>> {
241 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestXeGPUSgToLaneDistribute)
242
243 StringRef getArgument() const final {
244 return "test-xegpu-sg-to-lane-distribute";
245 }
246
247 StringRef getDescription() const final {
248 return "Test the implementation of XeGPU Subgroup to Lane Distribution";
249 }
250
251 void getDependentDialects(::mlir::DialectRegistry &registry) const override {
252 registry.insert<arith::ArithDialect>();
253 registry.insert<memref::MemRefDialect>();
254 registry.insert<xegpu::XeGPUDialect>();
255 registry.insert<vector::VectorDialect>();
256 registry.insert<index::IndexDialect>();
257 registry.insert<gpu::GPUDialect>();
258 }
259
260 TestXeGPUSgToLaneDistribute() = default;
261 TestXeGPUSgToLaneDistribute(const TestXeGPUSgToLaneDistribute &pass)
262 : PassWrapper(pass) {}
263
264 void runOnOperation() override {
265 Operation *op = getOperation();
266 if (!xegpu::recoverTemporaryLayouts(rootOp: op)) {
267 signalPassFailure();
268 return;
269 }
270
271 MLIRContext *ctx = &getContext();
272 TypeConverter typeConverter;
273 // Define type materializations using UnrealizedConversionCastOp.
274 auto materializeCast = [&](mlir::OpBuilder &builder, mlir::Type type,
275 mlir::ValueRange inputs,
276 mlir::Location loc) -> mlir::Value {
277 return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: type, operands: inputs)
278 .getResult(i: 0);
279 };
280 typeConverter.addSourceMaterialization(callback&: materializeCast);
281 typeConverter.addTargetMaterialization(callback&: materializeCast);
282
283 ConversionTarget target(*ctx);
284 RewritePatternSet patterns(ctx);
285 xegpu::populateXeGPUSgToLaneDistributeTypeConversionAndLegality(
286 typeConverter, patterns, target, topLevelOp: op);
287 (void)applyPartialConversion(op, target, patterns: std::move(patterns));
288 }
289};
290
291struct TestXeGPUPropagateLayouts
292 : public PassWrapper<TestXeGPUPropagateLayouts,
293 OperationPass<gpu::GPUModuleOp>> {
294 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestXeGPUPropagateLayouts)
295
296 StringRef getArgument() const final { return "test-xegpu-propagate-layouts"; }
297
298 StringRef getDescription() const final {
299 return "Test the implementation of XeGPU propagate layouts.";
300 }
301
302 void getDependentDialects(::mlir::DialectRegistry &registry) const override {
303 registry.insert<xegpu::XeGPUDialect>();
304 registry.insert<gpu::GPUDialect>();
305 }
306
307 TestXeGPUPropagateLayouts() = default;
308 TestXeGPUPropagateLayouts(const TestXeGPUPropagateLayouts &pass)
309 : PassWrapper(pass) {}
310
311 Option<std::string> layoutKind{
312 *this, "layout-kind",
313 llvm::cl::desc("Propagate `subgroup` / `inst` / `lane` level of xegpu "
314 "layouts."),
315 llvm::cl::init(Val: "lane")};
316
317 void runOnOperation() override {
318 OpBuilder builder(getOperation());
319 LayoutKind kind;
320 if (layoutKind == "subgroup")
321 kind = LayoutKind::Subgroup;
322 else if (layoutKind == "inst")
323 kind = LayoutKind::InstData;
324 else if (layoutKind == "lane")
325 kind = LayoutKind::Lane;
326 else {
327 signalPassFailure();
328 return;
329 }
330 if (failed(Result: xegpu::propagateLayouts(builder, target: getOperation(), layoutKind: kind, indexBitWidth: 32))) {
331 signalPassFailure();
332 }
333 }
334};
335
336struct TestXeGPUResolveLayoutConflicts
337 : public PassWrapper<TestXeGPUResolveLayoutConflicts,
338 OperationPass<gpu::GPUModuleOp>> {
339 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestXeGPUResolveLayoutConflicts)
340
341 StringRef getArgument() const final {
342 return "test-xegpu-resolve-layout-conflicts";
343 }
344
345 StringRef getDescription() const final {
346 return "Test the implementation of XeGPU layout conflict resolution.";
347 }
348
349 void getDependentDialects(::mlir::DialectRegistry &registry) const override {
350 registry.insert<xegpu::XeGPUDialect>();
351 registry.insert<gpu::GPUDialect>();
352 }
353
354 TestXeGPUResolveLayoutConflicts() = default;
355 TestXeGPUResolveLayoutConflicts(const TestXeGPUResolveLayoutConflicts &pass) =
356 default;
357
358 void runOnOperation() override {
359 if (failed(Result: xegpu::resolveLayoutConflicts(target: getOperation())))
360 signalPassFailure();
361 }
362};
363
364struct TestXeGPURematerializeConversionSources
365 : public PassWrapper<TestXeGPURematerializeConversionSources,
366 OperationPass<gpu::GPUModuleOp>> {
367 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
368 TestXeGPURematerializeConversionSources)
369
370 StringRef getArgument() const final {
371 return "test-xegpu-rematerialize-conversion-sources";
372 }
373
374 StringRef getDescription() const final {
375 return "Test replacing XeGPU layout conversions of trivially "
376 "rematerializable values by recomputation.";
377 }
378
379 void getDependentDialects(::mlir::DialectRegistry &registry) const override {
380 registry.insert<xegpu::XeGPUDialect>();
381 registry.insert<gpu::GPUDialect>();
382 registry.insert<vector::VectorDialect>();
383 }
384
385 TestXeGPURematerializeConversionSources() = default;
386 TestXeGPURematerializeConversionSources(
387 const TestXeGPURematerializeConversionSources &pass) = default;
388
389 void runOnOperation() override {
390 OpBuilder builder(&getContext());
391 xegpu::rematerializeConversionSources(builder, target: getOperation());
392 }
393};
394
395struct TestXeGPUSinkElementwiseConversions
396 : public PassWrapper<TestXeGPUSinkElementwiseConversions,
397 OperationPass<gpu::GPUModuleOp>> {
398 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
399 TestXeGPUSinkElementwiseConversions)
400
401 StringRef getArgument() const final {
402 return "test-xegpu-sink-elementwise-conversions";
403 }
404
405 StringRef getDescription() const final {
406 return "Test sinking of XeGPU layout conversions past elementwise ops.";
407 }
408
409 void getDependentDialects(::mlir::DialectRegistry &registry) const override {
410 registry.insert<xegpu::XeGPUDialect>();
411 registry.insert<gpu::GPUDialect>();
412 registry.insert<vector::VectorDialect>();
413 }
414
415 TestXeGPUSinkElementwiseConversions() = default;
416 TestXeGPUSinkElementwiseConversions(
417 const TestXeGPUSinkElementwiseConversions &pass) = default;
418
419 void runOnOperation() override {
420 OpBuilder builder(&getContext());
421 xegpu::sinkElementwiseConversions(builder, target: getOperation());
422 }
423};
424
425struct TestXeGPUArrayLengthOptimization
426 : public PassWrapper<TestXeGPUArrayLengthOptimization,
427 OperationPass<gpu::GPUModuleOp>> {
428 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestXeGPUArrayLengthOptimization)
429
430 StringRef getArgument() const final {
431 return "test-xegpu-array-length-optimization";
432 }
433
434 StringRef getDescription() const final {
435 return "Test XeGPU 2D block array load optimization patterns in isolation";
436 }
437
438 void getDependentDialects(::mlir::DialectRegistry &registry) const override {
439 registry.insert<xegpu::XeGPUDialect>();
440 registry.insert<vector::VectorDialect>();
441 }
442
443 TestXeGPUArrayLengthOptimization() = default;
444 TestXeGPUArrayLengthOptimization(const TestXeGPUArrayLengthOptimization &pass)
445 : PassWrapper(pass) {}
446
447 void runOnOperation() override {
448 RewritePatternSet patterns(&getContext());
449 xegpu::populateXeGPUArrayLengthOptimizationPatterns(patterns);
450 if (failed(Result: applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns))))
451 signalPassFailure();
452 }
453};
454
455struct TestXeGPULayoutInterface
456 : public PassWrapper<TestXeGPULayoutInterface,
457 OperationPass<gpu::GPUModuleOp>> {
458 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestXeGPULayoutInterface)
459
460 StringRef getArgument() const final { return "test-xegpu-layout-interface"; }
461
462 StringRef getDescription() const final {
463 return "Test the implementation of XeGPU Layout interfaces";
464 }
465
466 void getDependentDialects(::mlir::DialectRegistry &registry) const override {
467 registry.insert<arith::ArithDialect>();
468 registry.insert<memref::MemRefDialect>();
469 registry.insert<xegpu::XeGPUDialect>();
470 registry.insert<vector::VectorDialect>();
471 registry.insert<index::IndexDialect>();
472 }
473
474 TestXeGPULayoutInterface() = default;
475 TestXeGPULayoutInterface(const TestXeGPULayoutInterface &pass) = default;
476
477 void runOnOperation() override {
478 MLIRContext *ctx = &getContext();
479
480 TypeConverter typeConverter;
481 auto materializeCast = [&](mlir::OpBuilder &builder, mlir::Type type,
482 mlir::ValueRange inputs,
483 mlir::Location loc) -> mlir::Value {
484 return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: type, operands: inputs)
485 .getResult(i: 0);
486 };
487 typeConverter.addSourceMaterialization(callback&: materializeCast);
488 typeConverter.addTargetMaterialization(callback&: materializeCast);
489
490 RewritePatternSet patterns(ctx);
491 patterns.add<TestStepOpPattern>(arg&: typeConverter, args&: ctx);
492
493 ConversionTarget target(*ctx);
494 auto isLegal = [&](xegpu::SliceAttr layout) -> bool {
495 return !layout || !layout.isForWorkgroup();
496 };
497
498 target.addDynamicallyLegalOp<vector::StepOp>(
499 callback: [&](vector::StepOp op) -> bool {
500 auto layoutName = xegpu::getTemporaryLayoutName(result: op->getResult(idx: 0));
501 auto sliceAttr =
502 op->getDiscardableAttrOfType<xegpu::SliceAttr>(name: layoutName);
503 return isLegal(sliceAttr);
504 });
505
506 target.markUnknownOpDynamicallyLegal(fn: [](Operation *op) { return true; });
507
508 (void)applyPartialConversion(op: getOperation(), target, patterns: std::move(patterns));
509 }
510};
511
512struct TestXeGPUCoalesceGatherScatter
513 : public PassWrapper<TestXeGPUCoalesceGatherScatter, OperationPass<>> {
514 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestXeGPUCoalesceGatherScatter)
515
516 StringRef getArgument() const final {
517 return "test-xegpu-coalesce-gather-scatter";
518 }
519
520 StringRef getDescription() const final {
521 return "Test driver that turns the contiguity attribute into a lane_data "
522 "layout on gather/scatter ops.";
523 }
524
525 void getDependentDialects(::mlir::DialectRegistry &registry) const override {
526 registry.insert<arith::ArithDialect>();
527 registry.insert<vector::VectorDialect>();
528 registry.insert<xegpu::XeGPUDialect>();
529 }
530
531 TestXeGPUCoalesceGatherScatter() = default;
532 TestXeGPUCoalesceGatherScatter(const TestXeGPUCoalesceGatherScatter &pass)
533 : PassWrapper(pass) {}
534
535 Option<unsigned> maxChunkSize{
536 *this, "max-chunk-size",
537 llvm::cl::desc("Upper bound on the produced lane_data FCD."),
538 llvm::cl::init(Val: 8)};
539
540 Option<bool> analyzeOnly{
541 *this, "analyze-only",
542 llvm::cl::desc("Only run the analysis (stamp the contiguity attribute); "
543 "do not apply."),
544 llvm::cl::init(Val: false)};
545
546 void runOnOperation() override {
547 xegpu::runContiguityAnalysis(root: getOperation());
548 if (analyzeOnly)
549 return;
550 getOperation()->walk(callback: [&](Operation *op) {
551 if (auto load = dyn_cast<xegpu::LoadGatherOp>(Val: op))
552 applyContiguity(op: load, maxChunkSize);
553 else if (auto store = dyn_cast<xegpu::StoreScatterOp>(Val: op))
554 applyContiguity(op: store, maxChunkSize);
555 });
556 }
557
558private:
559 /// Build a `lane_layout`/`lane_data`/`inst_data` layout of rank `rank`, with
560 /// the given lane_layout / lane_data on the innermost dim (1 elsewhere).
561 /// `inst_data` is `lane_layout * lane_data` per dim.
562 static xegpu::LayoutAttr buildLaneDataLayout(MLIRContext *ctx, unsigned rank,
563 int64_t innerLaneLayout,
564 int64_t innerLaneData) {
565 SmallVector<int32_t> laneLayout(rank, 1);
566 SmallVector<int32_t> laneData(rank, 1);
567 SmallVector<int32_t> instData(rank, 1);
568 laneLayout.back() = static_cast<int32_t>(innerLaneLayout);
569 laneData.back() = static_cast<int32_t>(innerLaneData);
570 instData.back() = static_cast<int32_t>(innerLaneLayout * innerLaneData);
571 return xegpu::LayoutAttr::get(context: ctx, inst_data: instData, lane_layout: laneLayout, lane_data: laneData);
572 }
573
574 /// Minimal driver: read the `contiguity` attribute the analysis stamped and
575 /// turn it into a `lane_data` layout. This is only a stand-in for the real
576 /// consumer (layout propagation) so the analysis output can be checked
577 /// end-to-end; it handles just the simple power-of-two case.
578 template <typename OpTy>
579 static void applyContiguity(OpTy op, unsigned maxChunkSize) {
580 std::optional<uint64_t> contiguity = op.getContiguity();
581 if (!contiguity)
582 return;
583 op.removeContiguityAttr();
584
585 auto offsetsTy = dyn_cast<VectorType>(op.getOffsets().getType());
586 auto valueTy = op.getValueType();
587 if (!offsetsTy || !valueTy)
588 return;
589 const auto *uArch =
590 xegpu::uArch::getUArch(archName: xegpu::getChipStr(op).value_or(""));
591 if (!uArch)
592 return;
593 int64_t subgroupSize = uArch->getSubgroupSize();
594
595 // Tile size and subgroup size are powers of two, so the smaller is already
596 // the largest power-of-two divisor; min() suffices throughout.
597 int64_t inner = offsetsTy.getShape().back();
598 int64_t laneLayout = std::min<int64_t>(a: subgroupSize, b: inner);
599 int64_t perLane = inner / laneLayout;
600 int64_t laneData =
601 std::min<int64_t>(l: {static_cast<int64_t>(*contiguity),
602 static_cast<int64_t>(maxChunkSize), perLane});
603 if (laneData < 2)
604 return;
605
606 op.setLayoutAttr(buildLaneDataLayout(ctx: op.getContext(), rank: valueTy.getRank(),
607 innerLaneLayout: laneLayout, innerLaneData: laneData));
608 }
609};
610
611} // namespace
612
613namespace mlir {
614namespace test {
615void registerTestXeGPULowerings() {
616 PassRegistration<TestXeGPUUnrollingPatterns>();
617 PassRegistration<TestXeGPULayoutInterface>();
618 PassRegistration<TestXeGPURecoverTemporaryLayouts>();
619 PassRegistration<TestXeGPUSgToLaneDistribute>();
620 PassRegistration<TestXeGPUPropagateLayouts>();
621 PassRegistration<TestXeGPUResolveLayoutConflicts>();
622 PassRegistration<TestXeGPURematerializeConversionSources>();
623 PassRegistration<TestXeGPUSinkElementwiseConversions>();
624 PassRegistration<TestXeGPUArrayLengthOptimization>();
625 PassRegistration<TestXeGPUCoalesceGatherScatter>();
626}
627} // namespace test
628} // namespace mlir
629