| 1 | //===- TestXeGPUTransforms.cpp -- Test Vector transforms and lowerings ----===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 10 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 11 | #include "mlir/Dialect/Index/IR/IndexDialect.h" |
| 12 | #include "mlir/Dialect/Math/IR/Math.h" |
| 13 | #include "mlir/Dialect/SCF/Transforms/Patterns.h" |
| 14 | #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h" |
| 15 | #include "mlir/Dialect/XeGPU/IR/XeGPU.h" |
| 16 | #include "mlir/Dialect/XeGPU/Transforms/Transforms.h" |
| 17 | #include "mlir/Dialect/XeGPU/Transforms/XeGPULayoutImpl.h" |
| 18 | #include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h" |
| 19 | #include "mlir/Dialect/XeGPU/uArch/IntelGpuXe2.h" |
| 20 | #include "mlir/IR/BuiltinTypes.h" |
| 21 | #include "mlir/IR/Value.h" |
| 22 | #include "mlir/Pass/Pass.h" |
| 23 | #include "mlir/Pass/PassManager.h" |
| 24 | #include "mlir/Transforms/DialectConversion.h" |
| 25 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 26 | #include "llvm/Support/raw_ostream.h" |
| 27 | #include <optional> |
| 28 | |
| 29 | using namespace mlir; |
| 30 | using namespace mlir::xegpu; |
| 31 | |
| 32 | namespace { |
| 33 | |
| 34 | #define DEBUG_TYPE "test-xegpu-unroll" |
| 35 | |
| 36 | struct TestXeGPUUnrollingPatterns |
| 37 | : public PassWrapper<TestXeGPUUnrollingPatterns, |
| 38 | OperationPass<gpu::GPUModuleOp>> { |
| 39 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestXeGPUUnrollingPatterns) |
| 40 | |
| 41 | StringRef getArgument() const final { |
| 42 | return "test-xegpu-unrolling-patterns" ; |
| 43 | } |
| 44 | |
| 45 | StringRef getDescription() const final { |
| 46 | return "Test lowering patterns to unroll ops in the xegpu dialect" ; |
| 47 | } |
| 48 | |
| 49 | void getDependentDialects(::mlir::DialectRegistry ®istry) const override { |
| 50 | registry.insert<memref::MemRefDialect>(); |
| 51 | registry.insert<xegpu::XeGPUDialect>(); |
| 52 | registry.insert<vector::VectorDialect>(); |
| 53 | } |
| 54 | |
| 55 | TestXeGPUUnrollingPatterns() = default; |
| 56 | TestXeGPUUnrollingPatterns(const TestXeGPUUnrollingPatterns &pass) = default; |
| 57 | |
| 58 | void runOnOperation() override { |
| 59 | MLIRContext *ctx = &getContext(); |
| 60 | xegpu::UnrollOptions options; |
| 61 | options.setNativeShapeFn([&](Operation *op) |
| 62 | -> std::optional<SmallVector<int64_t>> { |
| 63 | if (isa<xegpu::CreateNdDescOp, xegpu::PrefetchNdOp, xegpu::LoadNdOp, |
| 64 | xegpu::StoreNdOp, xegpu::PrefetchOp, xegpu::LoadGatherOp, |
| 65 | xegpu::StoreScatterOp>(Val: op)) { |
| 66 | xegpu::TensorDescType tdescTy; |
| 67 | if (auto createNdOp = dyn_cast<xegpu::CreateNdDescOp>(Val: op)) { |
| 68 | tdescTy = createNdOp.getType(); |
| 69 | } else if (auto prefetchNdOp = dyn_cast<xegpu::PrefetchNdOp>(Val: op)) { |
| 70 | tdescTy = prefetchNdOp.getTensorDescType(); |
| 71 | } else if (auto loadNdOp = dyn_cast<xegpu::LoadNdOp>(Val: op)) { |
| 72 | tdescTy = loadNdOp.getTensorDescType(); |
| 73 | } else if (auto storeNdOp = dyn_cast<xegpu::StoreNdOp>(Val: op)) { |
| 74 | tdescTy = storeNdOp.getTensorDescType(); |
| 75 | } else if (isa<xegpu::PrefetchOp, xegpu::LoadGatherOp, |
| 76 | xegpu::StoreScatterOp>(Val: op)) { |
| 77 | auto anchorOp = cast<xegpu::AnchorLayoutInterface>(Val: op); |
| 78 | auto layout = |
| 79 | dyn_cast_or_null<xegpu::LayoutAttr>(Val: anchorOp.getAnchorLayout()); |
| 80 | if (layout && layout.isForSubgroup()) { |
| 81 | auto inst_data = layout.getEffectiveInstDataAsInt(); |
| 82 | if (!inst_data.empty()) |
| 83 | return SmallVector<int64_t>(inst_data.begin(), inst_data.end()); |
| 84 | } |
| 85 | return std::nullopt; |
| 86 | } |
| 87 | |
| 88 | if (auto layout = tdescTy.getLayoutAttr()) { |
| 89 | auto inst_data = layout.getEffectiveInstDataAsInt(); |
| 90 | if (!inst_data.empty() && layout.isForSubgroup()) |
| 91 | return SmallVector<int64_t>(inst_data.begin(), inst_data.end()); |
| 92 | } |
| 93 | } |
| 94 | |
| 95 | if (isa<xegpu::DpasOp>(Val: op)) |
| 96 | return SmallVector<int64_t>{8, 16, 16}; |
| 97 | |
| 98 | // For vector.multi_reduction, read tile shape from the layout attribute |
| 99 | // on the source operand (layout_operand_0). |
| 100 | if (isa<vector::MultiDimReductionOp>(Val: op)) { |
| 101 | xegpu::DistributeLayoutAttr layout = |
| 102 | xegpu::getDistributeLayoutAttr(opr: op->getOpOperand(idx: 0)); |
| 103 | if (layout) { |
| 104 | auto instData = layout.getEffectiveInstDataAsInt(); |
| 105 | if (!instData.empty()) |
| 106 | return instData; |
| 107 | } |
| 108 | return std::nullopt; |
| 109 | } |
| 110 | |
| 111 | return std::nullopt; |
| 112 | }); |
| 113 | |
| 114 | options.setUnrolledTypesFn( |
| 115 | [&](ShapedType type, ArrayRef<int64_t> tileShape) -> SmallVector<Type> { |
| 116 | Type elemTy = type.getElementType(); |
| 117 | Type newTy; |
| 118 | |
| 119 | // TensorDescType needs to drop the inst_data field in the layout |
| 120 | // attribute |
| 121 | if (auto tdescTy = dyn_cast<xegpu::TensorDescType>(Val&: type)) { |
| 122 | Attribute encoding = tdescTy.getEncoding(); |
| 123 | auto layout = tdescTy.getLayoutAttr(); |
| 124 | |
| 125 | if (layout) { |
| 126 | if (layout.getEffectiveLaneLayoutAsInt().empty()) |
| 127 | layout = xegpu::LayoutAttr(); |
| 128 | else |
| 129 | layout = layout.dropInstData(); |
| 130 | } |
| 131 | |
| 132 | newTy = xegpu::TensorDescType::get(context: ctx, shape: tileShape, elementType: elemTy, encoding, |
| 133 | layout); |
| 134 | // compute the product of batch (higher) dimensions |
| 135 | ArrayRef<int64_t> shape = type.getShape(); |
| 136 | int64_t batchCount = |
| 137 | shape.size() > 2 ? computeProduct(basis: shape.drop_back(N: 2)) : 1; |
| 138 | return SmallVector<Type>(batchCount, newTy); |
| 139 | } |
| 140 | |
| 141 | newTy = type.clone(shape: tileShape, elementType: elemTy); |
| 142 | std::optional<SmallVector<int64_t>> ratio = |
| 143 | computeShapeRatio(shape: type.getShape(), subShape: tileShape); |
| 144 | assert(ratio && "Expecting the ratio to be valid." ); |
| 145 | return SmallVector<Type>(computeProduct(basis: *ratio), newTy); |
| 146 | }); |
| 147 | |
| 148 | RewritePatternSet patterns(ctx); |
| 149 | |
| 150 | populateXeGPUUnrollPatterns(patterns, options); |
| 151 | (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns)); |
| 152 | } |
| 153 | }; |
| 154 | |
| 155 | #undef DEBUG_TYPE |
| 156 | #define DEBUG_TYPE "test-xegpu-layout-interface" |
| 157 | |
| 158 | // Test pattern for distributing vector::StepOp from workgroup to subgroup. |
| 159 | // Validates DistributeLayoutAttr interfaces for offset computation |
| 160 | // abstraction between LayoutAttr and SliceAttr. |
| 161 | class TestStepOpPattern : public OpConversionPattern<vector::StepOp> { |
| 162 | using OpConversionPattern<vector::StepOp>::OpConversionPattern; |
| 163 | |
| 164 | LogicalResult |
| 165 | matchAndRewrite(vector::StepOp op, OneToNOpAdaptor adaptor, |
| 166 | ConversionPatternRewriter &rewriter) const override { |
| 167 | |
| 168 | auto layoutName = xegpu::getTemporaryLayoutName(result: op->getResult(idx: 0)); |
| 169 | auto sliceAttr = op->getDiscardableAttrOfType<xegpu::SliceAttr>(name: layoutName); |
| 170 | if (!sliceAttr || sliceAttr.getRank() != 1) |
| 171 | return failure(); |
| 172 | |
| 173 | std::optional<SmallVector<int64_t>> sgShape = |
| 174 | sliceAttr.getEffectiveSgDataAsInt(); |
| 175 | if (!sgShape) |
| 176 | return failure(); |
| 177 | |
| 178 | Location loc = op.getLoc(); |
| 179 | VectorType type = op.getResult().getType(); |
| 180 | auto wgShape = type.getShape(); |
| 181 | |
| 182 | Value sgId = |
| 183 | gpu::SubgroupIdOp::create(builder&: rewriter, location: loc, /*upper_bound=*/nullptr); |
| 184 | auto maybeOffsets = |
| 185 | sliceAttr.computeDistributedCoords(builder&: rewriter, loc, linearId: sgId, shape: wgShape); |
| 186 | if (failed(Result: maybeOffsets)) |
| 187 | return failure(); |
| 188 | |
| 189 | VectorType newTy = type.cloneWith(shape: *sgShape, elementType: type.getElementType()); |
| 190 | Value base = vector::StepOp::create(builder&: rewriter, location: loc, result: newTy); |
| 191 | SmallVector<Value> newOps; |
| 192 | for (auto offsets : *maybeOffsets) { |
| 193 | Value bcast = |
| 194 | vector::BroadcastOp::create(builder&: rewriter, location: loc, result: newTy, source: offsets[0]); |
| 195 | Value add = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: base, rhs: bcast); |
| 196 | newOps.push_back(Elt: add); |
| 197 | } |
| 198 | rewriter.replaceOpWithMultiple(op, newValues: {newOps}); |
| 199 | return success(); |
| 200 | } |
| 201 | }; |
| 202 | |
| 203 | struct TestXeGPURecoverTemporaryLayouts |
| 204 | : public PassWrapper<TestXeGPURecoverTemporaryLayouts, |
| 205 | OperationPass<gpu::GPUModuleOp>> { |
| 206 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestXeGPURecoverTemporaryLayouts) |
| 207 | |
| 208 | StringRef getArgument() const final { |
| 209 | return "test-xegpu-recover-temporary-layouts" ; |
| 210 | } |
| 211 | |
| 212 | StringRef getDescription() const final { |
| 213 | return "Test the implementation of XeGPU temporary layout recovery" ; |
| 214 | } |
| 215 | |
| 216 | void getDependentDialects(::mlir::DialectRegistry ®istry) const override { |
| 217 | registry.insert<arith::ArithDialect>(); |
| 218 | registry.insert<memref::MemRefDialect>(); |
| 219 | registry.insert<xegpu::XeGPUDialect>(); |
| 220 | registry.insert<vector::VectorDialect>(); |
| 221 | registry.insert<gpu::GPUDialect>(); |
| 222 | } |
| 223 | |
| 224 | TestXeGPURecoverTemporaryLayouts() = default; |
| 225 | TestXeGPURecoverTemporaryLayouts(const TestXeGPURecoverTemporaryLayouts &pass) |
| 226 | : PassWrapper(pass) {} |
| 227 | |
| 228 | void runOnOperation() override { |
| 229 | Operation *op = getOperation(); |
| 230 | if (!xegpu::recoverTemporaryLayouts(rootOp: op)) |
| 231 | signalPassFailure(); |
| 232 | } |
| 233 | }; |
| 234 | |
| 235 | /// This test pass is intended to test the subgroup to lane distribution of |
| 236 | /// xegpu/vector/arith operations in isolation, it does not handle any |
| 237 | /// structural ops like scf.for etc. |
| 238 | struct TestXeGPUSgToLaneDistribute |
| 239 | : public PassWrapper<TestXeGPUSgToLaneDistribute, |
| 240 | OperationPass<gpu::GPUModuleOp>> { |
| 241 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestXeGPUSgToLaneDistribute) |
| 242 | |
| 243 | StringRef getArgument() const final { |
| 244 | return "test-xegpu-sg-to-lane-distribute" ; |
| 245 | } |
| 246 | |
| 247 | StringRef getDescription() const final { |
| 248 | return "Test the implementation of XeGPU Subgroup to Lane Distribution" ; |
| 249 | } |
| 250 | |
| 251 | void getDependentDialects(::mlir::DialectRegistry ®istry) const override { |
| 252 | registry.insert<arith::ArithDialect>(); |
| 253 | registry.insert<memref::MemRefDialect>(); |
| 254 | registry.insert<xegpu::XeGPUDialect>(); |
| 255 | registry.insert<vector::VectorDialect>(); |
| 256 | registry.insert<index::IndexDialect>(); |
| 257 | registry.insert<gpu::GPUDialect>(); |
| 258 | } |
| 259 | |
| 260 | TestXeGPUSgToLaneDistribute() = default; |
| 261 | TestXeGPUSgToLaneDistribute(const TestXeGPUSgToLaneDistribute &pass) |
| 262 | : PassWrapper(pass) {} |
| 263 | |
| 264 | void runOnOperation() override { |
| 265 | Operation *op = getOperation(); |
| 266 | if (!xegpu::recoverTemporaryLayouts(rootOp: op)) { |
| 267 | signalPassFailure(); |
| 268 | return; |
| 269 | } |
| 270 | |
| 271 | MLIRContext *ctx = &getContext(); |
| 272 | TypeConverter typeConverter; |
| 273 | // Define type materializations using UnrealizedConversionCastOp. |
| 274 | auto materializeCast = [&](mlir::OpBuilder &builder, mlir::Type type, |
| 275 | mlir::ValueRange inputs, |
| 276 | mlir::Location loc) -> mlir::Value { |
| 277 | return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: type, operands: inputs) |
| 278 | .getResult(i: 0); |
| 279 | }; |
| 280 | typeConverter.addSourceMaterialization(callback&: materializeCast); |
| 281 | typeConverter.addTargetMaterialization(callback&: materializeCast); |
| 282 | |
| 283 | ConversionTarget target(*ctx); |
| 284 | RewritePatternSet patterns(ctx); |
| 285 | xegpu::populateXeGPUSgToLaneDistributeTypeConversionAndLegality( |
| 286 | typeConverter, patterns, target, topLevelOp: op); |
| 287 | (void)applyPartialConversion(op, target, patterns: std::move(patterns)); |
| 288 | } |
| 289 | }; |
| 290 | |
| 291 | struct TestXeGPUPropagateLayouts |
| 292 | : public PassWrapper<TestXeGPUPropagateLayouts, |
| 293 | OperationPass<gpu::GPUModuleOp>> { |
| 294 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestXeGPUPropagateLayouts) |
| 295 | |
| 296 | StringRef getArgument() const final { return "test-xegpu-propagate-layouts" ; } |
| 297 | |
| 298 | StringRef getDescription() const final { |
| 299 | return "Test the implementation of XeGPU propagate layouts." ; |
| 300 | } |
| 301 | |
| 302 | void getDependentDialects(::mlir::DialectRegistry ®istry) const override { |
| 303 | registry.insert<xegpu::XeGPUDialect>(); |
| 304 | registry.insert<gpu::GPUDialect>(); |
| 305 | } |
| 306 | |
| 307 | TestXeGPUPropagateLayouts() = default; |
| 308 | TestXeGPUPropagateLayouts(const TestXeGPUPropagateLayouts &pass) |
| 309 | : PassWrapper(pass) {} |
| 310 | |
| 311 | Option<std::string> layoutKind{ |
| 312 | *this, "layout-kind" , |
| 313 | llvm::cl::desc("Propagate `subgroup` / `inst` / `lane` level of xegpu " |
| 314 | "layouts." ), |
| 315 | llvm::cl::init(Val: "lane" )}; |
| 316 | |
| 317 | void runOnOperation() override { |
| 318 | OpBuilder builder(getOperation()); |
| 319 | LayoutKind kind; |
| 320 | if (layoutKind == "subgroup" ) |
| 321 | kind = LayoutKind::Subgroup; |
| 322 | else if (layoutKind == "inst" ) |
| 323 | kind = LayoutKind::InstData; |
| 324 | else if (layoutKind == "lane" ) |
| 325 | kind = LayoutKind::Lane; |
| 326 | else { |
| 327 | signalPassFailure(); |
| 328 | return; |
| 329 | } |
| 330 | if (failed(Result: xegpu::propagateLayouts(builder, target: getOperation(), layoutKind: kind, indexBitWidth: 32))) { |
| 331 | signalPassFailure(); |
| 332 | } |
| 333 | } |
| 334 | }; |
| 335 | |
| 336 | struct TestXeGPUResolveLayoutConflicts |
| 337 | : public PassWrapper<TestXeGPUResolveLayoutConflicts, |
| 338 | OperationPass<gpu::GPUModuleOp>> { |
| 339 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestXeGPUResolveLayoutConflicts) |
| 340 | |
| 341 | StringRef getArgument() const final { |
| 342 | return "test-xegpu-resolve-layout-conflicts" ; |
| 343 | } |
| 344 | |
| 345 | StringRef getDescription() const final { |
| 346 | return "Test the implementation of XeGPU layout conflict resolution." ; |
| 347 | } |
| 348 | |
| 349 | void getDependentDialects(::mlir::DialectRegistry ®istry) const override { |
| 350 | registry.insert<xegpu::XeGPUDialect>(); |
| 351 | registry.insert<gpu::GPUDialect>(); |
| 352 | } |
| 353 | |
| 354 | TestXeGPUResolveLayoutConflicts() = default; |
| 355 | TestXeGPUResolveLayoutConflicts(const TestXeGPUResolveLayoutConflicts &pass) = |
| 356 | default; |
| 357 | |
| 358 | void runOnOperation() override { |
| 359 | if (failed(Result: xegpu::resolveLayoutConflicts(target: getOperation()))) |
| 360 | signalPassFailure(); |
| 361 | } |
| 362 | }; |
| 363 | |
| 364 | struct TestXeGPURematerializeConversionSources |
| 365 | : public PassWrapper<TestXeGPURematerializeConversionSources, |
| 366 | OperationPass<gpu::GPUModuleOp>> { |
| 367 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID( |
| 368 | TestXeGPURematerializeConversionSources) |
| 369 | |
| 370 | StringRef getArgument() const final { |
| 371 | return "test-xegpu-rematerialize-conversion-sources" ; |
| 372 | } |
| 373 | |
| 374 | StringRef getDescription() const final { |
| 375 | return "Test replacing XeGPU layout conversions of trivially " |
| 376 | "rematerializable values by recomputation." ; |
| 377 | } |
| 378 | |
| 379 | void getDependentDialects(::mlir::DialectRegistry ®istry) const override { |
| 380 | registry.insert<xegpu::XeGPUDialect>(); |
| 381 | registry.insert<gpu::GPUDialect>(); |
| 382 | registry.insert<vector::VectorDialect>(); |
| 383 | } |
| 384 | |
| 385 | TestXeGPURematerializeConversionSources() = default; |
| 386 | TestXeGPURematerializeConversionSources( |
| 387 | const TestXeGPURematerializeConversionSources &pass) = default; |
| 388 | |
| 389 | void runOnOperation() override { |
| 390 | OpBuilder builder(&getContext()); |
| 391 | xegpu::rematerializeConversionSources(builder, target: getOperation()); |
| 392 | } |
| 393 | }; |
| 394 | |
| 395 | struct TestXeGPUSinkElementwiseConversions |
| 396 | : public PassWrapper<TestXeGPUSinkElementwiseConversions, |
| 397 | OperationPass<gpu::GPUModuleOp>> { |
| 398 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID( |
| 399 | TestXeGPUSinkElementwiseConversions) |
| 400 | |
| 401 | StringRef getArgument() const final { |
| 402 | return "test-xegpu-sink-elementwise-conversions" ; |
| 403 | } |
| 404 | |
| 405 | StringRef getDescription() const final { |
| 406 | return "Test sinking of XeGPU layout conversions past elementwise ops." ; |
| 407 | } |
| 408 | |
| 409 | void getDependentDialects(::mlir::DialectRegistry ®istry) const override { |
| 410 | registry.insert<xegpu::XeGPUDialect>(); |
| 411 | registry.insert<gpu::GPUDialect>(); |
| 412 | registry.insert<vector::VectorDialect>(); |
| 413 | } |
| 414 | |
| 415 | TestXeGPUSinkElementwiseConversions() = default; |
| 416 | TestXeGPUSinkElementwiseConversions( |
| 417 | const TestXeGPUSinkElementwiseConversions &pass) = default; |
| 418 | |
| 419 | void runOnOperation() override { |
| 420 | OpBuilder builder(&getContext()); |
| 421 | xegpu::sinkElementwiseConversions(builder, target: getOperation()); |
| 422 | } |
| 423 | }; |
| 424 | |
| 425 | struct TestXeGPUArrayLengthOptimization |
| 426 | : public PassWrapper<TestXeGPUArrayLengthOptimization, |
| 427 | OperationPass<gpu::GPUModuleOp>> { |
| 428 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestXeGPUArrayLengthOptimization) |
| 429 | |
| 430 | StringRef getArgument() const final { |
| 431 | return "test-xegpu-array-length-optimization" ; |
| 432 | } |
| 433 | |
| 434 | StringRef getDescription() const final { |
| 435 | return "Test XeGPU 2D block array load optimization patterns in isolation" ; |
| 436 | } |
| 437 | |
| 438 | void getDependentDialects(::mlir::DialectRegistry ®istry) const override { |
| 439 | registry.insert<xegpu::XeGPUDialect>(); |
| 440 | registry.insert<vector::VectorDialect>(); |
| 441 | } |
| 442 | |
| 443 | TestXeGPUArrayLengthOptimization() = default; |
| 444 | TestXeGPUArrayLengthOptimization(const TestXeGPUArrayLengthOptimization &pass) |
| 445 | : PassWrapper(pass) {} |
| 446 | |
| 447 | void runOnOperation() override { |
| 448 | RewritePatternSet patterns(&getContext()); |
| 449 | xegpu::populateXeGPUArrayLengthOptimizationPatterns(patterns); |
| 450 | if (failed(Result: applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns)))) |
| 451 | signalPassFailure(); |
| 452 | } |
| 453 | }; |
| 454 | |
| 455 | struct TestXeGPULayoutInterface |
| 456 | : public PassWrapper<TestXeGPULayoutInterface, |
| 457 | OperationPass<gpu::GPUModuleOp>> { |
| 458 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestXeGPULayoutInterface) |
| 459 | |
| 460 | StringRef getArgument() const final { return "test-xegpu-layout-interface" ; } |
| 461 | |
| 462 | StringRef getDescription() const final { |
| 463 | return "Test the implementation of XeGPU Layout interfaces" ; |
| 464 | } |
| 465 | |
| 466 | void getDependentDialects(::mlir::DialectRegistry ®istry) const override { |
| 467 | registry.insert<arith::ArithDialect>(); |
| 468 | registry.insert<memref::MemRefDialect>(); |
| 469 | registry.insert<xegpu::XeGPUDialect>(); |
| 470 | registry.insert<vector::VectorDialect>(); |
| 471 | registry.insert<index::IndexDialect>(); |
| 472 | } |
| 473 | |
| 474 | TestXeGPULayoutInterface() = default; |
| 475 | TestXeGPULayoutInterface(const TestXeGPULayoutInterface &pass) = default; |
| 476 | |
| 477 | void runOnOperation() override { |
| 478 | MLIRContext *ctx = &getContext(); |
| 479 | |
| 480 | TypeConverter typeConverter; |
| 481 | auto materializeCast = [&](mlir::OpBuilder &builder, mlir::Type type, |
| 482 | mlir::ValueRange inputs, |
| 483 | mlir::Location loc) -> mlir::Value { |
| 484 | return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: type, operands: inputs) |
| 485 | .getResult(i: 0); |
| 486 | }; |
| 487 | typeConverter.addSourceMaterialization(callback&: materializeCast); |
| 488 | typeConverter.addTargetMaterialization(callback&: materializeCast); |
| 489 | |
| 490 | RewritePatternSet patterns(ctx); |
| 491 | patterns.add<TestStepOpPattern>(arg&: typeConverter, args&: ctx); |
| 492 | |
| 493 | ConversionTarget target(*ctx); |
| 494 | auto isLegal = [&](xegpu::SliceAttr layout) -> bool { |
| 495 | return !layout || !layout.isForWorkgroup(); |
| 496 | }; |
| 497 | |
| 498 | target.addDynamicallyLegalOp<vector::StepOp>( |
| 499 | callback: [&](vector::StepOp op) -> bool { |
| 500 | auto layoutName = xegpu::getTemporaryLayoutName(result: op->getResult(idx: 0)); |
| 501 | auto sliceAttr = |
| 502 | op->getDiscardableAttrOfType<xegpu::SliceAttr>(name: layoutName); |
| 503 | return isLegal(sliceAttr); |
| 504 | }); |
| 505 | |
| 506 | target.markUnknownOpDynamicallyLegal(fn: [](Operation *op) { return true; }); |
| 507 | |
| 508 | (void)applyPartialConversion(op: getOperation(), target, patterns: std::move(patterns)); |
| 509 | } |
| 510 | }; |
| 511 | |
| 512 | struct TestXeGPUCoalesceGatherScatter |
| 513 | : public PassWrapper<TestXeGPUCoalesceGatherScatter, OperationPass<>> { |
| 514 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestXeGPUCoalesceGatherScatter) |
| 515 | |
| 516 | StringRef getArgument() const final { |
| 517 | return "test-xegpu-coalesce-gather-scatter" ; |
| 518 | } |
| 519 | |
| 520 | StringRef getDescription() const final { |
| 521 | return "Test driver that turns the contiguity attribute into a lane_data " |
| 522 | "layout on gather/scatter ops." ; |
| 523 | } |
| 524 | |
| 525 | void getDependentDialects(::mlir::DialectRegistry ®istry) const override { |
| 526 | registry.insert<arith::ArithDialect>(); |
| 527 | registry.insert<vector::VectorDialect>(); |
| 528 | registry.insert<xegpu::XeGPUDialect>(); |
| 529 | } |
| 530 | |
| 531 | TestXeGPUCoalesceGatherScatter() = default; |
| 532 | TestXeGPUCoalesceGatherScatter(const TestXeGPUCoalesceGatherScatter &pass) |
| 533 | : PassWrapper(pass) {} |
| 534 | |
| 535 | Option<unsigned> maxChunkSize{ |
| 536 | *this, "max-chunk-size" , |
| 537 | llvm::cl::desc("Upper bound on the produced lane_data FCD." ), |
| 538 | llvm::cl::init(Val: 8)}; |
| 539 | |
| 540 | Option<bool> analyzeOnly{ |
| 541 | *this, "analyze-only" , |
| 542 | llvm::cl::desc("Only run the analysis (stamp the contiguity attribute); " |
| 543 | "do not apply." ), |
| 544 | llvm::cl::init(Val: false)}; |
| 545 | |
| 546 | void runOnOperation() override { |
| 547 | xegpu::runContiguityAnalysis(root: getOperation()); |
| 548 | if (analyzeOnly) |
| 549 | return; |
| 550 | getOperation()->walk(callback: [&](Operation *op) { |
| 551 | if (auto load = dyn_cast<xegpu::LoadGatherOp>(Val: op)) |
| 552 | applyContiguity(op: load, maxChunkSize); |
| 553 | else if (auto store = dyn_cast<xegpu::StoreScatterOp>(Val: op)) |
| 554 | applyContiguity(op: store, maxChunkSize); |
| 555 | }); |
| 556 | } |
| 557 | |
| 558 | private: |
| 559 | /// Build a `lane_layout`/`lane_data`/`inst_data` layout of rank `rank`, with |
| 560 | /// the given lane_layout / lane_data on the innermost dim (1 elsewhere). |
| 561 | /// `inst_data` is `lane_layout * lane_data` per dim. |
| 562 | static xegpu::LayoutAttr buildLaneDataLayout(MLIRContext *ctx, unsigned rank, |
| 563 | int64_t innerLaneLayout, |
| 564 | int64_t innerLaneData) { |
| 565 | SmallVector<int32_t> laneLayout(rank, 1); |
| 566 | SmallVector<int32_t> laneData(rank, 1); |
| 567 | SmallVector<int32_t> instData(rank, 1); |
| 568 | laneLayout.back() = static_cast<int32_t>(innerLaneLayout); |
| 569 | laneData.back() = static_cast<int32_t>(innerLaneData); |
| 570 | instData.back() = static_cast<int32_t>(innerLaneLayout * innerLaneData); |
| 571 | return xegpu::LayoutAttr::get(context: ctx, inst_data: instData, lane_layout: laneLayout, lane_data: laneData); |
| 572 | } |
| 573 | |
| 574 | /// Minimal driver: read the `contiguity` attribute the analysis stamped and |
| 575 | /// turn it into a `lane_data` layout. This is only a stand-in for the real |
| 576 | /// consumer (layout propagation) so the analysis output can be checked |
| 577 | /// end-to-end; it handles just the simple power-of-two case. |
| 578 | template <typename OpTy> |
| 579 | static void applyContiguity(OpTy op, unsigned maxChunkSize) { |
| 580 | std::optional<uint64_t> contiguity = op.getContiguity(); |
| 581 | if (!contiguity) |
| 582 | return; |
| 583 | op.removeContiguityAttr(); |
| 584 | |
| 585 | auto offsetsTy = dyn_cast<VectorType>(op.getOffsets().getType()); |
| 586 | auto valueTy = op.getValueType(); |
| 587 | if (!offsetsTy || !valueTy) |
| 588 | return; |
| 589 | const auto *uArch = |
| 590 | xegpu::uArch::getUArch(archName: xegpu::getChipStr(op).value_or("" )); |
| 591 | if (!uArch) |
| 592 | return; |
| 593 | int64_t subgroupSize = uArch->getSubgroupSize(); |
| 594 | |
| 595 | // Tile size and subgroup size are powers of two, so the smaller is already |
| 596 | // the largest power-of-two divisor; min() suffices throughout. |
| 597 | int64_t inner = offsetsTy.getShape().back(); |
| 598 | int64_t laneLayout = std::min<int64_t>(a: subgroupSize, b: inner); |
| 599 | int64_t perLane = inner / laneLayout; |
| 600 | int64_t laneData = |
| 601 | std::min<int64_t>(l: {static_cast<int64_t>(*contiguity), |
| 602 | static_cast<int64_t>(maxChunkSize), perLane}); |
| 603 | if (laneData < 2) |
| 604 | return; |
| 605 | |
| 606 | op.setLayoutAttr(buildLaneDataLayout(ctx: op.getContext(), rank: valueTy.getRank(), |
| 607 | innerLaneLayout: laneLayout, innerLaneData: laneData)); |
| 608 | } |
| 609 | }; |
| 610 | |
| 611 | } // namespace |
| 612 | |
| 613 | namespace mlir { |
| 614 | namespace test { |
| 615 | void registerTestXeGPULowerings() { |
| 616 | PassRegistration<TestXeGPUUnrollingPatterns>(); |
| 617 | PassRegistration<TestXeGPULayoutInterface>(); |
| 618 | PassRegistration<TestXeGPURecoverTemporaryLayouts>(); |
| 619 | PassRegistration<TestXeGPUSgToLaneDistribute>(); |
| 620 | PassRegistration<TestXeGPUPropagateLayouts>(); |
| 621 | PassRegistration<TestXeGPUResolveLayoutConflicts>(); |
| 622 | PassRegistration<TestXeGPURematerializeConversionSources>(); |
| 623 | PassRegistration<TestXeGPUSinkElementwiseConversions>(); |
| 624 | PassRegistration<TestXeGPUArrayLengthOptimization>(); |
| 625 | PassRegistration<TestXeGPUCoalesceGatherScatter>(); |
| 626 | } |
| 627 | } // namespace test |
| 628 | } // namespace mlir |
| 629 | |