| 1 | //===---- XeGPUBlocking.cpp ---- XeGPU Blocking Pass ----------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/XeGPU/Transforms/Passes.h" |
| 10 | |
| 11 | #include "mlir/Dialect/Index/IR/IndexDialect.h" |
| 12 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 13 | #include "mlir/Dialect/SCF/Transforms/Patterns.h" |
| 14 | #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h" |
| 15 | #include "mlir/Dialect/XeGPU/IR/XeGPU.h" |
| 16 | #include "mlir/Dialect/XeGPU/Transforms/Transforms.h" |
| 17 | #include "mlir/Dialect/XeGPU/Transforms/XeGPULayoutImpl.h" |
| 18 | #include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h" |
| 19 | #include "mlir/Interfaces/LoopLikeInterface.h" |
| 20 | #include "mlir/Pass/PassManager.h" |
| 21 | #include "mlir/Transforms/DialectConversion.h" |
| 22 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 23 | #include "llvm/ADT/STLExtras.h" |
| 24 | #include "llvm/ADT/SetVector.h" |
| 25 | #include "llvm/Support/DebugLog.h" |
| 26 | |
| 27 | namespace mlir { |
| 28 | namespace xegpu { |
| 29 | #define GEN_PASS_DEF_XEGPUBLOCKING |
| 30 | #include "mlir/Dialect/XeGPU/Transforms/Passes.h.inc" |
| 31 | } // namespace xegpu |
| 32 | } // namespace mlir |
| 33 | |
| 34 | #define DEBUG_TYPE "xegpu-blocking" |
| 35 | |
| 36 | using namespace mlir; |
| 37 | |
| 38 | namespace { |
| 39 | |
| 40 | //===------------------------------------------------------------------------===// |
| 41 | // The XeGPUBlockingPass leverages the unroll patterns for XeGPU and Vector ops |
| 42 | // to partition operations that process large shapes into multiple operations on |
| 43 | // smaller shapes, as specified by the inst_data in the layout attribute. This |
| 44 | // enables each resulting operation to be efficiently mapped to a hardware |
| 45 | // instruction. |
| 46 | //===------------------------------------------------------------------------===// |
| 47 | |
| 48 | class XeGPUBlockingPass final |
| 49 | : public xegpu::impl::XeGPUBlockingBase<XeGPUBlockingPass> { |
| 50 | public: |
| 51 | void runOnOperation() override; |
| 52 | |
| 53 | private: |
| 54 | // Get the tile shape for a given OpOperand or OpResult by examining the |
| 55 | // corresponding layout attribute. If layout is not present or is not a |
| 56 | // subgroup level layout, it returns std::nullopt. |
| 57 | template <typename T, |
| 58 | typename = std::enable_if_t<std::is_same_v<T, OpOperand> || |
| 59 | std::is_same_v<T, OpResult>>> |
| 60 | std::optional<SmallVector<int64_t>> |
| 61 | getTileShape(const T &operandOrResult) const; |
| 62 | |
| 63 | // Get the tile shape for a given operation. |
| 64 | std::optional<SmallVector<int64_t>> getTileShape(Operation *op) const; |
| 65 | |
| 66 | // Determine if the operation requires unrolling. Return false if all operands |
| 67 | // and results have tile shapes identical to their original types. Otherwise, |
| 68 | // return true. |
| 69 | bool needsUnroll(Operation *op) const; |
| 70 | }; |
| 71 | } // namespace |
| 72 | |
| 73 | template <typename T, typename> |
| 74 | std::optional<SmallVector<int64_t>> |
| 75 | XeGPUBlockingPass::getTileShape(const T &operandOrResult) const { |
| 76 | Value value; |
| 77 | if constexpr (std::is_same_v<T, OpOperand>) { |
| 78 | value = operandOrResult.get(); |
| 79 | } else { |
| 80 | value = (Value)operandOrResult; |
| 81 | } |
| 82 | |
| 83 | xegpu::DistributeLayoutAttr layout = |
| 84 | xegpu::getDistributeLayoutAttr(operandOrResult); |
| 85 | if (layout && layout.isForSubgroup()) { |
| 86 | if (!layout.getEffectiveInstDataAsInt().empty()) { |
| 87 | SmallVector<int64_t> instData = layout.getEffectiveInstDataAsInt(); |
| 88 | return instData; |
| 89 | } |
| 90 | if (auto type = dyn_cast<ShapedType>(Val: value.getType())) |
| 91 | return llvm::to_vector(Range: type.getShape()); |
| 92 | } |
| 93 | LDBG() << "failed to getTileShape for: " << value; |
| 94 | return std::nullopt; |
| 95 | } |
| 96 | |
| 97 | std::optional<SmallVector<int64_t>> |
| 98 | XeGPUBlockingPass::getTileShape(Operation *op) const { |
| 99 | if (isa<xegpu::CreateNdDescOp, xegpu::LoadMatrixOp>(Val: op)) |
| 100 | return getTileShape(operandOrResult: op->getOpResult(idx: 0)); |
| 101 | if (isa<xegpu::PrefetchNdOp, xegpu::LoadNdOp, xegpu::PrefetchOp, |
| 102 | xegpu::StoreMatrixOp>(Val: op)) |
| 103 | return getTileShape(operandOrResult: op->getOpOperand(idx: 0)); |
| 104 | if (isa<xegpu::StoreNdOp>(Val: op)) |
| 105 | return getTileShape(operandOrResult: op->getOpOperand(idx: 1)); |
| 106 | |
| 107 | if (isa<xegpu::LoadGatherOp>(Val: op)) |
| 108 | return getTileShape(operandOrResult: op->getOpResult(idx: 0)); |
| 109 | |
| 110 | if (auto convertLayoutOp = dyn_cast<xegpu::ConvertLayoutOp>(Val: op)) { |
| 111 | auto inputInstData = |
| 112 | convertLayoutOp.getEffectiveInputLayout().getEffectiveInstDataAsInt(); |
| 113 | auto targetInstData = |
| 114 | convertLayoutOp.getTargetLayout().getEffectiveInstDataAsInt(); |
| 115 | assert(inputInstData.size() == targetInstData.size() && |
| 116 | "convert_layout layouts must both carry inst_data of the same rank" ); |
| 117 | SmallVector<int64_t> tile(inputInstData.size()); |
| 118 | for (size_t i = 0; i < tile.size(); ++i) |
| 119 | tile[i] = std::max(a: inputInstData[i], b: targetInstData[i]); |
| 120 | return tile; |
| 121 | } |
| 122 | |
| 123 | if (isa<xegpu::StoreScatterOp>(Val: op)) |
| 124 | return getTileShape(operandOrResult: op->getOpOperand(idx: 0)); |
| 125 | |
| 126 | // Helper lambda to validate and get A/B tiles |
| 127 | auto validateABTiles = [&](Operation *op) |
| 128 | -> std::optional<std::pair<SmallVector<int64_t>, SmallVector<int64_t>>> { |
| 129 | std::optional<SmallVector<int64_t>> aTile = |
| 130 | getTileShape(operandOrResult: op->getOpOperand(idx: 0)); |
| 131 | std::optional<SmallVector<int64_t>> bTile = |
| 132 | getTileShape(operandOrResult: op->getOpOperand(idx: 1)); |
| 133 | |
| 134 | if (!aTile || aTile->size() < 2 || !bTile || bTile->size() < 2) |
| 135 | return std::nullopt; |
| 136 | |
| 137 | // Both must have the same number of batch dimensions. |
| 138 | int64_t aBatchRank = aTile->size() - 2; |
| 139 | int64_t bBatchRank = bTile->size() - 2; |
| 140 | if (aBatchRank != bBatchRank) |
| 141 | return std::nullopt; |
| 142 | |
| 143 | // Batch dimensions must match. |
| 144 | for (int64_t i = 0; i < aBatchRank; ++i) { |
| 145 | if ((*aTile)[i] != (*bTile)[i]) |
| 146 | return std::nullopt; |
| 147 | } |
| 148 | |
| 149 | // Semantic check for A and B: K dimension must match. |
| 150 | // A[..., M, K] x B[..., K, N] |
| 151 | if ((*aTile).back() != (*bTile)[bBatchRank]) |
| 152 | return std::nullopt; |
| 153 | |
| 154 | return std::make_pair(x&: *aTile, y&: *bTile); |
| 155 | }; |
| 156 | |
| 157 | // Helper lambda to validate C tile |
| 158 | auto validateCTile = [&](Operation *op, unsigned cOperandIdx, |
| 159 | const SmallVector<int64_t> &aTile, |
| 160 | const SmallVector<int64_t> &bTile) -> bool { |
| 161 | if (op->getNumOperands() <= cOperandIdx) |
| 162 | return true; |
| 163 | |
| 164 | std::optional<SmallVector<int64_t>> cTile = |
| 165 | getTileShape(operandOrResult: op->getOpOperand(idx: cOperandIdx)); |
| 166 | if (!cTile) |
| 167 | return false; |
| 168 | // Expected C tile: batch dims from A + [M, N] |
| 169 | int64_t aBatchRank = aTile.size() - 2; |
| 170 | SmallVector<int64_t> expectedCTile(aTile.begin(), |
| 171 | aTile.begin() + aBatchRank); |
| 172 | expectedCTile.push_back(Elt: aTile[aBatchRank]); // M from A |
| 173 | expectedCTile.push_back(Elt: bTile.back()); // N from B |
| 174 | if (!llvm::equal(LRange&: *cTile, RRange&: expectedCTile)) |
| 175 | return false; |
| 176 | return true; |
| 177 | }; |
| 178 | |
| 179 | // Helper lambda to validate scale A tile for DpasMxOp |
| 180 | auto validateScaleATile = |
| 181 | [&](Operation *op, unsigned scaleAOperandIdx, |
| 182 | const SmallVector<int64_t> &aTile) -> std::optional<int64_t> { |
| 183 | std::optional<SmallVector<int64_t>> aScaleTile = |
| 184 | getTileShape(operandOrResult: op->getOpOperand(idx: scaleAOperandIdx)); |
| 185 | |
| 186 | if (!aScaleTile || aScaleTile->size() < 2) |
| 187 | return std::nullopt; |
| 188 | |
| 189 | // Validate scale_a tile: [batch..., M_tile, K_scale] |
| 190 | // M dimension (second-to-last) must match A's M dimension |
| 191 | int64_t scaleRank = aScaleTile->size(); |
| 192 | int64_t aBatchRank = aTile.size() - 2; |
| 193 | if ((*aScaleTile)[scaleRank - 2] != aTile[aBatchRank]) |
| 194 | return std::nullopt; |
| 195 | |
| 196 | // Return the K scale factor (last dim) |
| 197 | return aScaleTile->back(); |
| 198 | }; |
| 199 | |
| 200 | // Helper lambda to validate scale B tile for DpasMxOp |
| 201 | auto validateScaleBTile = |
| 202 | [&](Operation *op, unsigned scaleBOperandIdx, |
| 203 | const SmallVector<int64_t> &bTile) -> std::optional<int64_t> { |
| 204 | std::optional<SmallVector<int64_t>> bScaleTile = |
| 205 | getTileShape(operandOrResult: op->getOpOperand(idx: scaleBOperandIdx)); |
| 206 | |
| 207 | if (!bScaleTile || bScaleTile->size() < 2) |
| 208 | return std::nullopt; |
| 209 | |
| 210 | // Validate scale_b tile: [batch..., K_scale, N_tile] |
| 211 | // N dimension (last) must match B's N dimension (last) |
| 212 | if (bScaleTile->back() != bTile.back()) |
| 213 | return std::nullopt; |
| 214 | |
| 215 | // Return the K scale factor (second-to-last dim) |
| 216 | int64_t scaleRank = bScaleTile->size(); |
| 217 | return (*bScaleTile)[scaleRank - 2]; |
| 218 | }; |
| 219 | |
| 220 | if (isa<xegpu::DpasOp>(Val: op)) { |
| 221 | auto abTiles = validateABTiles(op); |
| 222 | if (!abTiles) |
| 223 | return std::nullopt; |
| 224 | |
| 225 | auto [aTile, bTile] = *abTiles; |
| 226 | |
| 227 | // Semantic check for C. |
| 228 | if (!validateCTile(op, 2, aTile, bTile)) |
| 229 | return std::nullopt; |
| 230 | |
| 231 | // Return [batch..., M, K, N] as the target shape for unrolling. |
| 232 | int64_t aBatchRank = aTile.size() - 2; |
| 233 | SmallVector<int64_t> tileShape(aTile.begin(), aTile.begin() + aBatchRank); |
| 234 | tileShape.push_back(Elt: aTile[aBatchRank]); // M |
| 235 | tileShape.push_back(Elt: aTile[aBatchRank + 1]); // K |
| 236 | tileShape.push_back(Elt: bTile.back()); // N |
| 237 | return tileShape; |
| 238 | } |
| 239 | |
| 240 | if (auto dpasMxOp = dyn_cast<xegpu::DpasMxOp>(Val: op)) { |
| 241 | auto abTiles = validateABTiles(op); |
| 242 | if (!abTiles) |
| 243 | return std::nullopt; |
| 244 | |
| 245 | auto [aTile, bTile] = *abTiles; |
| 246 | |
| 247 | // Validate C tile if present using op-specific accessor |
| 248 | if (dpasMxOp.getAcc()) { |
| 249 | unsigned accOperandIdx = 2; // acc is the 3rd operand |
| 250 | if (!validateCTile(op, accOperandIdx, aTile, bTile)) |
| 251 | return std::nullopt; |
| 252 | } |
| 253 | |
| 254 | // Validate scale tiles if present using op-specific accessors |
| 255 | int64_t kScaleFactor = 1; |
| 256 | std::optional<int64_t> scaleAFactor; |
| 257 | std::optional<int64_t> scaleBFactor; |
| 258 | |
| 259 | if (dpasMxOp.getScaleA()) { |
| 260 | unsigned scaleAOperandIdx = 2 + (dpasMxOp.getAcc() ? 1 : 0); |
| 261 | scaleAFactor = validateScaleATile(op, scaleAOperandIdx, aTile); |
| 262 | if (!scaleAFactor) |
| 263 | return std::nullopt; |
| 264 | } |
| 265 | |
| 266 | if (dpasMxOp.getScaleB()) { |
| 267 | unsigned scaleBOperandIdx = |
| 268 | 2 + (dpasMxOp.getAcc() ? 1 : 0) + (dpasMxOp.getScaleA() ? 1 : 0); |
| 269 | scaleBFactor = validateScaleBTile(op, scaleBOperandIdx, bTile); |
| 270 | if (!scaleBFactor) |
| 271 | return std::nullopt; |
| 272 | } |
| 273 | |
| 274 | // If both scales are present, their K dimensions must match |
| 275 | if (scaleAFactor && scaleBFactor) { |
| 276 | if (*scaleAFactor != *scaleBFactor) |
| 277 | return std::nullopt; |
| 278 | kScaleFactor = *scaleAFactor; |
| 279 | } else if (scaleAFactor) { |
| 280 | kScaleFactor = *scaleAFactor; |
| 281 | } else if (scaleBFactor) { |
| 282 | kScaleFactor = *scaleBFactor; |
| 283 | } |
| 284 | |
| 285 | // Return [batch..., M, K, N, S] as the target shape for unrolling. |
| 286 | int64_t aBatchRank = aTile.size() - 2; |
| 287 | SmallVector<int64_t> tileShape(aTile.begin(), aTile.begin() + aBatchRank); |
| 288 | tileShape.push_back(Elt: aTile[aBatchRank]); // M |
| 289 | tileShape.push_back(Elt: aTile[aBatchRank + 1]); // K |
| 290 | tileShape.push_back(Elt: bTile.back()); // N |
| 291 | tileShape.push_back(Elt: kScaleFactor); // S |
| 292 | return tileShape; |
| 293 | } |
| 294 | |
| 295 | if (OpTrait::hasElementwiseMappableTraits(op) && op->getNumResults() == 1) |
| 296 | return getTileShape(operandOrResult: op->getOpResult(idx: 0)); |
| 297 | |
| 298 | if (isa<vector::MultiDimReductionOp>(Val: op)) |
| 299 | return getTileShape(operandOrResult: op->getOpOperand(idx: 0)); |
| 300 | |
| 301 | if (isa<vector::TransposeOp, vector::BroadcastOp, vector::StepOp, |
| 302 | vector::ShapeCastOp, vector::ConstantMaskOp, vector::CreateMaskOp, |
| 303 | vector::BitCastOp, vector::InterleaveOp, vector::DeinterleaveOp>(Val: op)) |
| 304 | return getTileShape(operandOrResult: op->getOpResult(idx: 0)); |
| 305 | |
| 306 | return std::nullopt; |
| 307 | } |
| 308 | |
| 309 | bool XeGPUBlockingPass::needsUnroll(Operation *op) const { |
| 310 | // skip the op if any of its operands or results has workgroup level layouts |
| 311 | bool hasWgLayoutOperands = |
| 312 | llvm::any_of(Range: op->getOpOperands(), P: [](OpOperand &opr) { |
| 313 | xegpu::DistributeLayoutAttr layout = |
| 314 | xegpu::getDistributeLayoutAttr(opr); |
| 315 | return layout && layout.isForWorkgroup(); |
| 316 | }); |
| 317 | bool hasWgLayoutResults = |
| 318 | llvm::any_of(Range: op->getOpResults(), P: [](OpResult result) { |
| 319 | xegpu::DistributeLayoutAttr layout = |
| 320 | xegpu::getDistributeLayoutAttr(value: result); |
| 321 | return layout && layout.isForWorkgroup(); |
| 322 | }); |
| 323 | if (hasWgLayoutOperands || hasWgLayoutResults) { |
| 324 | LDBG() << "skip unrolling for op with workgroup level layout: " << *op; |
| 325 | return false; |
| 326 | } |
| 327 | |
| 328 | auto isUnrollable = [](Value value, ArrayRef<int64_t> tileShape) { |
| 329 | Type valTy = value.getType(); |
| 330 | if (auto tdescTy = dyn_cast<xegpu::TensorDescType>(Val&: valTy)) { |
| 331 | xegpu::DistributeLayoutAttr layout = tdescTy.getLayoutAttr(); |
| 332 | return layout && !layout.getEffectiveInstDataAsInt().empty(); |
| 333 | } |
| 334 | auto shapedType = dyn_cast<ShapedType>(Val&: valTy); |
| 335 | return shapedType && !llvm::equal(LRange&: tileShape, RRange: shapedType.getShape()); |
| 336 | }; |
| 337 | |
| 338 | bool hasUnrollableOperands = |
| 339 | llvm::any_of(Range: op->getOpOperands(), P: [&](OpOperand &opr) { |
| 340 | std::optional<SmallVector<int64_t>> tileShape = getTileShape(operandOrResult: opr); |
| 341 | return tileShape.has_value() && isUnrollable(opr.get(), *tileShape); |
| 342 | }); |
| 343 | bool hasUnrollableResults = |
| 344 | llvm::any_of(Range: op->getOpResults(), P: [&](OpResult result) { |
| 345 | std::optional<SmallVector<int64_t>> tileShape = getTileShape(operandOrResult: result); |
| 346 | return tileShape.has_value() && isUnrollable(result, *tileShape); |
| 347 | }); |
| 348 | // ConvertLayoutOp must be processed to drop the inst_data in the layout |
| 349 | bool isConvertLayoutWithInstData = false; |
| 350 | if (auto convertLayoutOp = dyn_cast<xegpu::ConvertLayoutOp>(Val: op)) { |
| 351 | auto targettLayout = convertLayoutOp.getTargetLayout(); |
| 352 | if (targettLayout && !targettLayout.getEffectiveInstDataAsInt().empty()) { |
| 353 | isConvertLayoutWithInstData = true; |
| 354 | } |
| 355 | } |
| 356 | return hasUnrollableOperands || hasUnrollableResults || |
| 357 | isConvertLayoutWithInstData; |
| 358 | } |
| 359 | |
| 360 | void XeGPUBlockingPass::runOnOperation() { |
| 361 | MLIRContext *ctx = &getContext(); |
| 362 | Operation *op = getOperation(); |
| 363 | |
| 364 | if (!xegpu::recoverTemporaryLayouts(rootOp: op)) { |
| 365 | signalPassFailure(); |
| 366 | return; |
| 367 | } |
| 368 | |
| 369 | auto getTileShapeAndCount = [](llvm::ArrayRef<int64_t> shape, |
| 370 | xegpu::DistributeLayoutAttr layout) { |
| 371 | int count = 1; |
| 372 | SmallVector<int64_t> tileShape(shape); |
| 373 | if (layout && !layout.getEffectiveInstDataAsInt().empty()) { |
| 374 | tileShape = layout.getEffectiveInstDataAsInt(); |
| 375 | count = computeProduct(basis: shape) / computeProduct(basis: tileShape); |
| 376 | } |
| 377 | assert(count >= 1 && "count must be at least 1" ); |
| 378 | return std::make_pair(x&: tileShape, y&: count); |
| 379 | }; |
| 380 | |
| 381 | // Perform context-aware type conversion for SCF structural ops. |
| 382 | // Inspects Values to find inst_data layout information for 1:N conversion. |
| 383 | llvm::SmallSetVector<UnrealizedConversionCastOp, 8> existingCasts; |
| 384 | op->walk( |
| 385 | callback: [&](UnrealizedConversionCastOp castOp) { existingCasts.insert(X: castOp); }); |
| 386 | |
| 387 | { |
| 388 | TypeConverter converter; |
| 389 | converter.addConversion(callback: [](Type type) -> Type { return type; }); |
| 390 | |
| 391 | // TensorDescType 1:N converter (type-based, layout is in the type). |
| 392 | converter.addConversion( |
| 393 | callback: [&](xegpu::TensorDescType type, |
| 394 | SmallVectorImpl<Type> &result) -> std::optional<LogicalResult> { |
| 395 | Type elemTy = type.getElementType(); |
| 396 | ArrayRef<int64_t> shape = type.getShape(); |
| 397 | |
| 398 | xegpu::DistributeLayoutAttr layout = type.getLayoutAttr(); |
| 399 | if (layout && layout.isForWorkgroup()) |
| 400 | return failure(); |
| 401 | |
| 402 | int count; |
| 403 | SmallVector<int64_t> subShape; |
| 404 | std::tie(args&: subShape, args&: count) = getTileShapeAndCount(shape, layout); |
| 405 | |
| 406 | if (layout) |
| 407 | layout = layout.dropInstData(); |
| 408 | |
| 409 | auto newTy = xegpu::TensorDescType::get( |
| 410 | context: type.getContext(), shape: subShape, elementType: elemTy, encoding: type.getEncoding(), layout); |
| 411 | result.append(NumInputs: count, Elt: newTy); |
| 412 | return success(); |
| 413 | }); |
| 414 | |
| 415 | // Context-aware VectorType conversion based on inst_data (1:1 |
| 416 | // shape-changing or 1:N). |
| 417 | auto getSubShapeAndCount = [&](VectorType vecTy, |
| 418 | xegpu::DistributeLayoutAttr layout) |
| 419 | -> std::pair<SmallVector<int64_t>, int> { |
| 420 | return getTileShapeAndCount(vecTy.getShape(), layout); |
| 421 | }; |
| 422 | auto loopArgTypes = |
| 423 | xegpu::precomputeLoopBlockArgTypes(topLevelOp: op, getSubShapeAndCount); |
| 424 | xegpu::addVectorTypeConversion(converter, getSubShapeAndCount, |
| 425 | loopArgTypes: std::move(loopArgTypes)); |
| 426 | |
| 427 | // Loop-carried types are now in the converter's map, so the transient |
| 428 | // per-position layout attrs on SCF loop ops are no longer needed. Strip |
| 429 | // them before converting: the SCF converters copy old attrs onto the new |
| 430 | // op (ConvertForOpTypes::setAttrs), and after 1:N result expansion a stale |
| 431 | // `layout_result_N` lands on the wrong (renumbered) result, corrupting the |
| 432 | // count invariant and leaving the loop illegal. |
| 433 | op->walk(callback: [](Operation *loopOp) { |
| 434 | if (!isa<scf::ForOp, scf::WhileOp, scf::ConditionOp, scf::IfOp>(Val: loopOp)) |
| 435 | return; |
| 436 | SmallVector<StringRef> toRemove; |
| 437 | for (const NamedAttribute &attr : |
| 438 | loopOp->getDiscardableAttrDictionary().getValue()) { |
| 439 | StringRef name = attr.getName().strref(); |
| 440 | if (name.starts_with(Prefix: "layout_operand_" ) || |
| 441 | name.starts_with(Prefix: "layout_result_" )) |
| 442 | toRemove.push_back(Elt: name); |
| 443 | } |
| 444 | for (StringRef name : toRemove) |
| 445 | loopOp->removeDiscardableAttr(name); |
| 446 | }); |
| 447 | |
| 448 | // Source (N:1) and target (1:1) materializations using |
| 449 | // UnrealizedConversionCastOp. |
| 450 | auto materializeCast = [](OpBuilder &builder, Type type, ValueRange inputs, |
| 451 | Location loc) -> Value { |
| 452 | return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: type, operands: inputs) |
| 453 | .getResult(i: 0); |
| 454 | }; |
| 455 | converter.addSourceMaterialization(callback&: materializeCast); |
| 456 | converter.addTargetMaterialization(callback&: materializeCast); |
| 457 | // Blocking runs SCF conversion separately (not combined with XeGPU |
| 458 | // patterns), so it also needs a 1:N target materialization. |
| 459 | converter.addTargetMaterialization( |
| 460 | callback: [](mlir::OpBuilder &builder, mlir::TypeRange types, |
| 461 | mlir::ValueRange inputs, mlir::Location loc) -> SmallVector<Value> { |
| 462 | auto castOp = |
| 463 | UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: types, operands: inputs); |
| 464 | return SmallVector<Value>(castOp.getResults()); |
| 465 | }); |
| 466 | |
| 467 | ConversionTarget target(*ctx); |
| 468 | target.addLegalOp<UnrealizedConversionCastOp>(); |
| 469 | target.markUnknownOpDynamicallyLegal(fn: [](Operation *) { return true; }); |
| 470 | |
| 471 | RewritePatternSet scfPatterns(ctx); |
| 472 | scf::populateSCFStructuralTypeConversionsAndLegality(typeConverter: converter, patterns&: scfPatterns, |
| 473 | target); |
| 474 | if (failed(Result: applyPartialConversion(op, target, patterns: std::move(scfPatterns)))) |
| 475 | return signalPassFailure(); |
| 476 | |
| 477 | // Fold cancelling cast chains and erase dead casts. |
| 478 | xegpu::cleanupUnrealizedConversionCasts(root: op, existingCasts); |
| 479 | } |
| 480 | |
| 481 | xegpu::UnrollOptions options; |
| 482 | options.setFilterConstraint( |
| 483 | [&](Operation *op) -> LogicalResult { return success(IsSuccess: needsUnroll(op)); }); |
| 484 | |
| 485 | options.setNativeShapeFn([&](Operation *op) { return getTileShape(op); }); |
| 486 | |
| 487 | options.setUnrolledTypesFn([&](ShapedType type, ArrayRef<int64_t> tileShape) { |
| 488 | Type elemTy = type.getElementType(); |
| 489 | |
| 490 | if (auto tdescTy = dyn_cast<xegpu::TensorDescType>(Val&: type)) { |
| 491 | |
| 492 | Attribute encoding = tdescTy.getEncoding(); |
| 493 | |
| 494 | xegpu::TensorDescType newTy = |
| 495 | xegpu::TensorDescType::get(context: ctx, shape: tileShape, elementType: elemTy, encoding, |
| 496 | layout: tdescTy.getLayoutAttr().dropInstData()); |
| 497 | // Compute the product of batch (higher) dimensions. |
| 498 | ArrayRef<int64_t> shape = type.getShape(); |
| 499 | int64_t batchCount = |
| 500 | shape.size() > 2 ? computeProduct(basis: shape.drop_back(N: 2)) : 1; |
| 501 | return SmallVector<Type>(batchCount, newTy); |
| 502 | } |
| 503 | Type newTy = VectorType::get(shape: tileShape, elementType: elemTy); |
| 504 | |
| 505 | std::optional<SmallVector<int64_t>> ratio = |
| 506 | computeShapeRatio(shape: type.getShape(), subShape: tileShape); |
| 507 | assert(ratio && "The shape of the type must be a multiple of tileShape." ); |
| 508 | return SmallVector<Type>(computeProduct(basis: *ratio), newTy); |
| 509 | }); |
| 510 | |
| 511 | RewritePatternSet patterns(ctx); |
| 512 | vector::UnrollVectorOptions vectorOptions; |
| 513 | vectorOptions.setNativeShapeFn(options.nativeShape); |
| 514 | |
| 515 | populateXeGPUUnrollPatterns(patterns, options); |
| 516 | vector::populateVectorUnrollPatterns(patterns, options: vectorOptions); |
| 517 | |
| 518 | // Note: The pattern driver does op folding as well and clean up. |
| 519 | // But intermediate insert/extract strided slice ops with |
| 520 | // unrealized conversion cast ops in the middle does not get |
| 521 | // cleaned up in this step. One more round of folding is needed |
| 522 | // after the walk to resolve those unrealized conversion cast ops. |
| 523 | (void)applyPatternsGreedily(op, patterns: std::move(patterns)); |
| 524 | |
| 525 | op->walk(callback: [](Operation *op) { |
| 526 | // Remove the layout attributes cached per operands. |
| 527 | for (OpOperand &opr : op->getOpOperands()) { |
| 528 | std::string name = xegpu::getTemporaryLayoutName(operand: opr); |
| 529 | if (op->hasDiscardableAttrOfType<xegpu::DistributeLayoutAttr>(name)) |
| 530 | op->removeDiscardableAttr(name); |
| 531 | } |
| 532 | |
| 533 | // Update the layout attributes per result. |
| 534 | for (OpResult result : op->getOpResults()) { |
| 535 | std::string name = xegpu::getTemporaryLayoutName(result); |
| 536 | if (auto layout = |
| 537 | op->getDiscardableAttrOfType<xegpu::DistributeLayoutAttr>(name)) { |
| 538 | op->removeDiscardableAttr(name); |
| 539 | if (!isa<LoopLikeOpInterface>(Val: op)) |
| 540 | xegpu::setDistributeLayoutAttr(Result: result, layout: layout.dropInstData()); |
| 541 | } |
| 542 | } |
| 543 | |
| 544 | // Drop left-over inst_data if the unroll pattern does not being applied, |
| 545 | // say, inst_data just matches their shape. |
| 546 | SmallVector<NamedAttribute> newAttrs = xegpu::dropInstDataOnAttrs( |
| 547 | attrs: op->getDiscardableAttrDictionary().getValue()); |
| 548 | op->setDiscardableAttrs(newAttrs); |
| 549 | xegpu::dropInstDataOnInherentAttrs(op); |
| 550 | }); |
| 551 | |
| 552 | // Resolve UnrealizedConversionCastOps generated by SCF structural type |
| 553 | // conversion and by XeGPU/Vector unrolling (cancelling cast chains and |
| 554 | // unpaired pack/unpack casts). |
| 555 | xegpu::cleanupUnrealizedConversionCasts(root: op, existingCasts); |
| 556 | |
| 557 | // One more round of folding to clean up the intermediate |
| 558 | // insert/extract strided slice ops. |
| 559 | RewritePatternSet emptyPatterns(ctx); |
| 560 | (void)applyPatternsGreedily(op, patterns: std::move(emptyPatterns)); |
| 561 | } |
| 562 | |