| 1 | //===- XeGPUUnroll.cpp - patterns to do unrolling ---------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file contains patterns for unrolling XeGPU operations. It follows a |
| 10 | // similar concept and design as vector unroll patterns, serving as a complement |
| 11 | // to them. |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 16 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 17 | #include "mlir/Dialect/Utils/IndexingUtils.h" |
| 18 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 19 | #include "mlir/Dialect/XeGPU/IR/XeGPU.h" |
| 20 | #include "mlir/Dialect/XeGPU/Transforms/Transforms.h" |
| 21 | #include "mlir/Dialect/XeGPU/Transforms/XeGPULayoutImpl.h" |
| 22 | #include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h" |
| 23 | #include "llvm/ADT/STLExtras.h" |
| 24 | #include "llvm/Support/DebugLog.h" |
| 25 | |
| 26 | namespace mlir { |
| 27 | namespace xegpu { |
| 28 | #define GEN_PASS_DEF_XEGPUUNROLL |
| 29 | #include "mlir/Dialect/XeGPU/Transforms/Passes.h.inc" |
| 30 | } // namespace xegpu |
| 31 | } // namespace mlir |
| 32 | |
| 33 | #define DEBUG_TYPE "xegpu-unroll" |
| 34 | |
| 35 | using namespace mlir; |
| 36 | |
| 37 | namespace { |
| 38 | |
| 39 | // Forward declaration for use inside UnrollPattern below. |
| 40 | SmallVector<Value> |
| 41 | unrollByTile(SmallVector<OpFoldResult> mixedOffsets, |
| 42 | xegpu::TensorDescType tdescTy, ArrayRef<int64_t> targetShape, |
| 43 | const std::function<Value(SmallVector<OpFoldResult>)> &createOp, |
| 44 | Location loc, PatternRewriter &rewriter); |
| 45 | |
| 46 | template <typename SourceOp> |
| 47 | struct UnrollPattern : public OpRewritePattern<SourceOp> { |
| 48 | UnrollPattern(MLIRContext *context, const xegpu::UnrollOptions &options, |
| 49 | PatternBenefit benefit = 1) |
| 50 | : OpRewritePattern<SourceOp>(context, benefit), options(options) {} |
| 51 | |
| 52 | protected: |
| 53 | /// Return the target shape for the given `op`. Return std::nullopt if the |
| 54 | /// op shouldn't be or cannot be unrolled. |
| 55 | std::optional<SmallVector<int64_t>> getTargetShape(Operation *op) const { |
| 56 | LDBG() << "Get unroll shape for: " << *op; |
| 57 | |
| 58 | if (options.filterConstraint && failed(Result: options.filterConstraint(op))) { |
| 59 | LDBG() << "--no filter constraint -> BAIL" ; |
| 60 | return std::nullopt; |
| 61 | } |
| 62 | |
| 63 | assert(options.nativeShape && |
| 64 | "expects the native shape for native shape call back function." ); |
| 65 | auto nativeShape = options.nativeShape(op); |
| 66 | return nativeShape; |
| 67 | } |
| 68 | |
| 69 | SmallVector<Type> getUnrolledTypes(ShapedType type, |
| 70 | ArrayRef<int64_t> tileShape) const { |
| 71 | return options.getUnrolledTypes(type, tileShape); |
| 72 | } |
| 73 | |
| 74 | /// Emulate the the unpack behavior using insert_strided_slice for VectorType |
| 75 | /// values and unrealized_conversion_cast for TensorDescType values. |
| 76 | Value unpack(ValueRange srcs, Type destTy, ArrayRef<int64_t> blockSize, |
| 77 | Location loc, PatternRewriter &rewriter) const { |
| 78 | if (auto vecTy = dyn_cast<VectorType>(Val&: destTy)) { |
| 79 | auto shape = vecTy.getShape(); |
| 80 | return xegpu::createVectorWithShapeFromValues(builder&: rewriter, loc, values: srcs, shape); |
| 81 | } |
| 82 | |
| 83 | if (isa<xegpu::TensorDescType>(Val: destTy)) { |
| 84 | auto attr = NamedAttribute(rewriter.getStringAttr(bytes: unpackAttrName), |
| 85 | rewriter.getUnitAttr()); |
| 86 | auto blkAttr = NamedAttribute(rewriter.getStringAttr(bytes: blockAttrName), |
| 87 | rewriter.getDenseI64ArrayAttr(values: blockSize)); |
| 88 | auto castOp = UnrealizedConversionCastOp::create( |
| 89 | builder&: rewriter, location: loc, resultTypes: destTy, operands: srcs, |
| 90 | attributes: ArrayRef<NamedAttribute>({attr, blkAttr})); |
| 91 | return castOp.getResult(i: 0); |
| 92 | } |
| 93 | |
| 94 | llvm_unreachable("Unexpected destTy." ); |
| 95 | return Value(); |
| 96 | } |
| 97 | |
| 98 | /// Emulate the the pack behavior using extract_strided_slice for VectorType |
| 99 | /// values and unrealized_conversion_cast for TensorDescType values. |
| 100 | SmallVector<Value> pack(Value src, TypeRange destTypes, |
| 101 | ArrayRef<int64_t> blockSize, Location loc, |
| 102 | PatternRewriter &rewriter) const { |
| 103 | if (auto vecTy = dyn_cast<VectorType>(Val: src.getType())) { |
| 104 | return xegpu::extractVectorsWithShapeFromValue(builder&: rewriter, loc, value: src, |
| 105 | shape: blockSize); |
| 106 | } |
| 107 | |
| 108 | if (isa<xegpu::TensorDescType>(Val: src.getType())) { |
| 109 | auto attr = NamedAttribute(rewriter.getStringAttr(bytes: packAttrName), |
| 110 | rewriter.getUnitAttr()); |
| 111 | auto blkAttr = NamedAttribute(rewriter.getStringAttr(bytes: blockAttrName), |
| 112 | rewriter.getDenseI64ArrayAttr(values: blockSize)); |
| 113 | auto castOp = UnrealizedConversionCastOp::create( |
| 114 | builder&: rewriter, location: loc, resultTypes: destTypes, operands: src, |
| 115 | attributes: ArrayRef<NamedAttribute>({attr, blkAttr})); |
| 116 | return castOp.getResults(); |
| 117 | } |
| 118 | |
| 119 | llvm_unreachable("Unexpected src type." ); |
| 120 | return SmallVector<Value>(); |
| 121 | } |
| 122 | |
| 123 | /// Helper to pack operands for DPAS-like operations with early return if |
| 124 | /// no unrolling is needed. |
| 125 | SmallVector<Value> packOperandForDpas(Value operand, |
| 126 | ArrayRef<int64_t> blockSize, |
| 127 | Location loc, |
| 128 | PatternRewriter &rewriter) const { |
| 129 | auto vecType = cast<VectorType>(Val: operand.getType()); |
| 130 | std::optional<SmallVector<int64_t>> grids = |
| 131 | computeShapeRatio(shape: vecType.getShape(), subShape: blockSize); |
| 132 | assert(grids && "Expecting grids to be computed." ); |
| 133 | auto numNewOps = computeProduct(basis: *grids); |
| 134 | if (numNewOps == 1) |
| 135 | return SmallVector<Value>({operand}); |
| 136 | VectorType newVecTy = |
| 137 | vecType.cloneWith(shape: blockSize, elementType: vecType.getElementType()); |
| 138 | SmallVector<Type> convertedTypes(numNewOps, newVecTy); |
| 139 | return pack(src: operand, destTypes: convertedTypes, blockSize, loc, rewriter); |
| 140 | } |
| 141 | |
| 142 | private: |
| 143 | const char *const packAttrName = "__xegpu_blocking_pack__" ; |
| 144 | const char *const unpackAttrName = "__xegpu_blocking_unpack__" ; |
| 145 | const char *const blockAttrName = "__xegpu_blocking_tile_shape__" ; |
| 146 | |
| 147 | xegpu::UnrollOptions options; |
| 148 | }; |
| 149 | |
| 150 | // Walks tile offsets within the tensor descriptor shape and emits one op per |
| 151 | // tile by calling `createOp` with the per-tile offsets. Used by LoadNd, |
| 152 | // StoreNd, CreateNdDesc, and PrefetchNd unrollers, which all need to adjust |
| 153 | // their explicit offsets for each unrolled tile. |
| 154 | SmallVector<Value> |
| 155 | unrollByTile(SmallVector<OpFoldResult> mixedOffsets, |
| 156 | xegpu::TensorDescType tdescTy, ArrayRef<int64_t> targetShape, |
| 157 | const std::function<Value(SmallVector<OpFoldResult>)> &createOp, |
| 158 | Location loc, PatternRewriter &rewriter) { |
| 159 | int64_t rank = tdescTy.getRank(); |
| 160 | ArrayRef<int64_t> shape = tdescTy.getShape(); |
| 161 | |
| 162 | auto addi = [&](OpFoldResult a, int64_t b) -> Value { |
| 163 | std::optional<int64_t> maybeInt = getConstantIntValue(ofr: a); |
| 164 | if (maybeInt) { |
| 165 | return arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: *maybeInt + b); |
| 166 | } else { |
| 167 | auto aV = llvm::cast<Value>(Val&: a); |
| 168 | auto bV = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: b); |
| 169 | return rewriter.createOrFold<arith::AddIOp>(location: loc, args&: aV, args&: bV); |
| 170 | } |
| 171 | }; |
| 172 | |
| 173 | SmallVector<OpFoldResult> oldOffsets = llvm::to_vector( |
| 174 | Range: llvm::drop_begin(RangeOrContainer&: mixedOffsets, N: mixedOffsets.size() - rank)); |
| 175 | auto validIdxes = |
| 176 | llvm::seq<int64_t>(Begin: mixedOffsets.size() - rank, End: mixedOffsets.size()); |
| 177 | |
| 178 | SmallVector<Value> newOps; |
| 179 | for (SmallVector<int64_t> offsets : |
| 180 | StaticTileOffsetRange(shape, targetShape)) { |
| 181 | |
| 182 | for (auto [idx, oldOff, offset] : |
| 183 | llvm::zip(t&: validIdxes, u&: oldOffsets, args&: offsets)) |
| 184 | mixedOffsets[idx] = addi(oldOff, offset); |
| 185 | |
| 186 | auto newOp = createOp(mixedOffsets); |
| 187 | newOps.push_back(Elt: newOp); |
| 188 | } |
| 189 | return newOps; |
| 190 | } |
| 191 | |
| 192 | struct UnrollCreateNdOp : public UnrollPattern<xegpu::CreateNdDescOp> { |
| 193 | using UnrollPattern<xegpu::CreateNdDescOp>::UnrollPattern; |
| 194 | LogicalResult matchAndRewrite(xegpu::CreateNdDescOp op, |
| 195 | PatternRewriter &rewriter) const override { |
| 196 | Location loc = op.getLoc(); |
| 197 | xegpu::TensorDescType tdescTy = op.getType(); |
| 198 | |
| 199 | std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op); |
| 200 | if (!targetShape) |
| 201 | return failure(); |
| 202 | |
| 203 | // Keep the high-D source; only the tile shape shrinks. |
| 204 | Value src = op.getSource(); |
| 205 | auto makeCreateNd = [&](Type tdesc) -> Value { |
| 206 | auto ndTy = cast<xegpu::TensorDescType>(Val&: tdesc); |
| 207 | if (isa<MemRefType>(Val: src.getType())) |
| 208 | return xegpu::CreateNdDescOp::create(builder&: rewriter, location: loc, tdesc: ndTy, |
| 209 | source: cast<TypedValue<MemRefType>>(Val&: src)); |
| 210 | return xegpu::CreateNdDescOp::create( |
| 211 | builder&: rewriter, location: loc, tdesc: ndTy, source: src, shape: op.getMixedSizes(), strides: op.getMixedStrides()); |
| 212 | }; |
| 213 | |
| 214 | SmallVector<Type> newTdescTys = getUnrolledTypes(type: tdescTy, tileShape: *targetShape); |
| 215 | SmallVector<Value> newOps; |
| 216 | if (tdescTy.getRank() <= 2) { |
| 217 | // 2D: one tdesc, broadcast across tiles by pack/unpack. |
| 218 | newOps.push_back(Elt: makeCreateNd(newTdescTys[0])); |
| 219 | } else { |
| 220 | // >2D: one tdesc per tile, so the source count matches the pack count. |
| 221 | for (Type t : newTdescTys) |
| 222 | newOps.push_back(Elt: makeCreateNd(t)); |
| 223 | } |
| 224 | Value castOp = unpack(srcs: newOps, destTy: tdescTy, blockSize: *targetShape, loc, rewriter); |
| 225 | rewriter.replaceOp(op, newValues: castOp); |
| 226 | return success(); |
| 227 | } |
| 228 | }; |
| 229 | |
| 230 | struct UnrollPrefetchNdOp : public UnrollPattern<xegpu::PrefetchNdOp> { |
| 231 | using UnrollPattern<xegpu::PrefetchNdOp>::UnrollPattern; |
| 232 | LogicalResult matchAndRewrite(xegpu::PrefetchNdOp op, |
| 233 | PatternRewriter &rewriter) const override { |
| 234 | Location loc = op.getLoc(); |
| 235 | xegpu::TensorDescType tdescTy = op.getTensorDescType(); |
| 236 | |
| 237 | std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op); |
| 238 | if (!targetShape) |
| 239 | return failure(); |
| 240 | |
| 241 | xegpu::DistributeLayoutAttr layout = op.getLayoutAttr(); |
| 242 | if (layout) |
| 243 | layout = layout.dropInstData(); |
| 244 | |
| 245 | // Batch (leading) dims unroll to unit tiles; one tdesc serves all. |
| 246 | SmallVector<Type> convertedTdescTypes = |
| 247 | getUnrolledTypes(type: tdescTy, tileShape: *targetShape); |
| 248 | SmallVector<Value> convertedTdesc = pack( |
| 249 | src: op.getTensorDesc(), destTypes: convertedTdescTypes, blockSize: *targetShape, loc, rewriter); |
| 250 | |
| 251 | auto createPrefetch = [&](SmallVector<OpFoldResult> offsets) -> Value { |
| 252 | xegpu::PrefetchNdOp::create(builder&: rewriter, location: loc, TensorDesc: convertedTdesc[0], offsets, |
| 253 | l1_hint: op.getL1HintAttr(), l2_hint: op.getL2HintAttr(), |
| 254 | l3_hint: op.getL3HintAttr(), layout); |
| 255 | return nullptr; |
| 256 | }; |
| 257 | unrollByTile(mixedOffsets: op.getMixedOffsets(), tdescTy, targetShape: *targetShape, createOp: createPrefetch, |
| 258 | loc, rewriter); |
| 259 | |
| 260 | rewriter.eraseOp(op); |
| 261 | return success(); |
| 262 | } |
| 263 | }; |
| 264 | |
| 265 | struct UnrollLoadNdOp : public UnrollPattern<xegpu::LoadNdOp> { |
| 266 | using UnrollPattern<xegpu::LoadNdOp>::UnrollPattern; |
| 267 | LogicalResult matchAndRewrite(xegpu::LoadNdOp op, |
| 268 | PatternRewriter &rewriter) const override { |
| 269 | |
| 270 | Location loc = op.getLoc(); |
| 271 | VectorType valueTy = op.getType(); |
| 272 | xegpu::TensorDescType tdescTy = op.getTensorDescType(); |
| 273 | |
| 274 | std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op); |
| 275 | if (!targetShape) |
| 276 | return failure(); |
| 277 | |
| 278 | xegpu::DistributeLayoutAttr layout = op.getLayoutAttr(); |
| 279 | if (layout) |
| 280 | layout = layout.dropInstData(); |
| 281 | |
| 282 | Type elemTy = tdescTy.getElementType(); |
| 283 | VectorType newValueTy = valueTy.cloneWith(shape: *targetShape, elementType: elemTy); |
| 284 | |
| 285 | SmallVector<Value> newOps; |
| 286 | |
| 287 | // Batch (leading) dims unroll to unit tiles; one tdesc serves all. |
| 288 | SmallVector<Type> convertedTdescTypes = |
| 289 | getUnrolledTypes(type: tdescTy, tileShape: *targetShape); |
| 290 | SmallVector<Value> convertedTdescs = pack( |
| 291 | src: op.getTensorDesc(), destTypes: convertedTdescTypes, blockSize: *targetShape, loc, rewriter); |
| 292 | |
| 293 | auto createLoad = [&](SmallVector<OpFoldResult> offsets) -> Value { |
| 294 | return xegpu::LoadNdOp::create( |
| 295 | builder&: rewriter, location: loc, value: newValueTy, TensorDesc: convertedTdescs[0], offsets, |
| 296 | packed: op.getPackedAttr(), transpose: op.getTransposeAttr(), l1_hint: op.getL1HintAttr(), |
| 297 | l2_hint: op.getL2HintAttr(), l3_hint: op.getL3HintAttr(), layout); |
| 298 | }; |
| 299 | newOps = unrollByTile(mixedOffsets: op.getMixedOffsets(), tdescTy, targetShape: *targetShape, |
| 300 | createOp: createLoad, loc, rewriter); |
| 301 | |
| 302 | Value castOp = unpack(srcs: newOps, destTy: op.getType(), blockSize: *targetShape, loc, rewriter); |
| 303 | rewriter.replaceOp(op, newValues: castOp); |
| 304 | return success(); |
| 305 | } |
| 306 | }; |
| 307 | |
| 308 | struct UnrollStoreNdOp : public UnrollPattern<xegpu::StoreNdOp> { |
| 309 | using UnrollPattern<xegpu::StoreNdOp>::UnrollPattern; |
| 310 | LogicalResult matchAndRewrite(xegpu::StoreNdOp op, |
| 311 | PatternRewriter &rewriter) const override { |
| 312 | Location loc = op.getLoc(); |
| 313 | VectorType valueTy = op.getValueType(); |
| 314 | xegpu::TensorDescType tdescTy = op.getTensorDescType(); |
| 315 | |
| 316 | std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op); |
| 317 | if (!targetShape) |
| 318 | return failure(); |
| 319 | |
| 320 | xegpu::DistributeLayoutAttr layout = op.getLayoutAttr(); |
| 321 | if (layout) |
| 322 | layout = layout.dropInstData(); |
| 323 | |
| 324 | SmallVector<Type> convertedValTypes = |
| 325 | getUnrolledTypes(type: valueTy, tileShape: *targetShape); |
| 326 | |
| 327 | SmallVector<Value> convertedValues = |
| 328 | pack(src: op.getValue(), destTypes: convertedValTypes, blockSize: *targetShape, loc, rewriter); |
| 329 | |
| 330 | size_t valueIndex = 0; |
| 331 | |
| 332 | // Batch (leading) dims unroll to unit tiles like any other dim. valueIndex |
| 333 | // advances in unrollByTile's tile order, staying in sync with the packed |
| 334 | // values. |
| 335 | SmallVector<Type> convertedTdescTypes = |
| 336 | getUnrolledTypes(type: tdescTy, tileShape: *targetShape); |
| 337 | SmallVector<Value> convertedTdescs = pack( |
| 338 | src: op.getTensorDesc(), destTypes: convertedTdescTypes, blockSize: *targetShape, loc, rewriter); |
| 339 | |
| 340 | auto createStore = [&](SmallVector<OpFoldResult> offsets) { |
| 341 | xegpu::StoreNdOp::create(builder&: rewriter, location: loc, value: convertedValues[valueIndex++], |
| 342 | TensorDesc: convertedTdescs[0], offsets, l1_hint: op.getL1HintAttr(), |
| 343 | l2_hint: op.getL2HintAttr(), l3_hint: op.getL3HintAttr(), layout); |
| 344 | return (Value) nullptr; |
| 345 | }; |
| 346 | unrollByTile(mixedOffsets: op.getMixedOffsets(), tdescTy, targetShape: *targetShape, createOp: createStore, loc, |
| 347 | rewriter); |
| 348 | |
| 349 | rewriter.eraseOp(op); |
| 350 | return success(); |
| 351 | } |
| 352 | }; |
| 353 | |
| 354 | struct UnrollDpasOp : public UnrollPattern<xegpu::DpasOp> { |
| 355 | using UnrollPattern<xegpu::DpasOp>::UnrollPattern; |
| 356 | LogicalResult matchAndRewrite(xegpu::DpasOp op, |
| 357 | PatternRewriter &rewriter) const override { |
| 358 | Location loc = op.getLoc(); |
| 359 | |
| 360 | std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op); |
| 361 | if (!targetShape || targetShape->size() < 3) |
| 362 | return failure(); |
| 363 | |
| 364 | // targetShape is [batch..., M, K, N] |
| 365 | int64_t tsRank = targetShape->size(); |
| 366 | auto M = (*targetShape)[tsRank - 3]; |
| 367 | auto K = (*targetShape)[tsRank - 2]; |
| 368 | auto N = (*targetShape)[tsRank - 1]; |
| 369 | ArrayRef<int64_t> batchDims(targetShape->data(), tsRank - 3); |
| 370 | |
| 371 | // Build block sizes including batch dimensions. |
| 372 | SmallVector<int64_t> aBlockSize(batchDims); |
| 373 | aBlockSize.push_back(Elt: M); |
| 374 | aBlockSize.push_back(Elt: K); |
| 375 | SmallVector<int64_t> bBlockSize(batchDims); |
| 376 | bBlockSize.push_back(Elt: K); |
| 377 | bBlockSize.push_back(Elt: N); |
| 378 | SmallVector<int64_t> cBlockSize(batchDims); |
| 379 | cBlockSize.push_back(Elt: M); |
| 380 | cBlockSize.push_back(Elt: N); |
| 381 | |
| 382 | auto a = op.getLhs(); |
| 383 | auto b = op.getRhs(); |
| 384 | auto c = op.getAcc(); |
| 385 | |
| 386 | SmallVector<Value> aVals = packOperandForDpas(operand: a, blockSize: aBlockSize, loc, rewriter); |
| 387 | SmallVector<Value> bVals = packOperandForDpas(operand: b, blockSize: bBlockSize, loc, rewriter); |
| 388 | SmallVector<Value> cVals; |
| 389 | if (c) |
| 390 | cVals = packOperandForDpas(operand: c, blockSize: cBlockSize, loc, rewriter); |
| 391 | |
| 392 | auto ranges = c ? SmallVector<ValueRange>({aVals, bVals, cVals}) |
| 393 | : SmallVector<ValueRange>({aVals, bVals}); |
| 394 | if (llvm::any_of(Range&: ranges, P: [](auto &v) { return v.size() == 0; }) || |
| 395 | llvm::all_of(Range&: ranges, P: [](auto &v) { return v.size() == 1; })) |
| 396 | return failure(); |
| 397 | |
| 398 | VectorType resultTy = op.getResult().getType(); |
| 399 | auto vecTy = VectorType::get(shape: cBlockSize, elementType: resultTy.getElementType()); |
| 400 | |
| 401 | auto aShape = a.getType().getShape(); |
| 402 | auto bShape = b.getType().getShape(); |
| 403 | |
| 404 | // Compute iteration counts. Batch dims only iterate over M and N (not |
| 405 | // K-reduction), so compute batch iterations from the C block size. |
| 406 | int64_t batchRank = batchDims.size(); |
| 407 | int64_t mIters = aShape[batchRank] / M; |
| 408 | int64_t kIters = aShape[batchRank + 1] / K; |
| 409 | int64_t nIters = bShape[batchRank + 1] / N; |
| 410 | |
| 411 | // Compute batch iterations (product of batch dim ratios). |
| 412 | int64_t batchIters = 1; |
| 413 | for (int64_t d = 0; d < batchRank; ++d) |
| 414 | batchIters *= aShape[d] / batchDims[d]; |
| 415 | |
| 416 | SmallVector<Value> newOps; |
| 417 | for (int64_t batch = 0; batch < batchIters; ++batch) { |
| 418 | for (int64_t i = 0; i < mIters; ++i) { |
| 419 | for (int64_t j = 0; j < nIters; ++j) { |
| 420 | Value tmpC; |
| 421 | if (c) |
| 422 | tmpC = cVals[batch * (mIters * nIters) + i * nIters + j]; |
| 423 | |
| 424 | for (int64_t k = 0; k < kIters; ++k) { |
| 425 | Value aVec = aVals[batch * (mIters * kIters) + i * kIters + k]; |
| 426 | Value bVec = bVals[batch * (kIters * nIters) + k * nIters + j]; |
| 427 | SmallVector<Value> operands({aVec, bVec}); |
| 428 | if (tmpC) |
| 429 | operands.push_back(Elt: tmpC); |
| 430 | |
| 431 | auto newDpasOp = xegpu::DpasOp::create( |
| 432 | builder&: rewriter, location: loc, resultTypes: TypeRange{vecTy}, operands, properties: op.getProperties(), |
| 433 | discardableAttributes: xegpu::dropInstDataOnAttrs( |
| 434 | attrs: op->getDiscardableAttrDictionary().getValue())); |
| 435 | xegpu::dropInstDataOnInherentAttrs(op: newDpasOp); |
| 436 | tmpC = newDpasOp.getResult(); |
| 437 | } |
| 438 | newOps.push_back(Elt: tmpC); |
| 439 | } |
| 440 | } |
| 441 | } |
| 442 | Value castOp = unpack(srcs: newOps, destTy: resultTy, blockSize: cBlockSize, loc, rewriter); |
| 443 | rewriter.replaceOp(op, newValues: castOp); |
| 444 | return success(); |
| 445 | } |
| 446 | }; |
| 447 | |
| 448 | struct UnrollDpasMxOp : public UnrollPattern<xegpu::DpasMxOp> { |
| 449 | using UnrollPattern<xegpu::DpasMxOp>::UnrollPattern; |
| 450 | LogicalResult matchAndRewrite(xegpu::DpasMxOp op, |
| 451 | PatternRewriter &rewriter) const override { |
| 452 | Location loc = op.getLoc(); |
| 453 | |
| 454 | std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op); |
| 455 | if (!targetShape || targetShape->size() < 4) |
| 456 | return failure(); |
| 457 | |
| 458 | // targetShape is [batch..., M, K, N, S] |
| 459 | int64_t tsRank = targetShape->size(); |
| 460 | auto M = (*targetShape)[tsRank - 4]; |
| 461 | auto K = (*targetShape)[tsRank - 3]; |
| 462 | auto N = (*targetShape)[tsRank - 2]; |
| 463 | auto S = (*targetShape)[tsRank - 1]; |
| 464 | ArrayRef<int64_t> batchDims(targetShape->data(), tsRank - 4); |
| 465 | |
| 466 | SmallVector<int64_t> aBlockSize(batchDims); |
| 467 | aBlockSize.push_back(Elt: M); |
| 468 | aBlockSize.push_back(Elt: K); |
| 469 | SmallVector<int64_t> bBlockSize(batchDims); |
| 470 | bBlockSize.push_back(Elt: K); |
| 471 | bBlockSize.push_back(Elt: N); |
| 472 | SmallVector<int64_t> cBlockSize(batchDims); |
| 473 | cBlockSize.push_back(Elt: M); |
| 474 | cBlockSize.push_back(Elt: N); |
| 475 | SmallVector<int64_t> aScaleBlockSize(batchDims); |
| 476 | aScaleBlockSize.push_back(Elt: M); |
| 477 | aScaleBlockSize.push_back(Elt: S); |
| 478 | SmallVector<int64_t> bScaleBlockSize(batchDims); |
| 479 | bScaleBlockSize.push_back(Elt: S); |
| 480 | bScaleBlockSize.push_back(Elt: N); |
| 481 | |
| 482 | auto a = op.getA(); |
| 483 | auto b = op.getB(); |
| 484 | auto c = op.getAcc(); |
| 485 | auto ascale = dyn_cast<TypedValue<VectorType>>(Val: op.getScaleA()); |
| 486 | auto bscale = dyn_cast<TypedValue<VectorType>>(Val: op.getScaleB()); |
| 487 | |
| 488 | SmallVector<Value> aVals = packOperandForDpas(operand: a, blockSize: aBlockSize, loc, rewriter); |
| 489 | SmallVector<Value> bVals = packOperandForDpas(operand: b, blockSize: bBlockSize, loc, rewriter); |
| 490 | SmallVector<Value> cVals; |
| 491 | if (c) |
| 492 | cVals = packOperandForDpas(operand: c, blockSize: cBlockSize, loc, rewriter); |
| 493 | SmallVector<Value> aScaleVals; |
| 494 | if (ascale) |
| 495 | aScaleVals = packOperandForDpas(operand: ascale, blockSize: aScaleBlockSize, loc, rewriter); |
| 496 | SmallVector<Value> bScaleVals; |
| 497 | if (bscale) |
| 498 | bScaleVals = packOperandForDpas(operand: bscale, blockSize: bScaleBlockSize, loc, rewriter); |
| 499 | |
| 500 | VectorType resultTy = op.getResult().getType(); |
| 501 | auto vecTy = VectorType::get(shape: cBlockSize, elementType: resultTy.getElementType()); |
| 502 | |
| 503 | auto aShape = a.getType().getShape(); |
| 504 | auto bShape = b.getType().getShape(); |
| 505 | int64_t batchRank = batchDims.size(); |
| 506 | int64_t mIters = aShape[batchRank] / M; |
| 507 | int64_t kIters = aShape[batchRank + 1] / K; |
| 508 | int64_t nIters = bShape[batchRank + 1] / N; |
| 509 | |
| 510 | int64_t batchIters = 1; |
| 511 | for (int64_t d = 0; d < batchRank; ++d) |
| 512 | batchIters *= aShape[d] / batchDims[d]; |
| 513 | |
| 514 | SmallVector<Value> newOps; |
| 515 | xegpu::DpasMxOp newDpasMxOp; |
| 516 | for (int64_t batch = 0; batch < batchIters; ++batch) { |
| 517 | for (int64_t i = 0; i < mIters; ++i) { |
| 518 | for (int64_t j = 0; j < nIters; ++j) { |
| 519 | Value tmpC; |
| 520 | if (c) |
| 521 | tmpC = cVals[batch * (mIters * nIters) + i * nIters + j]; |
| 522 | |
| 523 | for (int64_t k = 0; k < kIters; ++k) { |
| 524 | Value aVec = aVals[batch * (mIters * kIters) + i * kIters + k]; |
| 525 | Value bVec = bVals[batch * (kIters * nIters) + k * nIters + j]; |
| 526 | SmallVector<Value> operands({aVec, bVec}); |
| 527 | if (tmpC) |
| 528 | operands.push_back(Elt: tmpC); |
| 529 | if (ascale) |
| 530 | operands.push_back( |
| 531 | Elt: aScaleVals[batch * (mIters * kIters) + i * kIters + k]); |
| 532 | if (bscale) |
| 533 | operands.push_back( |
| 534 | Elt: bScaleVals[batch * (kIters * nIters) + k * nIters + j]); |
| 535 | |
| 536 | newDpasMxOp = xegpu::DpasMxOp::create( |
| 537 | builder&: rewriter, location: loc, resultTypes: TypeRange{vecTy}, operands, properties: op.getProperties(), |
| 538 | discardableAttributes: xegpu::dropInstDataOnAttrs( |
| 539 | attrs: op->getDiscardableAttrDictionary().getValue())); |
| 540 | xegpu::dropInstDataOnInherentAttrs(op: newDpasMxOp); |
| 541 | tmpC = newDpasMxOp.getResult(); |
| 542 | } |
| 543 | newOps.push_back(Elt: newDpasMxOp); |
| 544 | } |
| 545 | } |
| 546 | } |
| 547 | Value castOp = unpack(srcs: newOps, destTy: resultTy, blockSize: cBlockSize, loc, rewriter); |
| 548 | rewriter.replaceOp(op, newValues: castOp); |
| 549 | return success(); |
| 550 | } |
| 551 | }; |
| 552 | |
| 553 | /// This pattern handles the unrolling of LoadGatherOp with offsets (gathered |
| 554 | /// load). |
| 555 | /// It unrolls the offsets and mask operands accordingly, and creates multiple |
| 556 | /// LoadGatherOp with the unrolled operands. |
| 557 | struct UnrollLoadGatherOp : public UnrollPattern<xegpu::LoadGatherOp> { |
| 558 | using UnrollPattern<xegpu::LoadGatherOp>::UnrollPattern; |
| 559 | LogicalResult matchAndRewrite(xegpu::LoadGatherOp op, |
| 560 | PatternRewriter &rewriter) const override { |
| 561 | Location loc = op.getLoc(); |
| 562 | VectorType valueTy = llvm::dyn_cast<VectorType>(Val: op.getType()); |
| 563 | Value offsets = op.getOffsets(); |
| 564 | Value mask = op.getMask(); |
| 565 | |
| 566 | std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op); |
| 567 | if (!targetShape) |
| 568 | return failure(); |
| 569 | |
| 570 | SmallVector<int64_t> targetMaskShape(*targetShape); |
| 571 | |
| 572 | // Unroll mask and offsets with correct shape |
| 573 | VectorType maskTy = llvm::dyn_cast<VectorType>(Val: mask.getType()); |
| 574 | VectorType offsetsTy = llvm::dyn_cast<VectorType>(Val: offsets.getType()); |
| 575 | Type elemTy = valueTy.getElementType(); |
| 576 | VectorType newValueTy = VectorType::get(shape: *targetShape, elementType: elemTy); |
| 577 | |
| 578 | SmallVector<Type> convertedMaskTypes = |
| 579 | getUnrolledTypes(type: maskTy, tileShape: targetMaskShape); |
| 580 | SmallVector<Value> convertedMasks = |
| 581 | pack(src: mask, destTypes: convertedMaskTypes, blockSize: targetMaskShape, loc, rewriter); |
| 582 | |
| 583 | SmallVector<Type> convertedOffsetTypes = |
| 584 | getUnrolledTypes(type: offsetsTy, tileShape: *targetShape); |
| 585 | SmallVector<Value> convertedOffsets = |
| 586 | pack(src: offsets, destTypes: convertedOffsetTypes, blockSize: *targetShape, loc, rewriter); |
| 587 | |
| 588 | auto layout = op.getLayoutAttr(); |
| 589 | if (layout) |
| 590 | layout = layout.dropInstData(); |
| 591 | |
| 592 | SmallVector<Value> newOps; |
| 593 | for (auto [o, m] : llvm::zip(t&: convertedOffsets, u&: convertedMasks)) { |
| 594 | auto newOp = xegpu::LoadGatherOp::create( |
| 595 | builder&: rewriter, location: loc, value: newValueTy, source: op.getSource(), offsets: o, mask: m, l1_hint: op.getL1HintAttr(), |
| 596 | l2_hint: op.getL2HintAttr(), l3_hint: op.getL3HintAttr(), layout, |
| 597 | /*contiguity=*/nullptr); |
| 598 | newOps.push_back(Elt: newOp); |
| 599 | } |
| 600 | |
| 601 | Value castOp = unpack(srcs: newOps, destTy: op.getType(), blockSize: *targetShape, loc, rewriter); |
| 602 | rewriter.replaceOp(op, newValues: castOp); |
| 603 | return success(); |
| 604 | } |
| 605 | }; |
| 606 | |
| 607 | /// This pattern handles the unrolling of StoreScatterOp with offsets (scattered |
| 608 | /// store). |
| 609 | /// It unrolls the offsets and mask operands accordingly, and creates multiple |
| 610 | /// StoreScatterOp with the unrolled operands. |
| 611 | struct UnrollStoreScatterOp : public UnrollPattern<xegpu::StoreScatterOp> { |
| 612 | using UnrollPattern<xegpu::StoreScatterOp>::UnrollPattern; |
| 613 | LogicalResult matchAndRewrite(xegpu::StoreScatterOp op, |
| 614 | PatternRewriter &rewriter) const override { |
| 615 | Location loc = op.getLoc(); |
| 616 | VectorType valueTy = llvm::dyn_cast<VectorType>(Val: op.getValue().getType()); |
| 617 | Value offsets = op.getOffsets(); |
| 618 | Value mask = op.getMask(); |
| 619 | |
| 620 | std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op); |
| 621 | if (!targetShape) |
| 622 | return failure(); |
| 623 | |
| 624 | SmallVector<int64_t> targetMaskShape(*targetShape); |
| 625 | VectorType maskTy = llvm::dyn_cast<VectorType>(Val: mask.getType()); |
| 626 | VectorType offsetsTy = llvm::dyn_cast<VectorType>(Val: offsets.getType()); |
| 627 | |
| 628 | SmallVector<Type> convertedMaskTypes = |
| 629 | getUnrolledTypes(type: maskTy, tileShape: targetMaskShape); |
| 630 | SmallVector<Value> convertedMasks = |
| 631 | pack(src: mask, destTypes: convertedMaskTypes, blockSize: targetMaskShape, loc, rewriter); |
| 632 | |
| 633 | SmallVector<Type> convertedOffsetTypes = |
| 634 | getUnrolledTypes(type: offsetsTy, tileShape: *targetShape); |
| 635 | SmallVector<Value> convertedOffsets = |
| 636 | pack(src: offsets, destTypes: convertedOffsetTypes, blockSize: *targetShape, loc, rewriter); |
| 637 | |
| 638 | SmallVector<Type> convertedValTypes = |
| 639 | getUnrolledTypes(type: valueTy, tileShape: *targetShape); |
| 640 | SmallVector<Value> convertedValues = |
| 641 | pack(src: op.getValue(), destTypes: convertedValTypes, blockSize: *targetShape, loc, rewriter); |
| 642 | |
| 643 | auto layout = op.getLayoutAttr(); |
| 644 | if (layout) |
| 645 | layout = layout.dropInstData(); |
| 646 | |
| 647 | for (auto [v, o, m] : |
| 648 | llvm::zip(t&: convertedValues, u&: convertedOffsets, args&: convertedMasks)) { |
| 649 | xegpu::StoreScatterOp::create(builder&: rewriter, location: loc, value: v, dest: op.getDest(), offsets: o, mask: m, |
| 650 | l1_hint: op.getL1HintAttr(), l2_hint: op.getL2HintAttr(), |
| 651 | l3_hint: op.getL3HintAttr(), layout, |
| 652 | /*contiguity=*/nullptr); |
| 653 | } |
| 654 | |
| 655 | rewriter.eraseOp(op); |
| 656 | return success(); |
| 657 | } |
| 658 | }; |
| 659 | |
| 660 | struct UnrollLoadMatrixOp : public UnrollPattern<xegpu::LoadMatrixOp> { |
| 661 | using UnrollPattern<xegpu::LoadMatrixOp>::UnrollPattern; |
| 662 | LogicalResult matchAndRewrite(xegpu::LoadMatrixOp op, |
| 663 | PatternRewriter &rewriter) const override { |
| 664 | Location loc = op.getLoc(); |
| 665 | VectorType valueTy = llvm::dyn_cast<VectorType>(Val: op.getType()); |
| 666 | assert(valueTy && "the value type must be vector type!" ); |
| 667 | |
| 668 | std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op); |
| 669 | if (!targetShape || targetShape->size() != (size_t)valueTy.getRank()) |
| 670 | return failure(); |
| 671 | |
| 672 | Type elemTy = valueTy.getElementType(); |
| 673 | ArrayRef<int64_t> shape = valueTy.getShape(); |
| 674 | xegpu::DistributeLayoutAttr layout = op.getLayoutAttr(); |
| 675 | |
| 676 | VectorType newValueTy = valueTy.cloneWith(shape: *targetShape, elementType: elemTy); |
| 677 | |
| 678 | SmallVector<OpFoldResult> mixedOffsets = op.getMixedOffsets(); |
| 679 | SmallVector<SmallVector<OpFoldResult>> offsetsList; |
| 680 | for (SmallVector<int64_t> offsets : |
| 681 | StaticTileOffsetRange(shape, *targetShape)) { |
| 682 | auto adds = xegpu::addElementwise( |
| 683 | builder&: rewriter, loc, lhs: mixedOffsets, |
| 684 | rhs: getAsIndexOpFoldResult(ctx: op.getContext(), values: offsets)); |
| 685 | offsetsList.push_back(Elt: adds); |
| 686 | } |
| 687 | |
| 688 | SmallVector<Value> newOps; |
| 689 | if (layout) |
| 690 | layout = layout.dropInstData(); |
| 691 | for (SmallVector<OpFoldResult> offsets : offsetsList) { |
| 692 | auto newOp = xegpu::LoadMatrixOp::create( |
| 693 | builder&: rewriter, location: op.getLoc(), res: newValueTy, mem_desc: op.getMemDesc(), offsets, layout); |
| 694 | newOps.push_back(Elt: newOp); |
| 695 | } |
| 696 | Value castOp = unpack(srcs: newOps, destTy: op.getType(), blockSize: *targetShape, loc, rewriter); |
| 697 | rewriter.replaceOp(op, newValues: castOp); |
| 698 | return success(); |
| 699 | } |
| 700 | }; |
| 701 | |
| 702 | struct UnrollStoreMatrixOp : public UnrollPattern<xegpu::StoreMatrixOp> { |
| 703 | using UnrollPattern<xegpu::StoreMatrixOp>::UnrollPattern; |
| 704 | LogicalResult matchAndRewrite(xegpu::StoreMatrixOp op, |
| 705 | PatternRewriter &rewriter) const override { |
| 706 | std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op); |
| 707 | if (!targetShape) |
| 708 | return failure(); |
| 709 | |
| 710 | Location loc = op.getLoc(); |
| 711 | VectorType valueTy = llvm::dyn_cast<VectorType>(Val: op.getData().getType()); |
| 712 | assert(valueTy && "the value type must be vector type!" ); |
| 713 | ArrayRef<int64_t> shape = valueTy.getShape(); |
| 714 | xegpu::DistributeLayoutAttr layout = op.getLayoutAttr(); |
| 715 | if (layout) |
| 716 | layout = layout.dropInstData(); |
| 717 | |
| 718 | SmallVector<Type> convertedValTypes = |
| 719 | getUnrolledTypes(type: valueTy, tileShape: *targetShape); |
| 720 | SmallVector<Value> convertedValues = |
| 721 | pack(src: op.getData(), destTypes: convertedValTypes, blockSize: *targetShape, loc, rewriter); |
| 722 | |
| 723 | SmallVector<OpFoldResult> mixedOffsets = op.getMixedOffsets(); |
| 724 | SmallVector<SmallVector<OpFoldResult>> offsetsList; |
| 725 | for (SmallVector<int64_t> offsets : |
| 726 | StaticTileOffsetRange(shape, *targetShape)) { |
| 727 | auto adds = xegpu::addElementwise( |
| 728 | builder&: rewriter, loc, lhs: mixedOffsets, |
| 729 | rhs: getAsIndexOpFoldResult(ctx: op.getContext(), values: offsets)); |
| 730 | offsetsList.push_back(Elt: adds); |
| 731 | } |
| 732 | |
| 733 | for (auto [v, offsets] : llvm::zip_equal(t&: convertedValues, u&: offsetsList)) |
| 734 | xegpu::StoreMatrixOp::create(builder&: rewriter, location: loc, data: v, mem_desc: op.getMemDesc(), offsets, |
| 735 | layout); |
| 736 | |
| 737 | rewriter.eraseOp(op); |
| 738 | return success(); |
| 739 | } |
| 740 | }; |
| 741 | |
| 742 | /// UnrollConvertLayoutOp pattern for unrolling xegpu::ConvertLayoutOp |
| 743 | /// operations. It first check whether the convert layout op has valid layouts |
| 744 | /// after inst_data stripped. If it does, it will unroll the vector into |
| 745 | /// multiple smaller vectors according to the target shape, and create multiple |
| 746 | /// ConvertLayoutOp with the unrolled vectors and the stripped layouts. |
| 747 | /// |
| 748 | /// When the input and target layouts have different inst_data, the source is |
| 749 | /// extracted at the input inst_data granularity and the result is inserted at |
| 750 | /// the target inst_data granularity, enabling slice cancellation during |
| 751 | /// canonicalization. |
| 752 | struct UnrollConvertLayoutOp : public UnrollPattern<xegpu::ConvertLayoutOp> { |
| 753 | using UnrollPattern<xegpu::ConvertLayoutOp>::UnrollPattern; |
| 754 | |
| 755 | /// Extracts source in `inTile` slices, regroups into `convTile`-sized |
| 756 | /// ConvertLayoutOps, and inserts the result in `outTile` slices. |
| 757 | /// Returns failure if the tiles do not evenly divide. |
| 758 | LogicalResult |
| 759 | rewriteWithRegrouping(xegpu::ConvertLayoutOp op, VectorType valueTy, |
| 760 | ArrayRef<int64_t> convTile, ArrayRef<int64_t> inTile, |
| 761 | ArrayRef<int64_t> outTile, |
| 762 | xegpu::DistributeLayoutAttr inputLayout, |
| 763 | xegpu::DistributeLayoutAttr targetLayout, Location loc, |
| 764 | PatternRewriter &rewriter) const { |
| 765 | ArrayRef<int64_t> vecShape = valueTy.getShape(); |
| 766 | if (!computeShapeRatio(shape: vecShape, subShape: convTile) || |
| 767 | !computeShapeRatio(shape: convTile, subShape: inTile) || |
| 768 | !computeShapeRatio(shape: convTile, subShape: outTile)) |
| 769 | return failure(); |
| 770 | |
| 771 | Type elemTy = valueTy.getElementType(); |
| 772 | int64_t rank = valueTy.getRank(); |
| 773 | VectorType convTy = VectorType::get(shape: convTile, elementType: elemTy); |
| 774 | SmallVector<int64_t> strides(rank, 1); |
| 775 | |
| 776 | Value source = op.getSource(); |
| 777 | auto zeroOf = [&](VectorType ty) -> Value { |
| 778 | return arith::ConstantOp::create( |
| 779 | builder&: rewriter, location: loc, result: ty, |
| 780 | value: DenseElementsAttr::get(type: ty, values: rewriter.getZeroAttr(type: elemTy))); |
| 781 | }; |
| 782 | auto addOffsets = [](ArrayRef<int64_t> a, |
| 783 | ArrayRef<int64_t> b) -> SmallVector<int64_t> { |
| 784 | SmallVector<int64_t> res(a); |
| 785 | for (auto [r, v] : llvm::zip_equal(t&: res, u&: b)) |
| 786 | r += v; |
| 787 | return res; |
| 788 | }; |
| 789 | |
| 790 | Value result = zeroOf(valueTy); |
| 791 | for (SmallVector<int64_t> convOff : |
| 792 | StaticTileOffsetRange(vecShape, convTile)) { |
| 793 | // Build the convert tile from inTile-sized slices of the source. |
| 794 | Value conv; |
| 795 | if (convTile == inTile) { |
| 796 | conv = vector::ExtractStridedSliceOp::create( |
| 797 | builder&: rewriter, location: loc, source, offsets: convOff, sizes: convTile, strides); |
| 798 | } else { |
| 799 | conv = zeroOf(convTy); |
| 800 | for (SmallVector<int64_t> inLocal : |
| 801 | StaticTileOffsetRange(convTile, inTile)) { |
| 802 | Value piece = vector::ExtractStridedSliceOp::create( |
| 803 | builder&: rewriter, location: loc, source, offsets: addOffsets(convOff, inLocal), sizes: inTile, |
| 804 | strides); |
| 805 | conv = vector::InsertStridedSliceOp::create(builder&: rewriter, location: loc, valueToStore: piece, |
| 806 | dest: conv, offsets: inLocal, strides); |
| 807 | } |
| 808 | } |
| 809 | |
| 810 | conv = xegpu::ConvertLayoutOp::create(builder&: rewriter, location: loc, result: convTy, source: conv, |
| 811 | input_layout: inputLayout, target_layout: targetLayout); |
| 812 | |
| 813 | // Write the converted tile into the result as outTile-sized slices. |
| 814 | if (convTile == outTile) { |
| 815 | result = vector::InsertStridedSliceOp::create(builder&: rewriter, location: loc, valueToStore: conv, |
| 816 | dest: result, offsets: convOff, strides); |
| 817 | } else { |
| 818 | for (SmallVector<int64_t> outLocal : |
| 819 | StaticTileOffsetRange(convTile, outTile)) { |
| 820 | Value piece = vector::ExtractStridedSliceOp::create( |
| 821 | builder&: rewriter, location: loc, source: conv, offsets: outLocal, sizes: outTile, strides); |
| 822 | result = vector::InsertStridedSliceOp::create( |
| 823 | builder&: rewriter, location: loc, valueToStore: piece, dest: result, offsets: addOffsets(convOff, outLocal), |
| 824 | strides); |
| 825 | } |
| 826 | } |
| 827 | } |
| 828 | |
| 829 | rewriter.replaceOp(op, newValues: result); |
| 830 | return success(); |
| 831 | } |
| 832 | |
| 833 | LogicalResult matchAndRewrite(xegpu::ConvertLayoutOp op, |
| 834 | PatternRewriter &rewriter) const override { |
| 835 | Location loc = op.getLoc(); |
| 836 | Type valType = op.getType(); |
| 837 | |
| 838 | xegpu::DistributeLayoutAttr inputLayout = op.getEffectiveInputLayout(); |
| 839 | xegpu::DistributeLayoutAttr targetLayout = op.getTargetLayoutAttr(); |
| 840 | if (!inputLayout || !targetLayout) |
| 841 | return rewriter.notifyMatchFailure(arg&: op, msg: "missing layout attributes." ); |
| 842 | |
| 843 | if (valType.isIntOrFloat()) { |
| 844 | rewriter.replaceOp(op, newValues: op.getSource()); |
| 845 | return success(); |
| 846 | } |
| 847 | |
| 848 | // Capture inst_data granularities before stripping them. |
| 849 | SmallVector<int64_t> inTile = inputLayout.getEffectiveInstDataAsInt(); |
| 850 | SmallVector<int64_t> outTile = targetLayout.getEffectiveInstDataAsInt(); |
| 851 | if (inTile.empty() || outTile.empty()) |
| 852 | return rewriter.notifyMatchFailure(arg&: op, msg: "Not a target ConvertLayoutOp." ); |
| 853 | |
| 854 | inputLayout = inputLayout.dropInstData(); |
| 855 | targetLayout = targetLayout.dropInstData(); |
| 856 | |
| 857 | VectorType valueTy = llvm::dyn_cast<VectorType>(Val: op.getType()); |
| 858 | assert(valueTy && "the value type must be vector type!" ); |
| 859 | |
| 860 | std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op); |
| 861 | if (!targetShape || targetShape->size() != (size_t)valueTy.getRank()) |
| 862 | return failure(); |
| 863 | |
| 864 | // Nothing to convert if layouts match after stripping inst_data. |
| 865 | if (!inputLayout || !targetLayout || inputLayout.isEqualTo(other: targetLayout)) { |
| 866 | rewriter.replaceOp(op, newValues: op.getSource()); |
| 867 | return success(); |
| 868 | } |
| 869 | |
| 870 | // Try regrouping: extract at inTile, convert, insert at outTile. |
| 871 | if (succeeded(Result: rewriteWithRegrouping(op, valueTy, convTile: *targetShape, inTile, |
| 872 | outTile, inputLayout, targetLayout, loc, |
| 873 | rewriter))) |
| 874 | return success(); |
| 875 | |
| 876 | // Fallback: pack/unpack at the convert tile granularity. |
| 877 | SmallVector<Type> convertedValTypes = |
| 878 | getUnrolledTypes(type: valueTy, tileShape: *targetShape); |
| 879 | SmallVector<Value> convertedValues = |
| 880 | pack(src: op.getOperand(), destTypes: convertedValTypes, blockSize: *targetShape, loc, rewriter); |
| 881 | SmallVector<Value> newOps; |
| 882 | for (auto [v, t] : llvm::zip(t&: convertedValues, u&: convertedValTypes)) { |
| 883 | auto newOp = xegpu::ConvertLayoutOp::create(builder&: rewriter, location: loc, result: t, source: v, |
| 884 | input_layout: inputLayout, target_layout: targetLayout); |
| 885 | newOps.push_back(Elt: newOp); |
| 886 | } |
| 887 | Value newSource = unpack(srcs: newOps, destTy: op.getType(), blockSize: *targetShape, loc, rewriter); |
| 888 | |
| 889 | rewriter.replaceOp(op, newValues: newSource); |
| 890 | return success(); |
| 891 | } |
| 892 | }; |
| 893 | |
| 894 | /// Unrolls vector.multi_reduction by sequentially reducing tiles with |
| 895 | /// elementwise arith operations first, then a single multi_reduction |
| 896 | /// per non-reduced tile position. This avoids generating long chains of |
| 897 | /// multi_reduction ops (as the upstream pattern does) and is more efficient. |
| 898 | /// |
| 899 | /// Example: |
| 900 | /// vector.multi_reduction <32x64xf16> to <32xf16> (tile_shape=32, 32) |
| 901 | /// -- Upstream pattern generates: |
| 902 | /// %tmp1 = vector.multi_reduction %tile0, %zero_acc <32x32xf16> to <32xf16> |
| 903 | /// %res = vector.multi_reduction %tmp1, %tile1 <32x32xf16> to <32xf16> |
| 904 | /// -- This pattern generates: |
| 905 | /// %tmp1 = arith.reduction %tile0, %tile1 <32x32xf16> -> <32x32xf16> // |
| 906 | /// elementwise %res = vector.multi_reduction %tmp1, %zero_acc <32x32xf16> to |
| 907 | /// <32xf16> |
| 908 | struct UnrollMultiReductionOp |
| 909 | : public UnrollPattern<vector::MultiDimReductionOp> { |
| 910 | UnrollMultiReductionOp(MLIRContext *context, |
| 911 | const xegpu::UnrollOptions &options, |
| 912 | PatternBenefit benefit = 2) |
| 913 | : UnrollPattern<vector::MultiDimReductionOp>(context, options, benefit) {} |
| 914 | |
| 915 | LogicalResult matchAndRewrite(vector::MultiDimReductionOp reductionOp, |
| 916 | PatternRewriter &rewriter) const override { |
| 917 | VectorType srcTy = reductionOp.getSourceVectorType(); |
| 918 | ArrayRef<int64_t> srcShape = srcTy.getShape(); |
| 919 | int64_t srcRank = srcTy.getRank(); |
| 920 | |
| 921 | Location loc = reductionOp.getLoc(); |
| 922 | Value source = reductionOp.getSource(); |
| 923 | Value acc = reductionOp.getAcc(); |
| 924 | vector::CombiningKind kind = reductionOp.getKind(); |
| 925 | |
| 926 | // Result must be a vector (not scalar). |
| 927 | auto resultType = dyn_cast<VectorType>(Val: reductionOp.getDestType()); |
| 928 | if (!resultType) |
| 929 | return failure(); |
| 930 | |
| 931 | std::optional<SmallVector<int64_t>> targetShapeOpt = |
| 932 | getTargetShape(op: reductionOp); |
| 933 | if (!targetShapeOpt || |
| 934 | static_cast<int64_t>(targetShapeOpt->size()) != srcRank) |
| 935 | return failure(); |
| 936 | |
| 937 | SmallVector<int64_t> targetShape = *targetShapeOpt; |
| 938 | |
| 939 | // Check divisibility for all dimensions. |
| 940 | for (int64_t i = 0; i < srcRank; ++i) { |
| 941 | if (srcShape[i] % targetShape[i] != 0) |
| 942 | return failure(); |
| 943 | } |
| 944 | |
| 945 | SmallVector<bool> reductionMask = reductionOp.getReductionMask(); |
| 946 | // Identify reduced and kept dimensions from the reduction mask. |
| 947 | SmallVector<int64_t> reducedDims, keptDims; |
| 948 | for (int64_t i = 0; i < srcRank; ++i) { |
| 949 | if (reductionMask[i]) |
| 950 | reducedDims.push_back(Elt: i); |
| 951 | else |
| 952 | keptDims.push_back(Elt: i); |
| 953 | } |
| 954 | |
| 955 | // Compute the number of tiles along each reduced dimension and their |
| 956 | // product |
| 957 | SmallVector<int64_t> numReducedTilesPerDim; |
| 958 | for (int64_t d : reducedDims) |
| 959 | numReducedTilesPerDim.push_back(Elt: srcShape[d] / targetShape[d]); |
| 960 | |
| 961 | // Build kept shapes for iterating over non-reduced dimensions. |
| 962 | SmallVector<int64_t> keptShape, keptTileShape; |
| 963 | for (int64_t d : keptDims) { |
| 964 | keptShape.push_back(Elt: srcShape[d]); |
| 965 | keptTileShape.push_back(Elt: targetShape[d]); |
| 966 | } |
| 967 | |
| 968 | // Initialize the result vector for assembly. |
| 969 | Value result = arith::ConstantOp::create(builder&: rewriter, location: loc, result: resultType, |
| 970 | value: rewriter.getZeroAttr(type: resultType)); |
| 971 | |
| 972 | // Iterate over all tile positions in the kept dimensions. |
| 973 | // Ex: [off0, off1, _ _ off4] |
| 974 | // blanks are offsets for the reduced dims, they will be |
| 975 | // generated in the inner loop below |
| 976 | for (SmallVector<int64_t> keptOffsets : |
| 977 | StaticTileOffsetRange(keptShape, keptTileShape)) { |
| 978 | |
| 979 | // Reconstruct full-rank base offsets with 0 for reduced dims. |
| 980 | // Ex: [off0, off1, 0, 0, off4] |
| 981 | SmallVector<int64_t> baseOffsets(srcRank, 0); |
| 982 | for (auto [idx, dim] : llvm::enumerate(First&: keptDims)) |
| 983 | baseOffsets[dim] = keptOffsets[idx]; |
| 984 | |
| 985 | // Generate the full tile indices for the reduced dimensions. |
| 986 | // Ex: if reduceDimShapes = [32, 64] and |
| 987 | // reducedDimTargetShapes = [16, 16], then reducedTileCoords: |
| 988 | // [(0, 0), (0, 1), (0, 2), (0, 3), |
| 989 | // (1, 0), (1, 1), (1, 2), (1, 3)] |
| 990 | auto reducedTileCoords = StaticTileOffsetRange( |
| 991 | numReducedTilesPerDim, SmallVector<int64_t>(reducedDims.size(), 1)); |
| 992 | |
| 993 | // Step 1: Fill "blanks" in the offsets for the reduced dimensions |
| 994 | // using 'reducedTileCoords' and extract according tiles. |
| 995 | // Ex: tiles = [source[off0, off1, off2_red, off3_red, off4], ...] |
| 996 | SmallVector<Value> tiles; |
| 997 | for (SmallVector<int64_t> reducedTileIdx : reducedTileCoords) { |
| 998 | SmallVector<int64_t> offsets(baseOffsets); |
| 999 | for (auto [idx, dim] : llvm::enumerate(First&: reducedDims)) |
| 1000 | offsets[dim] = reducedTileIdx[idx] * targetShape[dim]; |
| 1001 | SmallVector<int64_t> strides(srcRank, 1); |
| 1002 | Value tile = vector::ExtractStridedSliceOp::create( |
| 1003 | builder&: rewriter, location: loc, source, offsets, sizes: targetShape, strides); |
| 1004 | tiles.push_back(Elt: tile); |
| 1005 | } |
| 1006 | |
| 1007 | // Step 2: Sequentially reduce tiles using elementwise arith operations. |
| 1008 | Value reduced = tiles[0]; |
| 1009 | for (size_t i = 1; i < tiles.size(); ++i) |
| 1010 | reduced = |
| 1011 | vector::makeArithReduction(b&: rewriter, loc, kind, v1: reduced, acc: tiles[i], |
| 1012 | fastmath: reductionOp.getFastmathAttr()); |
| 1013 | |
| 1014 | // Step 3: Perform a single multi_reduction with the accumulator slice. |
| 1015 | SmallVector<int64_t> accStrides(keptTileShape.size(), 1); |
| 1016 | Value accSlice = vector::ExtractStridedSliceOp::create( |
| 1017 | builder&: rewriter, location: loc, source: acc, offsets: keptOffsets, sizes: keptTileShape, strides: accStrides); |
| 1018 | |
| 1019 | auto newReduction = vector::MultiDimReductionOp::create( |
| 1020 | builder&: rewriter, location: loc, source: reduced, acc: accSlice, reductionMask, kind, |
| 1021 | fastMathFlags: reductionOp.getFastmath()); |
| 1022 | |
| 1023 | // Step 4: Insert the reduced result into the output vector. |
| 1024 | SmallVector<int64_t> dstStrides(keptTileShape.size(), 1); |
| 1025 | result = vector::InsertStridedSliceOp::create( |
| 1026 | builder&: rewriter, location: loc, valueToStore: newReduction, dest: result, offsets: keptOffsets, strides: dstStrides); |
| 1027 | } |
| 1028 | |
| 1029 | rewriter.replaceOp(op: reductionOp, newValues: result); |
| 1030 | return success(); |
| 1031 | } |
| 1032 | }; |
| 1033 | |
| 1034 | } // namespace |
| 1035 | |
| 1036 | void mlir::xegpu::populateXeGPUUnrollPatterns( |
| 1037 | RewritePatternSet &patterns, const xegpu::UnrollOptions &options) { |
| 1038 | patterns |
| 1039 | .add<UnrollCreateNdOp, UnrollPrefetchNdOp, UnrollLoadNdOp, |
| 1040 | UnrollStoreNdOp, UnrollDpasOp, UnrollDpasMxOp, UnrollLoadMatrixOp, |
| 1041 | UnrollStoreMatrixOp, UnrollLoadGatherOp, UnrollStoreScatterOp, |
| 1042 | UnrollConvertLayoutOp, UnrollMultiReductionOp>(arg: patterns.getContext(), |
| 1043 | args: options); |
| 1044 | } |
| 1045 | |