| 1 | //===- VectorContractToAMXDotProduct.cpp ----------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 10 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 11 | #include "mlir/Dialect/Vector/Utils/VectorUtils.h" |
| 12 | #include "mlir/Dialect/X86/Transforms.h" |
| 13 | #include "mlir/Dialect/X86/Utils/X86Utils.h" |
| 14 | #include "mlir/Dialect/X86/X86Dialect.h" |
| 15 | |
| 16 | #include "mlir/IR/BuiltinAttributes.h" |
| 17 | #include "mlir/IR/Dominance.h" |
| 18 | #include "mlir/IR/PatternMatch.h" |
| 19 | #include "llvm/Support/Casting.h" |
| 20 | |
| 21 | #include "mlir/Pass/Pass.h" |
| 22 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 23 | |
| 24 | using namespace mlir; |
| 25 | using namespace mlir::vector; |
| 26 | using namespace mlir::x86; |
| 27 | |
| 28 | namespace { |
| 29 | |
| 30 | // Return true if the operation is a constant dense element |
| 31 | // attribute. |
| 32 | static bool isZeroVectorConstant(Operation *op) { |
| 33 | auto constantOp = dyn_cast_if_present<arith::ConstantOp>(Val: op); |
| 34 | if (!constantOp) |
| 35 | return false; |
| 36 | |
| 37 | if (!isa<VectorType>(Val: constantOp.getType())) |
| 38 | return false; |
| 39 | |
| 40 | auto denseAttr = dyn_cast<DenseElementsAttr>(Val: constantOp.getValue()); |
| 41 | if (!denseAttr || !denseAttr.isSplat()) |
| 42 | return false; |
| 43 | |
| 44 | return isZeroIntegerOrFloat(v: denseAttr.getSplatValue<Attribute>()); |
| 45 | } |
| 46 | |
| 47 | // Recursively follows single-use values through scf.yield operations |
| 48 | // and returns the first non-yield user result in the contraction chain. |
| 49 | static Value contractionUsersAfterYield(Value v) { |
| 50 | if (v.getNumUses() != 1) |
| 51 | return nullptr; |
| 52 | |
| 53 | OpOperand &use = *v.use_begin(); |
| 54 | Operation *user = use.getOwner(); |
| 55 | |
| 56 | if (!isa<scf::YieldOp>(Val: user)) |
| 57 | return v; |
| 58 | |
| 59 | auto yield = cast<scf::YieldOp>(Val: user); |
| 60 | Operation *parent = yield->getParentOp(); |
| 61 | unsigned idx = use.getOperandNumber(); |
| 62 | |
| 63 | return contractionUsersAfterYield(v: parent->getResult(idx)); |
| 64 | } |
| 65 | |
| 66 | // Function to collapse the last two dimension (k or n, and vnni) to help the |
| 67 | // amx.tile_load to correctly load the packed element type. |
| 68 | static Value collapseInnerDims(OpBuilder &builder, Location loc, Value input) { |
| 69 | ShapedType inputType = cast<ShapedType>(Val: input.getType()); |
| 70 | int64_t firstDimToCollapse = inputType.getRank() - 2; |
| 71 | |
| 72 | if (inputType.getRank() == 1) |
| 73 | return input; |
| 74 | |
| 75 | SmallVector<ReassociationIndices> reassociation; |
| 76 | for (int64_t i = 0; i < firstDimToCollapse; ++i) |
| 77 | reassociation.push_back(Elt: ReassociationIndices{i}); |
| 78 | |
| 79 | ReassociationIndices collapsedIndices; |
| 80 | for (int64_t i = firstDimToCollapse; i < inputType.getRank(); ++i) |
| 81 | collapsedIndices.push_back(Elt: i); |
| 82 | |
| 83 | reassociation.push_back(Elt: collapsedIndices); |
| 84 | return memref::CollapseShapeOp::create(builder, location: loc, src: input, reassociation); |
| 85 | } |
| 86 | |
| 87 | // Check if a vector.contract operand has a memref read source. |
| 88 | static bool isReadSrcMemref(Value operand) { |
| 89 | Operation *defOp = operand.getDefiningOp(); |
| 90 | if (!defOp) |
| 91 | return false; |
| 92 | |
| 93 | Value srcBuff; |
| 94 | llvm::TypeSwitch<Operation *>(operand.getDefiningOp()) |
| 95 | .Case<TransferReadOp, LoadOp>( |
| 96 | caseFn: [&](auto readOp) { srcBuff = readOp.getOperand(0); }); |
| 97 | |
| 98 | return srcBuff && isa<MemRefType>(Val: srcBuff.getType()); |
| 99 | } |
| 100 | |
| 101 | // Check if a vector.contract operand reads whole VNNI groups: the memref's |
| 102 | // static innermost dim is the VNNI factor and the read starts at its origin. |
| 103 | static bool isVnniOperandRead(Value operand, int64_t vnni) { |
| 104 | Operation *defOp = operand.getDefiningOp(); |
| 105 | if (!defOp) |
| 106 | return false; |
| 107 | |
| 108 | Value srcBuff; |
| 109 | OpFoldResult vnniIndex; |
| 110 | llvm::TypeSwitch<Operation *>(defOp).Case<TransferReadOp, LoadOp>( |
| 111 | caseFn: [&](auto readOp) { |
| 112 | srcBuff = readOp.getBase(); |
| 113 | vnniIndex = readOp.getIndices().back(); |
| 114 | }); |
| 115 | |
| 116 | if (!srcBuff) |
| 117 | return false; |
| 118 | auto srcType = dyn_cast<MemRefType>(Val: srcBuff.getType()); |
| 119 | return srcType && srcType.getRank() > 0 && |
| 120 | srcType.getShape().back() == vnni && isZeroInteger(v: vnniIndex); |
| 121 | } |
| 122 | |
| 123 | // Replaces the indices of the two innermost dims of a VNNI operand by the |
| 124 | // index of the dim they collapse into. |
| 125 | static void collapseVnniIndices(OpBuilder &rewriter, Location loc, |
| 126 | SmallVectorImpl<Value> &indices, int64_t vnni) { |
| 127 | Value inner = indices.pop_back_val(); |
| 128 | Value cVnni = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: vnni); |
| 129 | Value scaled = arith::MulIOp::create(builder&: rewriter, location: loc, lhs: indices.back(), rhs: cVnni); |
| 130 | indices.back() = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: scaled, rhs: inner); |
| 131 | } |
| 132 | |
| 133 | // Get the MemRef source and offset index for the operands of |
| 134 | // vector.contract. |
| 135 | static FailureOr<std::pair<Value, SmallVector<Value>>> |
| 136 | getSrcIndxValue(OpBuilder &rewriter, Location loc, Value operand, |
| 137 | bool isNotAcc) { |
| 138 | Operation *defOp = operand.getDefiningOp(); |
| 139 | if (!defOp) |
| 140 | return failure(); |
| 141 | |
| 142 | Value srcBuff; |
| 143 | SmallVector<OpFoldResult> indexVals; |
| 144 | llvm::TypeSwitch<Operation *>(operand.getDefiningOp()) |
| 145 | .Case<TransferReadOp, LoadOp>(caseFn: [&](auto readOp) { |
| 146 | indexVals = SmallVector<OpFoldResult>(readOp.getIndices().begin(), |
| 147 | readOp.getIndices().end()); |
| 148 | srcBuff = readOp.getOperand(0); |
| 149 | }); |
| 150 | |
| 151 | if (!srcBuff || !isa<MemRefType>(Val: srcBuff.getType())) |
| 152 | return failure(); |
| 153 | |
| 154 | SmallVector<Value> indices; |
| 155 | indices.reserve(N: indexVals.size()); |
| 156 | |
| 157 | for (OpFoldResult ofr : indexVals) { |
| 158 | indices.push_back( |
| 159 | Elt: mlir::getValueOrCreateConstantIndexOp(b&: rewriter, loc, ofr)); |
| 160 | } |
| 161 | |
| 162 | if (isNotAcc) { |
| 163 | collapseVnniIndices(rewriter, loc, indices, |
| 164 | vnni: cast<MemRefType>(Val: srcBuff.getType()).getShape().back()); |
| 165 | srcBuff = collapseInnerDims(builder&: rewriter, loc, input: srcBuff); |
| 166 | } |
| 167 | |
| 168 | return std::make_pair(x&: srcBuff, y&: indices); |
| 169 | } |
| 170 | |
| 171 | // Function to validate the loop step value. |
| 172 | static LogicalResult validateLoopStep(OpBuilder &rewriter, Value step, |
| 173 | int64_t value) { |
| 174 | |
| 175 | auto cst = step.getDefiningOp<arith::ConstantIndexOp>(); |
| 176 | if (!cst) |
| 177 | return failure(); |
| 178 | |
| 179 | if (cst.value() != value && cst.value() != 1) |
| 180 | return failure(); |
| 181 | |
| 182 | return success(); |
| 183 | } |
| 184 | |
| 185 | // Function to validate the vector.contract operation. |
| 186 | static LogicalResult validateContractOps(OpBuilder &rewriter, |
| 187 | vector::ContractionOp contractOp, |
| 188 | unsigned int blockingFactor, |
| 189 | Value srcBuffLhs, Value srcBuffRhs, |
| 190 | bool srcValidate) { |
| 191 | |
| 192 | if (srcValidate) { |
| 193 | // Get the MemRef buffer of LHS operand. |
| 194 | auto srcIndxLhs = getSrcIndxValue(rewriter, loc: contractOp.getLoc(), |
| 195 | operand: contractOp.getLhs(), isNotAcc: false); |
| 196 | if (failed(Result: srcIndxLhs)) |
| 197 | return failure(); |
| 198 | auto [buffLhs, indicesLhs] = *srcIndxLhs; |
| 199 | |
| 200 | // Get the MemRef buffer of RHS operand. |
| 201 | auto srcIndxRhs = getSrcIndxValue(rewriter, loc: contractOp.getLoc(), |
| 202 | operand: contractOp.getRhs(), isNotAcc: false); |
| 203 | if (failed(Result: srcIndxRhs)) |
| 204 | return failure(); |
| 205 | auto [buffRhs, indicesRhs] = *srcIndxRhs; |
| 206 | |
| 207 | // Return failure if the Memref buff didn't match. |
| 208 | if (buffLhs != srcBuffLhs) |
| 209 | return failure(); |
| 210 | |
| 211 | if (buffRhs != srcBuffRhs) |
| 212 | return failure(); |
| 213 | } |
| 214 | |
| 215 | if (!contractionUsersAfterYield(v: contractOp.getResult())) |
| 216 | return failure(); |
| 217 | |
| 218 | VectorType accTy = dyn_cast<VectorType>(Val: contractOp.getAccType()); |
| 219 | if (!accTy) |
| 220 | return failure(); |
| 221 | |
| 222 | // The Accumulator dims should be 16 or 1. Like <1x16x16> or <16x16>. |
| 223 | ArrayRef<int64_t> accShape = accTy.getShape(); |
| 224 | SmallVector<int64_t> nonUnitDimAcc; |
| 225 | llvm::copy_if(Range&: accShape, Out: std::back_inserter(x&: nonUnitDimAcc), |
| 226 | P: [](int64_t dim) { return (dim != 16 && dim != 1); }); |
| 227 | |
| 228 | if (nonUnitDimAcc.size() != 0) |
| 229 | return failure(); |
| 230 | |
| 231 | // The LHS dims should be 16 or vnni or 1. Like <1x16x16x2> or |
| 232 | // <16x16x4>. The vnni dims should be 2 or 4. |
| 233 | VectorType lhsTy = contractOp.getLhsType(); |
| 234 | ArrayRef<int64_t> lhsShape = lhsTy.getShape(); |
| 235 | SmallVector<int64_t> nonUnitDimLhs; |
| 236 | llvm::copy_if(Range&: lhsShape, Out: std::back_inserter(x&: nonUnitDimLhs), |
| 237 | P: [](int64_t dim) { return (dim != 16 && dim != 1); }); |
| 238 | |
| 239 | if (nonUnitDimLhs.size() != 1) |
| 240 | return failure(); |
| 241 | |
| 242 | if (nonUnitDimLhs[0] != blockingFactor) |
| 243 | return failure(); |
| 244 | |
| 245 | // The RHS dims should be 16 or vnni or 1. Like <1x16x16x2> or |
| 246 | // <16x16x4>. The vnni dims should be 2 or 4. |
| 247 | VectorType rhsTy = contractOp.getRhsType(); |
| 248 | ArrayRef<int64_t> rhsShape = rhsTy.getShape(); |
| 249 | llvm::SmallVector<int64_t> nonUnitDimRhs; |
| 250 | llvm::copy_if(Range&: rhsShape, Out: std::back_inserter(x&: nonUnitDimRhs), |
| 251 | P: [](int64_t dim) { return (dim != 16 && dim != 1); }); |
| 252 | |
| 253 | if (nonUnitDimRhs.size() != 1) |
| 254 | return failure(); |
| 255 | |
| 256 | if (nonUnitDimRhs[0] != blockingFactor) |
| 257 | return failure(); |
| 258 | |
| 259 | return success(); |
| 260 | } |
| 261 | |
| 262 | // Traces an accumulator back through the iter_args of the given loops and |
| 263 | // returns the first value that is not one of them, e.g., the init value of the |
| 264 | // outermost loop or a value computed within the loops. |
| 265 | static Value getLoopNestInit(Value acc, ArrayRef<scf::ForOp> loops) { |
| 266 | while (auto arg = dyn_cast<BlockArgument>(Val&: acc)) { |
| 267 | auto loop = dyn_cast_if_present<scf::ForOp>(Val: arg.getOwner()->getParentOp()); |
| 268 | if (!loop || !llvm::is_contained(Range&: loops, Element: loop)) |
| 269 | break; |
| 270 | OpOperand *init = loop.getTiedLoopInit(bbArg: arg); |
| 271 | if (!init) |
| 272 | break; |
| 273 | acc = init->get(); |
| 274 | } |
| 275 | return acc; |
| 276 | } |
| 277 | |
| 278 | // Returns the loop index position to get mapped during the |
| 279 | // MemRef type clone. |
| 280 | static unsigned getIndexPosition(Value operand, scf::ForOp loop) { |
| 281 | Value iv = loop.getInductionVar(); |
| 282 | |
| 283 | Value srcBuff; |
| 284 | llvm::TypeSwitch<Operation *>(operand.getDefiningOp()) |
| 285 | .Case<TransferReadOp, LoadOp>( |
| 286 | caseFn: [&](auto readOp) { srcBuff = readOp.getOperand(0); }); |
| 287 | |
| 288 | auto subview = srcBuff.getDefiningOp<memref::SubViewOp>(); |
| 289 | if (!subview) |
| 290 | return 0; |
| 291 | |
| 292 | auto offsets = subview.getOffsets(); |
| 293 | |
| 294 | for (auto it : llvm::enumerate(First&: offsets)) { |
| 295 | if (it.value() == iv) |
| 296 | return it.index(); |
| 297 | } |
| 298 | |
| 299 | return 0; |
| 300 | } |
| 301 | |
| 302 | // Creates amx.tile_loads. |
| 303 | static amx::TileLoadOp createTileLoads(OpBuilder &rewriter, Location loc, |
| 304 | Value operand, Value mat, Type ipType, |
| 305 | unsigned int offset, bool isVnni) { |
| 306 | |
| 307 | auto srcIndx = getSrcIndxValue(rewriter, loc, operand, isNotAcc: false); |
| 308 | auto [srcBuff, indices] = *srcIndx; |
| 309 | if (isVnni) |
| 310 | collapseVnniIndices(rewriter, loc, indices, |
| 311 | vnni: cast<MemRefType>(Val: srcBuff.getType()).getShape().back()); |
| 312 | |
| 313 | amx::TileType tileType = amx::TileType::get(shape: {16, (16 * offset)}, elementType: ipType); |
| 314 | return amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType, base: mat, indices); |
| 315 | } |
| 316 | |
| 317 | static void performShuffle(OpBuilder &rewriter, Location loc, Value matB, |
| 318 | Type ipType, unsigned int offset, Value packedBuffer, |
| 319 | Value indxToStoreInBuffer) { |
| 320 | |
| 321 | Value c0 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 322 | Value c16 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 16); |
| 323 | SmallVector<Value> subviewOffset( |
| 324 | llvm::cast<MemRefType>(Val: matB.getType()).getRank(), c0); |
| 325 | |
| 326 | Value cStep = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: offset); |
| 327 | Value cBound = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: (16 * offset)); |
| 328 | Value offsetIndx = |
| 329 | arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: (offset / 2)); |
| 330 | |
| 331 | scf::ForOp::create( |
| 332 | builder&: rewriter, location: loc, lowerBound: c0, upperBound: cBound, step: cStep, initArgs: ValueRange{}, |
| 333 | odsArg4: [&](OpBuilder &nestedBuilder, Location loc, Value iv, |
| 334 | ValueRange iterArgs) { |
| 335 | subviewOffset[subviewOffset.size() - 2] = iv; |
| 336 | |
| 337 | // Retrieve two rows of vector (32) for int8 and f8 type. For bf16, |
| 338 | // retrieve one row of vector (32). |
| 339 | auto vectorType = VectorType::get(shape: {2, (16 * (offset / 2))}, elementType: ipType); |
| 340 | if (ipType.isBF16()) |
| 341 | vectorType = VectorType::get(shape: (16 * offset), elementType: ipType); |
| 342 | |
| 343 | int64_t srcRank = (dyn_cast<ShapedType>(Val: matB.getType())).getRank(); |
| 344 | Value padding = ub::PoisonOp::create(builder&: rewriter, location: loc, result: ipType); |
| 345 | auto map = AffineMap::getMinorIdentityMap(dims: srcRank, results: vectorType.getRank(), |
| 346 | context: rewriter.getContext()); |
| 347 | SmallVector<bool> inBounds(vectorType.getRank(), true); |
| 348 | Value vec1 = vector::TransferReadOp::create( |
| 349 | builder&: rewriter, location: loc, vectorType, source: matB, indices: ValueRange(subviewOffset), padding, |
| 350 | permutationMap: map, inBounds); |
| 351 | |
| 352 | if (!ipType.isBF16()) |
| 353 | vec1 = vector::ShapeCastOp::create( |
| 354 | builder&: rewriter, location: loc, result: VectorType::get(shape: (16 * offset), elementType: ipType), source: vec1); |
| 355 | |
| 356 | // Increment the iv by 1 or 2 based on the type to load the next 32/64 |
| 357 | // elements |
| 358 | Value incIV = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: offsetIndx, rhs: iv); |
| 359 | subviewOffset[subviewOffset.size() - 2] = incIV; |
| 360 | |
| 361 | Value vec2 = vector::TransferReadOp::create( |
| 362 | builder&: rewriter, location: loc, vectorType, source: matB, indices: ValueRange(subviewOffset), padding, |
| 363 | permutationMap: map, inBounds); |
| 364 | if (!ipType.isBF16()) |
| 365 | vec2 = vector::ShapeCastOp::create( |
| 366 | builder&: rewriter, location: loc, result: VectorType::get(shape: (16 * offset), elementType: ipType), source: vec2); |
| 367 | |
| 368 | vector::ShuffleOp writeVal1; |
| 369 | vector::ShuffleOp writeVal2; |
| 370 | |
| 371 | if (ipType.isBF16()) { |
| 372 | |
| 373 | writeVal1 = vector::ShuffleOp::create( |
| 374 | builder&: rewriter, location: loc, result: VectorType::get(shape: {(16 * offset)}, elementType: ipType), v1: vec1, |
| 375 | v2: vec2, |
| 376 | mask: ArrayRef<int64_t>{0, 32, 1, 33, 2, 34, 3, 35, 8, 40, 9, |
| 377 | 41, 10, 42, 11, 43, 16, 48, 17, 49, 18, 50, |
| 378 | 19, 51, 24, 56, 25, 57, 26, 58, 27, 59}); |
| 379 | |
| 380 | writeVal2 = vector::ShuffleOp::create( |
| 381 | builder&: rewriter, location: loc, result: VectorType::get(shape: {(16 * offset)}, elementType: ipType), v1: vec1, |
| 382 | v2: vec2, |
| 383 | mask: ArrayRef<int64_t>{4, 36, 5, 37, 6, 38, 7, 39, 12, 44, 13, |
| 384 | 45, 14, 46, 15, 47, 20, 52, 21, 53, 22, 54, |
| 385 | 23, 55, 28, 60, 29, 61, 30, 62, 31, 63}); |
| 386 | } |
| 387 | |
| 388 | if (ipType.isSignlessInteger(width: 8) || ipType.isF8E5M2() || |
| 389 | ipType.isF8E4M3FN()) { |
| 390 | |
| 391 | writeVal1 = vector::ShuffleOp::create( |
| 392 | builder&: rewriter, location: loc, result: VectorType::get(shape: {(16 * offset)}, elementType: ipType), v1: vec1, |
| 393 | v2: vec2, |
| 394 | mask: ArrayRef<int64_t>{ |
| 395 | 0, 32, 64, 96, 1, 33, 65, 97, 2, 34, 66, 98, 3, |
| 396 | 35, 67, 99, 8, 40, 72, 104, 9, 41, 73, 105, 10, 42, |
| 397 | 74, 106, 11, 43, 75, 107, 16, 48, 80, 112, 17, 49, 81, |
| 398 | 113, 18, 50, 82, 114, 19, 51, 83, 115, 24, 56, 88, 120, |
| 399 | 25, 57, 89, 121, 26, 58, 90, 122, 27, 59, 91, 123}); |
| 400 | |
| 401 | writeVal2 = vector::ShuffleOp::create( |
| 402 | builder&: rewriter, location: loc, result: VectorType::get(shape: {(16 * offset)}, elementType: ipType), v1: vec1, |
| 403 | v2: vec2, |
| 404 | mask: ArrayRef<int64_t>{ |
| 405 | 4, 36, 68, 100, 5, 37, 69, 101, 6, 38, 70, 102, 7, 39, |
| 406 | 71, 103, 12, 44, 76, 108, 13, 45, 77, 109, 14, 46, 78, 110, |
| 407 | 15, 47, 79, 111, 20, 52, 84, 116, 21, 53, 85, 117, 22, 54, |
| 408 | 86, 118, 23, 55, 87, 119, 28, 60, 92, 124, 29, 61, 93, 125, |
| 409 | 30, 62, 94, 126, 31, 63, 95, 127}); |
| 410 | } |
| 411 | |
| 412 | // iv to store the shuffled elements |
| 413 | Value ivShuff1 = arith::DivUIOp::create(builder&: rewriter, location: loc, lhs: iv, rhs: cStep); |
| 414 | Value ivShuff2 = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: ivShuff1, rhs: c16); |
| 415 | |
| 416 | vector::StoreOp::create(builder&: rewriter, location: loc, valueToStore: writeVal1, base: packedBuffer, |
| 417 | indices: ValueRange{indxToStoreInBuffer, ivShuff1, c0}); |
| 418 | vector::StoreOp::create(builder&: rewriter, location: loc, valueToStore: writeVal2, base: packedBuffer, |
| 419 | indices: ValueRange{indxToStoreInBuffer, ivShuff2, c0}); |
| 420 | |
| 421 | scf::YieldOp::create(builder&: nestedBuilder, location: loc); |
| 422 | }); |
| 423 | } |
| 424 | |
| 425 | static llvm::DenseMap<Operation *, amx::TileLoadOp> |
| 426 | packInputs(OpBuilder &rewriter, Location loc, |
| 427 | SmallVector<vector::ContractionOp> ops, Value matB, Type ipType, |
| 428 | unsigned int offset, Value packedBuffer, bool pack, |
| 429 | Value indxToStoreInBuffer, Value indxToLoadFromMatB) { |
| 430 | |
| 431 | llvm::DenseMap<Operation *, amx::TileLoadOp> readsToTileLoads; |
| 432 | Value c0 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 433 | Value c16 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 16); |
| 434 | |
| 435 | for (size_t j = 0; j < ops.size(); j++) { |
| 436 | for (size_t i = 0; i < ops.size(); i++) { |
| 437 | |
| 438 | if (i != j && validatePairVectorContract(contractOp: ops[j], pairContOp: ops[i], rhsHasMultipleNonUnitDims: true, nonUnitDimValue: 16)) { |
| 439 | |
| 440 | Operation *readOpRhs = ops[j].getRhs().getDefiningOp(); |
| 441 | auto itRhs = readsToTileLoads.find(Val: readOpRhs); |
| 442 | if (itRhs != readsToTileLoads.end()) { |
| 443 | continue; |
| 444 | } |
| 445 | |
| 446 | if (pack) { |
| 447 | performShuffle(rewriter, loc, matB, ipType, offset, packedBuffer, |
| 448 | indxToStoreInBuffer); |
| 449 | } |
| 450 | |
| 451 | amx::TileType tileType = |
| 452 | amx::TileType::get(shape: {16, (16 * offset)}, elementType: ipType); |
| 453 | auto loadRow1 = |
| 454 | amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType, base: packedBuffer, |
| 455 | indices: ValueRange{indxToLoadFromMatB, c0, c0}); |
| 456 | |
| 457 | auto loadRow2 = |
| 458 | amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType, base: packedBuffer, |
| 459 | indices: ValueRange{indxToLoadFromMatB, c16, c0}); |
| 460 | |
| 461 | readsToTileLoads.try_emplace(Key: readOpRhs, Args&: loadRow1); |
| 462 | readsToTileLoads.try_emplace(Key: ops[i].getRhs().getDefiningOp(), Args&: loadRow2); |
| 463 | } |
| 464 | } |
| 465 | } |
| 466 | |
| 467 | return readsToTileLoads; |
| 468 | } |
| 469 | |
| 470 | // Creates tiled amx dot-products. |
| 471 | static SmallVector<Value> |
| 472 | createTiledDp(OpBuilder &rewriter, Location loc, |
| 473 | SmallVector<vector::ContractionOp> ops, Value matA, Value matB, |
| 474 | Type ipType, Type opType, ValueRange accIterArgs, |
| 475 | unsigned int offset, bool isVnni, Value packedBuffer, bool pack, |
| 476 | Value indxToStoreInBuffer, Value indxToLoadFromMatB) { |
| 477 | |
| 478 | if (isVnni) { |
| 479 | matA = collapseInnerDims(builder&: rewriter, loc, input: matA); |
| 480 | matB = collapseInnerDims(builder&: rewriter, loc, input: matB); |
| 481 | } |
| 482 | |
| 483 | SmallVector<Value> accumulators; |
| 484 | // Stores the amx.tile_load operation vs it's equivalent vector tranfer_read |
| 485 | // or load operations. |
| 486 | llvm::DenseMap<Operation *, amx::TileLoadOp> readsToTileLoads; |
| 487 | |
| 488 | // function call to online pack the input B matrix |
| 489 | if (!isVnni) { |
| 490 | readsToTileLoads = |
| 491 | packInputs(rewriter, loc, ops, matB, ipType, offset, packedBuffer, pack, |
| 492 | indxToStoreInBuffer, indxToLoadFromMatB); |
| 493 | } |
| 494 | |
| 495 | // Iterate over the contraction operations and compute the tiled dot-product. |
| 496 | for (size_t i = 0; i < ops.size(); i++) { |
| 497 | |
| 498 | Operation *readOpLhs = ops[i].getLhs().getDefiningOp(); |
| 499 | amx::TileLoadOp tilesLhs; |
| 500 | auto itLhs = readsToTileLoads.find(Val: readOpLhs); |
| 501 | if (itLhs != readsToTileLoads.end()) { |
| 502 | tilesLhs = itLhs->second; |
| 503 | } else { |
| 504 | tilesLhs = createTileLoads(rewriter, loc, operand: ops[i].getLhs(), mat: matA, ipType, |
| 505 | offset, isVnni); |
| 506 | readsToTileLoads.try_emplace(Key: readOpLhs, Args&: tilesLhs); |
| 507 | } |
| 508 | |
| 509 | Operation *readOpRhs = ops[i].getRhs().getDefiningOp(); |
| 510 | amx::TileLoadOp tilesRhs; |
| 511 | auto itRhs = readsToTileLoads.find(Val: readOpRhs); |
| 512 | if (itRhs != readsToTileLoads.end()) { |
| 513 | tilesRhs = itRhs->second; |
| 514 | } else { |
| 515 | tilesRhs = createTileLoads(rewriter, loc, operand: ops[i].getRhs(), mat: matB, ipType, |
| 516 | offset, isVnni); |
| 517 | readsToTileLoads.try_emplace(Key: readOpRhs, Args&: tilesRhs); |
| 518 | } |
| 519 | |
| 520 | auto accTileType = amx::TileType::get(shape: {16, 16}, elementType: opType); |
| 521 | |
| 522 | Value dp; |
| 523 | if (ipType.isBF16() || ipType.isF8E5M2() || ipType.isF8E4M3FN()) |
| 524 | dp = amx::TileMulFOp::create(builder&: rewriter, location: loc, res: accTileType, lhs: tilesLhs, |
| 525 | rhs: tilesRhs, acc: accIterArgs[i]); |
| 526 | |
| 527 | if (ipType.isSignlessInteger(width: 8)) |
| 528 | dp = amx::TileMulIOp::create(builder&: rewriter, location: loc, res: accTileType, lhs: tilesLhs, |
| 529 | rhs: tilesRhs, acc: accIterArgs[i]); |
| 530 | |
| 531 | accumulators.push_back(Elt: dp); |
| 532 | } |
| 533 | return accumulators; |
| 534 | } |
| 535 | |
| 536 | static SmallVector<Value> createTileZeros(OpBuilder &rewriter, Location loc, |
| 537 | Type opType, scf::ForOp outerLoop, |
| 538 | int64_t size) { |
| 539 | OpBuilder::InsertionGuard guard(rewriter); |
| 540 | rewriter.setInsertionPoint(outerLoop); |
| 541 | |
| 542 | SmallVector<Value> loopItrArgs; |
| 543 | auto zeroTileType = amx::TileType::get(shape: {16, 16}, elementType: opType); |
| 544 | |
| 545 | for (int i = 0; i < size; i++) { |
| 546 | auto zeroTile = amx::TileZeroOp::create(builder&: rewriter, location: loc, res: zeroTileType); |
| 547 | loopItrArgs.push_back(Elt: zeroTile); |
| 548 | } |
| 549 | return loopItrArgs; |
| 550 | } |
| 551 | |
| 552 | // Find memref source and indices of the accumulator tile for an AMX |
| 553 | // contraction. Returns nullopt if the access is not suitable for AMX tile |
| 554 | // load/store. |
| 555 | static std::optional<std::pair<Value, SmallVector<Value>>> |
| 556 | getAccTileAccess(Operation *op) { |
| 557 | if (!op) |
| 558 | return std::nullopt; |
| 559 | |
| 560 | Value base; |
| 561 | VectorType vecTy; |
| 562 | ValueRange indices; |
| 563 | if (auto transfer = dyn_cast<VectorTransferOpInterface>(Val: op)) { |
| 564 | if (transfer.getMask() || transfer.hasOutOfBoundsDim() || |
| 565 | !transfer.getPermutationMap().isMinorIdentity()) |
| 566 | return std::nullopt; |
| 567 | base = transfer.getBase(); |
| 568 | vecTy = transfer.getVectorType(); |
| 569 | indices = transfer.getIndices(); |
| 570 | } else if (auto load = dyn_cast<vector::LoadOp>(Val: op)) { |
| 571 | base = load.getBase(); |
| 572 | vecTy = load.getVectorType(); |
| 573 | indices = load.getIndices(); |
| 574 | } else if (auto store = dyn_cast<vector::StoreOp>(Val: op)) { |
| 575 | base = store.getBase(); |
| 576 | vecTy = store.getVectorType(); |
| 577 | indices = store.getIndices(); |
| 578 | } else { |
| 579 | return std::nullopt; |
| 580 | } |
| 581 | |
| 582 | auto memrefTy = dyn_cast<MemRefType>(Val: base.getType()); |
| 583 | ArrayRef<int64_t> shape = vecTy.getShape(); |
| 584 | if (!memrefTy || memrefTy.getRank() < 2 || !memrefTy.isLastDimUnitStride() || |
| 585 | shape.size() < 2 || shape.end()[-1] != 16 || shape.end()[-2] != 16 || |
| 586 | llvm::any_of(Range: shape.drop_back(N: 2), P: [](int64_t d) { return d != 1; })) |
| 587 | return std::nullopt; |
| 588 | |
| 589 | return std::make_pair(x&: base, y: SmallVector<Value>(indices)); |
| 590 | } |
| 591 | |
| 592 | // Returns the accumulator read and result write for each contraction if the |
| 593 | // accumulator can be updated in place through direct AMX tile accesses. |
| 594 | // If writeOnly is true, only the result writes are considered, and the read |
| 595 | // accesses are ignored. |
| 596 | static std::pair<std::optional<SmallVector<Operation *>>, |
| 597 | std::optional<SmallVector<Operation *>>> |
| 598 | getInPlaceAccAccesses(ArrayRef<vector::ContractionOp> ops, bool writeOnly) { |
| 599 | SmallVector<Operation *> reads, writes; |
| 600 | for (vector::ContractionOp op : ops) { |
| 601 | Value result = contractionUsersAfterYield(v: op.getResult()); |
| 602 | if (!result || !result.hasOneUse()) |
| 603 | return {std::nullopt, std::nullopt}; |
| 604 | Operation *write = *result.user_begin(); |
| 605 | if (!isa<vector::TransferWriteOp, vector::StoreOp>(Val: write) || |
| 606 | write->getOperand(idx: 0) != result || !getAccTileAccess(op: write)) |
| 607 | return {std::nullopt, std::nullopt}; |
| 608 | writes.push_back(Elt: write); |
| 609 | |
| 610 | if (writeOnly) |
| 611 | continue; |
| 612 | |
| 613 | Operation *read = traceToVectorReadLikeParentOperation(v: op.getAcc()); |
| 614 | if (!read || !getAccTileAccess(op: read)) |
| 615 | return {std::nullopt, std::nullopt}; |
| 616 | reads.push_back(Elt: read); |
| 617 | } |
| 618 | |
| 619 | if (writeOnly) |
| 620 | return {std::nullopt, writes}; |
| 621 | return std::make_pair(x&: reads, y&: writes); |
| 622 | } |
| 623 | |
| 624 | // Loads the initial accumulator tiles before the given loop. |
| 625 | static SmallVector<Value> createAccTileLoads(OpBuilder &rewriter, Location loc, |
| 626 | Type opType, scf::ForOp loop, |
| 627 | ArrayRef<Operation *> reads) { |
| 628 | OpBuilder::InsertionGuard guard(rewriter); |
| 629 | rewriter.setInsertionPoint(loop); |
| 630 | auto tileType = amx::TileType::get(shape: {16, 16}, elementType: opType); |
| 631 | SmallVector<Value> tiles; |
| 632 | for (Operation *read : reads) { |
| 633 | auto [base, indices] = *getAccTileAccess(op: read); |
| 634 | tiles.push_back( |
| 635 | Elt: amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType, base, indices)); |
| 636 | } |
| 637 | return tiles; |
| 638 | } |
| 639 | |
| 640 | static Value getIndxToLoadStoreFromPckBuffer(OpBuilder &rewriter, Location loc, |
| 641 | Value ivInnerLoop, |
| 642 | Value ivOuterLoop, |
| 643 | bool isInnerLoopUBHasOddQuot, |
| 644 | bool isInnerLoopUBLarger, |
| 645 | bool pack, Value blockStride) { |
| 646 | |
| 647 | Value c2 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 2); |
| 648 | |
| 649 | // `blockStride` is the reduction (K) loop step, i.e. the amount by which the |
| 650 | // induction variable advances for one K-block. Dividing the induction value |
| 651 | // by it yields the K-block index regardless of whether the loop counts |
| 652 | // K-elements (step == 16*blockingFactor) or pre-blocked K-tiles (step == 1). |
| 653 | Value quotientInnerLoop = |
| 654 | arith::DivUIOp::create(builder&: rewriter, location: loc, lhs: ivInnerLoop, rhs: blockStride); |
| 655 | Value remInnerLoop = arith::RemUIOp::create( |
| 656 | builder&: rewriter, location: loc, result: rewriter.getIndexType(), lhs: quotientInnerLoop, rhs: c2); |
| 657 | |
| 658 | if (!isInnerLoopUBLarger && !pack) { |
| 659 | remInnerLoop = arith::RemUIOp::create( |
| 660 | builder&: rewriter, location: loc, result: rewriter.getIndexType(), lhs: ivOuterLoop, rhs: c2); |
| 661 | } |
| 662 | |
| 663 | if (isInnerLoopUBHasOddQuot) { |
| 664 | auto remOuterLoop = arith::RemUIOp::create( |
| 665 | builder&: rewriter, location: loc, result: rewriter.getIndexType(), lhs: ivOuterLoop, rhs: c2); |
| 666 | auto remAdd = arith::AddIOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(), |
| 667 | lhs: remInnerLoop, rhs: remOuterLoop); |
| 668 | remInnerLoop = arith::RemUIOp::create(builder&: rewriter, location: loc, |
| 669 | result: rewriter.getIndexType(), lhs: remAdd, rhs: c2); |
| 670 | } |
| 671 | |
| 672 | return remInnerLoop; |
| 673 | } |
| 674 | |
| 675 | static scf::ForOp |
| 676 | createLoops(OpBuilder &rewriter, Location loc, Value lowerBound, |
| 677 | Value upperBound, Value step, SmallVector<Value> loopItrArgs, |
| 678 | Type ipType, Type opType, unsigned int blockingFactor, bool isVnni, |
| 679 | Operation *vectorOpLhs, Operation *vectorOpRhs, |
| 680 | vector::ContractionOp contractOp, scf::ForOp outerLoop, |
| 681 | scf::ForOp innerLoop, SmallVector<vector::ContractionOp> ops, |
| 682 | Value ivOuterLoop, Value packedBuffer, bool pack, |
| 683 | arith::ConstantIndexOp innerLoopIndex, bool isInnerLoopUBLarger, |
| 684 | bool isInnerLoopUBHasOddQuot) { |
| 685 | |
| 686 | Value c0 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 687 | Value c1 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1); |
| 688 | Value c2 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 2); |
| 689 | |
| 690 | int64_t offset = 16 * blockingFactor; |
| 691 | if (auto cst = step.getDefiningOp<arith::ConstantIndexOp>()) |
| 692 | offset = cst.value(); |
| 693 | |
| 694 | auto newLoop = scf::ForOp::create( |
| 695 | builder&: rewriter, location: loc, lowerBound, upperBound, step, initArgs: loopItrArgs, |
| 696 | odsArg4: [&](OpBuilder &rewriterNewInnerLoop, Location locNewInnerLoop, |
| 697 | Value ivNewInnerLoop, ValueRange iterArgsNewInnerLoop) { |
| 698 | IRMapping mapping; |
| 699 | if (outerLoop) |
| 700 | mapping.map(from: vectorOpLhs->getOperand( |
| 701 | idx: getIndexPosition(operand: contractOp.getLhs(), loop: outerLoop) + 1), |
| 702 | to: ivOuterLoop); |
| 703 | |
| 704 | mapping.map(from: vectorOpLhs->getOperand( |
| 705 | idx: getIndexPosition(operand: contractOp.getLhs(), loop: innerLoop) + 1), |
| 706 | to: ivNewInnerLoop); |
| 707 | auto lhsClone = rewriterNewInnerLoop.clone(op&: *vectorOpLhs, mapper&: mapping); |
| 708 | |
| 709 | Value indxToStoreInBuffer = c0; |
| 710 | Value indxToLoadFromBuffer = c0; |
| 711 | if (!isVnni) { |
| 712 | if (outerLoop) { |
| 713 | if (innerLoopIndex.value() == 0) { |
| 714 | if (pack) { |
| 715 | ivNewInnerLoop = c0; |
| 716 | ivOuterLoop = arith::AddIOp::create(builder&: rewriter, location: locNewInnerLoop, |
| 717 | lhs: c1, rhs: ivOuterLoop); |
| 718 | |
| 719 | if (!isInnerLoopUBLarger || isInnerLoopUBHasOddQuot) { |
| 720 | indxToStoreInBuffer = arith::RemUIOp::create( |
| 721 | builder&: rewriter, location: locNewInnerLoop, result: rewriter.getIndexType(), |
| 722 | lhs: ivOuterLoop, rhs: c2); |
| 723 | } |
| 724 | |
| 725 | Value indxToLoadFromMatB = arith::AddIOp::create( |
| 726 | builder&: rewriter, location: loc, lhs: indxToStoreInBuffer, rhs: c1); |
| 727 | indxToLoadFromBuffer = arith::RemUIOp::create( |
| 728 | builder&: rewriter, location: loc, result: rewriter.getIndexType(), lhs: indxToLoadFromMatB, |
| 729 | rhs: c2); |
| 730 | } |
| 731 | |
| 732 | } else { |
| 733 | Value nLoadIndx = arith::ConstantIndexOp::create( |
| 734 | builder&: rewriter, location: locNewInnerLoop, value: offset); |
| 735 | ivNewInnerLoop = arith::AddIOp::create(builder&: rewriter, location: locNewInnerLoop, |
| 736 | lhs: nLoadIndx, rhs: ivNewInnerLoop); |
| 737 | indxToStoreInBuffer = getIndxToLoadStoreFromPckBuffer( |
| 738 | rewriter, loc, ivInnerLoop: ivNewInnerLoop, ivOuterLoop, |
| 739 | isInnerLoopUBHasOddQuot, isInnerLoopUBLarger, pack, blockStride: step); |
| 740 | Value indxToLoadFromMatB = |
| 741 | arith::AddIOp::create(builder&: rewriter, location: loc, lhs: indxToStoreInBuffer, rhs: c1); |
| 742 | indxToLoadFromBuffer = |
| 743 | arith::RemUIOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(), |
| 744 | lhs: indxToLoadFromMatB, rhs: c2); |
| 745 | } |
| 746 | } else { |
| 747 | if (pack) { |
| 748 | Value nLoadIndx = arith::ConstantIndexOp::create( |
| 749 | builder&: rewriter, location: locNewInnerLoop, value: offset); |
| 750 | ivNewInnerLoop = arith::AddIOp::create(builder&: rewriter, location: locNewInnerLoop, |
| 751 | lhs: nLoadIndx, rhs: ivNewInnerLoop); |
| 752 | Value quotient_K = arith::DivUIOp::create( |
| 753 | builder&: rewriter, location: loc, lhs: ivNewInnerLoop, rhs: nLoadIndx); |
| 754 | indxToStoreInBuffer = arith::RemUIOp::create( |
| 755 | builder&: rewriter, location: loc, result: rewriter.getIndexType(), lhs: quotient_K, rhs: c2); |
| 756 | |
| 757 | Value indxToLoadFromMatB = |
| 758 | arith::AddIOp::create(builder&: rewriter, location: loc, lhs: indxToStoreInBuffer, rhs: c1); |
| 759 | indxToLoadFromBuffer = |
| 760 | arith::RemUIOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(), |
| 761 | lhs: indxToLoadFromMatB, rhs: c2); |
| 762 | } |
| 763 | } |
| 764 | } |
| 765 | IRMapping rhsMapping; |
| 766 | |
| 767 | Value matB; |
| 768 | Operation *rhsOp = vectorOpRhs; |
| 769 | |
| 770 | // Clone for the subview type operations |
| 771 | if (rhsOp->getNumOperands() > 0) { |
| 772 | |
| 773 | if (outerLoop) { |
| 774 | int64_t outerPos = getIndexPosition(operand: contractOp.getRhs(), loop: outerLoop); |
| 775 | |
| 776 | if (outerPos >= 0) { |
| 777 | unsigned operandIdx = static_cast<unsigned>(outerPos + 1); |
| 778 | |
| 779 | if (operandIdx < rhsOp->getNumOperands()) |
| 780 | rhsMapping.map(from: rhsOp->getOperand(idx: operandIdx), to: ivOuterLoop); |
| 781 | } |
| 782 | } |
| 783 | |
| 784 | int64_t innerPos = getIndexPosition(operand: contractOp.getRhs(), loop: innerLoop); |
| 785 | |
| 786 | if (innerPos >= 0) { |
| 787 | unsigned operandIdx = static_cast<unsigned>(innerPos + 1); |
| 788 | |
| 789 | if (operandIdx < rhsOp->getNumOperands()) |
| 790 | rhsMapping.map(from: rhsOp->getOperand(idx: operandIdx), to: ivNewInnerLoop); |
| 791 | } |
| 792 | |
| 793 | auto rhsClone = rewriterNewInnerLoop.clone(op&: *rhsOp, mapper&: rhsMapping); |
| 794 | matB = rhsClone->getResult(idx: 0); |
| 795 | |
| 796 | } else { |
| 797 | // The mat B is of kind 'memref.get_global @__constant' |
| 798 | matB = rhsOp->getResult(idx: 0); |
| 799 | } |
| 800 | |
| 801 | if (!isVnni) { |
| 802 | if (outerLoop) { |
| 803 | if (!pack) { |
| 804 | matB = Value(); |
| 805 | indxToLoadFromBuffer = c0; |
| 806 | // Use the real spill-block induction value (== spillInnerLoop) |
| 807 | // together with the loop step so the computed ping-pong slot |
| 808 | // matches the prefetch store side for any number of register |
| 809 | // blocks, including odd counts (e.g. 96 = 3 blocks). Passing a |
| 810 | // constant here mis-parities the slot for odd block counts. |
| 811 | indxToLoadFromBuffer = getIndxToLoadStoreFromPckBuffer( |
| 812 | rewriter, loc, ivInnerLoop: ivNewInnerLoop, ivOuterLoop, |
| 813 | isInnerLoopUBHasOddQuot, isInnerLoopUBLarger, pack, blockStride: step); |
| 814 | } |
| 815 | } else { |
| 816 | if (!pack) { |
| 817 | Value nLoadIndx = arith::ConstantIndexOp::create( |
| 818 | builder&: rewriter, location: locNewInnerLoop, value: offset); |
| 819 | matB = Value(); |
| 820 | Value quotient_K = arith::DivUIOp::create( |
| 821 | builder&: rewriter, location: loc, lhs: ivNewInnerLoop, rhs: nLoadIndx); |
| 822 | indxToLoadFromBuffer = arith::RemUIOp::create( |
| 823 | builder&: rewriter, location: loc, result: rewriter.getIndexType(), lhs: quotient_K, rhs: c2); |
| 824 | } |
| 825 | } |
| 826 | } |
| 827 | // compute tiled dot-product |
| 828 | SmallVector<Value> accumulators = createTiledDp( |
| 829 | rewriter, loc: locNewInnerLoop, ops, matA: lhsClone->getResult(idx: 0), matB, |
| 830 | ipType, opType, accIterArgs: iterArgsNewInnerLoop, offset: blockingFactor, isVnni, |
| 831 | packedBuffer, pack, indxToStoreInBuffer, indxToLoadFromMatB: indxToLoadFromBuffer); |
| 832 | |
| 833 | scf::YieldOp::create(builder&: rewriterNewInnerLoop, location: locNewInnerLoop, |
| 834 | results: accumulators); |
| 835 | }); |
| 836 | |
| 837 | return newLoop; |
| 838 | } |
| 839 | |
| 840 | // Implements tiled dot-product operation for a vector.contract operation or a |
| 841 | // sequence of vector.contracts inside the reduction loops. |
| 842 | // |
| 843 | // For example: |
| 844 | // Case 1: register blocked vector.contract with prepacked input |
| 845 | // ``` |
| 846 | // vector.transfer_read %arg0 {{.}*} : memref<16x32x4xi8>, vector<16x16x4xi8> |
| 847 | // vector.transfer_read %arg1 {{.}*} : memref<16x32x4xi8>, vector<16x16x4xi8> |
| 848 | // vector.contract <16x16x4xi8>, <16x16x4xi8> into <16x16xi32> |
| 849 | // vector.transfer_write arg2 {{.}*} : vector<16x16xi32>, memref<32x32xi32> |
| 850 | // ``` |
| 851 | // to |
| 852 | // ``` |
| 853 | // amx.tile_load %arg0 {{.}*} : memref<16x32x4xi8> into !amx.tile<16x64xi8> |
| 854 | // amx.tile_load %arg1 {{.}*} : memref<16x32x4xi8> into !amx.tile<16x64xi8> |
| 855 | // amx.tile_muli !amx.tile<16x64xi8> -> !amx.tile<16x16xi32> |
| 856 | // amx.tile_store %arg2{{.}*} : memref<32x32xi32>, !amx.tile<16x16xi32> |
| 857 | // ``` |
| 858 | // |
| 859 | // |
| 860 | // Case2: vector.contract with register blocked |
| 861 | // |
| 862 | // Output IR with online packing (with s/w pipeline advantage): |
| 863 | // s/w pipeline: load, pack to VNNI, and store the B sub matrix |
| 864 | // of the 0th batch-reduce and K iteration. |
| 865 | // scf.for (0 to 31) { |
| 866 | // - load 0th and 1st vector<32xbf16>, pack into VNNI, store the |
| 867 | // first shuffle in 0th and 2nd shuffle in 16th index of the |
| 868 | // buffer. |
| 869 | // } |
| 870 | // scf.for (0 to br-2) { batch-reduce loop |
| 871 | // scf.for (0 to k-2) { K loop |
| 872 | // - load A matrix |
| 873 | // - scf.loop for s/w pipeline: load, pack to VNNI, and store the B sub |
| 874 | // matrix for the next K loop iteration (c) load VNNI pack B matrix of K |
| 875 | // iteration from the buffer (d) compute the tiled dot-product |
| 876 | // } |
| 877 | // Last iteration of the the K Loop (k-1) { |
| 878 | // - load A matrix |
| 879 | // - scf.loop for s/w pipeline: load, pack to VNNI, and store the B sub |
| 880 | // matrix for the next batch-reduce + K loop iteration (c) load VNNI pack B |
| 881 | // matrix of K iteration from the buffer (d) compute the tiled dot-product |
| 882 | // } |
| 883 | // } |
| 884 | // Last iteration of the batch-reduce loop (br-1) { |
| 885 | // scf.for (0 to k-2) { K loop |
| 886 | // - load A matrix |
| 887 | // - scf.loop for s/w pipeline: load, pack to VNNI, and store the B sub |
| 888 | // matrix for the next K loop iteration (c) load VNNI pack B matrix of K |
| 889 | // iteration from the buffer (d) compute the tiled dot-product |
| 890 | // } |
| 891 | // Last iteration of the the K Loop (k-1) { |
| 892 | // - load A matrix |
| 893 | // - load VNNI pack B matrix of K iteration from the buffer |
| 894 | // - compute the tiled dot-product |
| 895 | // } |
| 896 | // } |
| 897 | // |
| 898 | // scf.for (0 to M) |
| 899 | // scf.for (0 to N) |
| 900 | // - Load the ith and i+1th acc |
| 901 | // - Shuffle them as we packed using vpunpack |
| 902 | // - Load C matrix and do arith.add with the shuffle |
| 903 | // - Store back into C matrix |
| 904 | struct VectorContractToAMXDotProduct : public AddContractionOpRewritePattern { |
| 905 | using Base::Base; |
| 906 | |
| 907 | LogicalResult |
| 908 | matchAndRewriteAddContractOp(vector::ContractionOp contractOp, |
| 909 | PatternRewriter &rewriter) const override { |
| 910 | |
| 911 | unsigned int blockingFactor = |
| 912 | contractOp.getLhsType().getElementType().isBF16() ? 2 : 4; |
| 913 | bool isVnni = |
| 914 | isInVnniLayout(op: contractOp.getOperation(), |
| 915 | indexingMaps: contractOp.getIndexingMapsArray(), blockingFactor); |
| 916 | |
| 917 | VectorType lhsTy = contractOp.getLhsType(); |
| 918 | if (!lhsTy.getElementType().isBF16() && |
| 919 | !lhsTy.getElementType().isSignlessInteger(width: 8) && |
| 920 | !lhsTy.getElementType().isF8E4M3FN() && |
| 921 | !lhsTy.getElementType().isF8E5M2()) |
| 922 | return rewriter.notifyMatchFailure( |
| 923 | arg&: contractOp, msg: "Only BF16/Int8/F8 lowering is supported." ); |
| 924 | |
| 925 | if (lhsTy.getElementType() != contractOp.getRhsType().getElementType()) |
| 926 | return rewriter.notifyMatchFailure( |
| 927 | arg&: contractOp, msg: "Contraction should have same lhs and rhs type." ); |
| 928 | |
| 929 | VectorType accTy = dyn_cast<VectorType>(Val: contractOp.getAccType()); |
| 930 | if (!accTy) |
| 931 | return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Wrong accmulator type." ); |
| 932 | |
| 933 | if (((lhsTy.getElementType().isBF16() || |
| 934 | lhsTy.getElementType().isF8E4M3FN() || |
| 935 | lhsTy.getElementType().isF8E5M2()) && |
| 936 | !accTy.getElementType().isF32()) || |
| 937 | (lhsTy.getElementType().isSignlessInteger(width: 8) && |
| 938 | !accTy.getElementType().isSignlessInteger(width: 32))) |
| 939 | return rewriter.notifyMatchFailure(arg&: contractOp, |
| 940 | msg: "Only F32 for BF16 or Int32 for Int8 " |
| 941 | "accumulation type is supported." ); |
| 942 | |
| 943 | Operation *accReadOp = |
| 944 | traceToVectorReadLikeParentOperation(v: contractOp.getAcc()); |
| 945 | |
| 946 | // Only the contract result's first consumer is needed, not the final |
| 947 | // store. This keeps the lowering independent of the epilogue ops (truncf, |
| 948 | // bias add, ReLU, ...) that sit between the contraction and the write. |
| 949 | Value resultChainEnd = contractionUsersAfterYield(v: contractOp.getResult()); |
| 950 | |
| 951 | if (!accReadOp || !resultChainEnd) |
| 952 | return rewriter.notifyMatchFailure( |
| 953 | arg&: contractOp, msg: "The ACC operand of the vector.contract should be a " |
| 954 | "transfer_read or a load. And, the result should have a " |
| 955 | "single-use chain to its consumer." ); |
| 956 | |
| 957 | bool isAccZeroVectorConstant = isZeroVectorConstant(op: accReadOp); |
| 958 | |
| 959 | Block *resultBlock = resultChainEnd.user_begin()->getBlock(); |
| 960 | |
| 961 | Type ipType = rewriter.getBF16Type(); |
| 962 | Type opType = rewriter.getF32Type(); |
| 963 | |
| 964 | if (lhsTy.getElementType().isSignlessInteger(width: 8)) { |
| 965 | ipType = rewriter.getIntegerType(width: 8); |
| 966 | opType = rewriter.getIntegerType(width: 32); |
| 967 | } |
| 968 | |
| 969 | if (lhsTy.getElementType().isF8E4M3FN()) |
| 970 | ipType = rewriter.getF8E4M3FNType(); |
| 971 | |
| 972 | if (lhsTy.getElementType().isF8E5M2()) |
| 973 | ipType = rewriter.getF8E5M2Type(); |
| 974 | |
| 975 | if (accReadOp->getBlock() == contractOp->getBlock() && |
| 976 | resultBlock != contractOp->getBlock()) |
| 977 | return rewriter.notifyMatchFailure( |
| 978 | arg&: contractOp, msg: "The accumulator store is in different block." ); |
| 979 | |
| 980 | if (accReadOp->getBlock() != contractOp->getBlock() && |
| 981 | resultBlock == contractOp->getBlock()) |
| 982 | return rewriter.notifyMatchFailure( |
| 983 | arg&: contractOp, msg: "The accumulator read is in different block." ); |
| 984 | |
| 985 | if (!(isReadSrcMemref(operand: contractOp.getLhs()) && |
| 986 | isReadSrcMemref(operand: contractOp.getRhs()))) |
| 987 | return rewriter.notifyMatchFailure( |
| 988 | arg&: contractOp, msg: "The LHS or RHS src is not a MemRef type." ); |
| 989 | |
| 990 | if (isVnni && !(isVnniOperandRead(operand: contractOp.getLhs(), vnni: blockingFactor) && |
| 991 | isVnniOperandRead(operand: contractOp.getRhs(), vnni: blockingFactor))) |
| 992 | return rewriter.notifyMatchFailure( |
| 993 | arg&: contractOp, msg: "The LHS or RHS src does not read whole VNNI groups." ); |
| 994 | |
| 995 | unsigned int dimValue = blockingFactor; |
| 996 | if (!isVnni) |
| 997 | dimValue = 16 * blockingFactor; |
| 998 | |
| 999 | // Case 1: For just one VC rewrite. Where all accumulator read/write |
| 1000 | // within the same block. |
| 1001 | if (accReadOp->getBlock() == contractOp->getBlock() && |
| 1002 | resultBlock == contractOp->getBlock()) { |
| 1003 | |
| 1004 | if (!isReadSrcMemref(operand: contractOp.getAcc())) |
| 1005 | return rewriter.notifyMatchFailure(arg&: contractOp, |
| 1006 | msg: "The ACC src is not a MemRef type." ); |
| 1007 | |
| 1008 | bool collapse = false; |
| 1009 | if (isVnni) |
| 1010 | collapse = true; |
| 1011 | |
| 1012 | LogicalResult validate = validateContractOps( |
| 1013 | rewriter, contractOp, blockingFactor: dimValue, srcBuffLhs: Value(), srcBuffRhs: Value(), srcValidate: false); |
| 1014 | |
| 1015 | if (failed(Result: validate)) |
| 1016 | return rewriter.notifyMatchFailure( |
| 1017 | arg&: contractOp, msg: "The contract operation doesn't satisfy the operands " |
| 1018 | "dimensions. M, N, and vnni dims are 16, 16, and 2/4. " |
| 1019 | "The rest dims should be 1. Op should have one user." ); |
| 1020 | |
| 1021 | Location loc = contractOp.getLoc(); |
| 1022 | |
| 1023 | auto srcIndxLhs = getSrcIndxValue(rewriter, loc: contractOp.getLoc(), |
| 1024 | operand: contractOp.getLhs(), isNotAcc: collapse); |
| 1025 | if (failed(Result: srcIndxLhs)) |
| 1026 | return rewriter.notifyMatchFailure(arg&: contractOp, |
| 1027 | msg: "Failed to get the LHS src." ); |
| 1028 | auto [srcBuffLhs, indicesLhs] = *srcIndxLhs; |
| 1029 | |
| 1030 | auto srcIndxRhs = getSrcIndxValue(rewriter, loc: contractOp.getLoc(), |
| 1031 | operand: contractOp.getRhs(), isNotAcc: collapse); |
| 1032 | if (failed(Result: srcIndxRhs)) |
| 1033 | return rewriter.notifyMatchFailure(arg&: contractOp, |
| 1034 | msg: "Failed to get the RHS src." ); |
| 1035 | auto rhsSrc = *srcIndxRhs; |
| 1036 | auto srcBuffRhs = rhsSrc.first; |
| 1037 | auto indicesRhs = rhsSrc.second; |
| 1038 | |
| 1039 | auto srcIndxAcc = getSrcIndxValue(rewriter, loc: contractOp.getLoc(), |
| 1040 | operand: contractOp.getAcc(), isNotAcc: false); |
| 1041 | if (failed(Result: srcIndxAcc)) |
| 1042 | return rewriter.notifyMatchFailure(arg&: contractOp, |
| 1043 | msg: "Failed to get the ACC src." ); |
| 1044 | auto [srcBuffAcc, indicesAcc] = *srcIndxAcc; |
| 1045 | |
| 1046 | Value c0 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 1047 | |
| 1048 | // amx.tile_loads |
| 1049 | auto tileType = amx::TileType::get(shape: {16, (16 * blockingFactor)}, elementType: ipType); |
| 1050 | auto loadLhs = amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType, |
| 1051 | base: srcBuffLhs, indices: indicesLhs); |
| 1052 | |
| 1053 | // Create the subview and then load. |
| 1054 | amx::TileLoadOp loadRhs; |
| 1055 | if (!isVnni) { |
| 1056 | VectorType vecTy; |
| 1057 | SmallVector<OpFoldResult> indexVals; |
| 1058 | llvm::TypeSwitch<Operation *>(contractOp.getRhs().getDefiningOp()) |
| 1059 | .Case<TransferReadOp, LoadOp>(caseFn: [&](auto readOp) { |
| 1060 | indexVals = SmallVector<OpFoldResult>(readOp.getIndices().begin(), |
| 1061 | readOp.getIndices().end()); |
| 1062 | vecTy = readOp.getType(); |
| 1063 | }); |
| 1064 | auto one = rewriter.getIndexAttr(value: 1); |
| 1065 | SmallVector<OpFoldResult> strides(indexVals.size(), one); |
| 1066 | // A rank-reducing read drops leading unit dims. Append them back for |
| 1067 | // the subview. |
| 1068 | SmallVector<int64_t> subviewShape(indexVals.size() - vecTy.getRank(), |
| 1069 | 1); |
| 1070 | llvm::append_range(C&: subviewShape, R: vecTy.getShape()); |
| 1071 | SmallVector<OpFoldResult> sizes = |
| 1072 | getAsIndexOpFoldResult(ctx: contractOp->getContext(), values: subviewShape); |
| 1073 | auto subview = memref::SubViewOp::create(builder&: rewriter, location: loc, source: srcBuffRhs, |
| 1074 | offsets: indexVals, sizes, strides); |
| 1075 | auto bufferType = MemRefType::get(shape: {16, (16 * blockingFactor)}, elementType: ipType); |
| 1076 | auto packedBuffer = memref::AllocaOp::create(builder&: rewriter, location: loc, memrefType: bufferType); |
| 1077 | |
| 1078 | // Create a loop that does online packing. |
| 1079 | Value step = |
| 1080 | arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: blockingFactor); |
| 1081 | Value uBound = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, |
| 1082 | value: (blockingFactor * 16)); |
| 1083 | Value nextLoadIndx = |
| 1084 | arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: (blockingFactor / 2)); |
| 1085 | Value nextStoreIndx = arith::ConstantIndexOp::create( |
| 1086 | builder&: rewriter, location: loc, value: 16 * (blockingFactor / 2)); |
| 1087 | |
| 1088 | scf::ForOp::create( |
| 1089 | builder&: rewriter, location: loc, lowerBound: c0, upperBound: uBound, step, initArgs: ValueRange{}, |
| 1090 | odsArg4: [&](OpBuilder &nestedBuilder, Location loc, Value iv, |
| 1091 | ValueRange iterArgs) { |
| 1092 | Value i1_load = |
| 1093 | arith::AddIOp::create(builder&: rewriter, location: loc, lhs: nextLoadIndx, rhs: iv); |
| 1094 | |
| 1095 | // The source subview already accounts for the leading unit |
| 1096 | // dimensions. |
| 1097 | size_t numOffsets = indicesRhs.size(); |
| 1098 | SmallVector<Value> indices(numOffsets, c0); |
| 1099 | indices[numOffsets - 2] = iv; |
| 1100 | auto vec1 = vector::LoadOp::create( |
| 1101 | builder&: rewriter, location: loc, |
| 1102 | resultType: VectorType::get(shape: 16 * (blockingFactor / 2), elementType: ipType), base: subview, |
| 1103 | indices); |
| 1104 | |
| 1105 | indices[numOffsets - 2] = i1_load; |
| 1106 | auto vec2 = vector::LoadOp::create( |
| 1107 | builder&: rewriter, location: loc, |
| 1108 | resultType: VectorType::get(shape: 16 * (blockingFactor / 2), elementType: ipType), base: subview, |
| 1109 | indices); |
| 1110 | |
| 1111 | vector::ShuffleOp writeVal1; |
| 1112 | vector::ShuffleOp writeVal2; |
| 1113 | |
| 1114 | if (blockingFactor == 2) { |
| 1115 | |
| 1116 | writeVal1 = vector::ShuffleOp::create( |
| 1117 | builder&: rewriter, location: loc, result: VectorType::get(shape: {16}, elementType: ipType), v1: vec1, v2: vec2, |
| 1118 | mask: ArrayRef<int64_t>{0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, |
| 1119 | 6, 22, 7, 23}); |
| 1120 | |
| 1121 | writeVal2 = vector::ShuffleOp::create( |
| 1122 | builder&: rewriter, location: loc, result: VectorType::get(shape: {16}, elementType: ipType), v1: vec1, v2: vec2, |
| 1123 | mask: ArrayRef<int64_t>{8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, |
| 1124 | 29, 14, 30, 15, 31}); |
| 1125 | } |
| 1126 | |
| 1127 | if (blockingFactor == 4) { |
| 1128 | writeVal1 = vector::ShuffleOp::create( |
| 1129 | builder&: rewriter, location: loc, result: VectorType::get(shape: {32}, elementType: ipType), v1: vec1, v2: vec2, |
| 1130 | mask: ArrayRef<int64_t>{0, 16, 32, 48, 1, 17, 33, 49, |
| 1131 | 2, 18, 34, 50, 3, 19, 35, 51, |
| 1132 | 4, 20, 36, 52, 5, 21, 37, 53, |
| 1133 | 6, 22, 38, 54, 7, 23, 39, 55}); |
| 1134 | |
| 1135 | writeVal2 = vector::ShuffleOp::create( |
| 1136 | builder&: rewriter, location: loc, result: VectorType::get(shape: {32}, elementType: ipType), v1: vec1, v2: vec2, |
| 1137 | mask: ArrayRef<int64_t>{8, 24, 40, 56, 9, 25, 41, 57, |
| 1138 | 10, 26, 42, 58, 11, 27, 43, 59, |
| 1139 | 12, 28, 44, 60, 13, 29, 45, 61, |
| 1140 | 14, 30, 46, 62, 15, 31, 47, 63}); |
| 1141 | } |
| 1142 | |
| 1143 | auto rem = arith::DivUIOp::create( |
| 1144 | builder&: rewriter, location: loc, result: rewriter.getIndexType(), lhs: iv, rhs: step); |
| 1145 | |
| 1146 | vector::StoreOp::create(builder&: rewriter, location: loc, valueToStore: writeVal1, base: packedBuffer, |
| 1147 | indices: ValueRange{rem, c0}); |
| 1148 | vector::StoreOp::create(builder&: rewriter, location: loc, valueToStore: writeVal2, base: packedBuffer, |
| 1149 | indices: ValueRange{rem, nextStoreIndx}); |
| 1150 | |
| 1151 | scf::YieldOp::create(builder&: nestedBuilder, location: loc); |
| 1152 | }); |
| 1153 | loadRhs = amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType, base: packedBuffer, |
| 1154 | indices: ValueRange{c0, c0}); |
| 1155 | } else { |
| 1156 | |
| 1157 | loadRhs = amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType, base: srcBuffRhs, |
| 1158 | indices: indicesRhs); |
| 1159 | } |
| 1160 | |
| 1161 | auto tileTypeAcc = amx::TileType::get(shape: {16, 16}, elementType: opType); |
| 1162 | auto loadAcc = amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileTypeAcc, |
| 1163 | base: srcBuffAcc, indices: indicesAcc); |
| 1164 | |
| 1165 | // Tiled dot-product. |
| 1166 | Value dp; |
| 1167 | if (ipType.isBF16() || ipType.isF8E5M2() || ipType.isF8E4M3FN()) |
| 1168 | dp = amx::TileMulFOp::create(builder&: rewriter, location: loc, res: tileTypeAcc, lhs: loadLhs, |
| 1169 | rhs: loadRhs, acc: loadAcc); |
| 1170 | |
| 1171 | if (ipType.isSignlessInteger(width: 8)) |
| 1172 | dp = amx::TileMulIOp::create(builder&: rewriter, location: loc, res: tileTypeAcc, lhs: loadLhs, |
| 1173 | rhs: loadRhs, acc: loadAcc); |
| 1174 | |
| 1175 | auto bufferType = MemRefType::get(shape: {16, 16}, elementType: opType); |
| 1176 | auto resultBuffer = memref::AllocaOp::create(builder&: rewriter, location: loc, memrefType: bufferType); |
| 1177 | |
| 1178 | amx::TileStoreOp::create(builder&: rewriter, location: loc, base: resultBuffer, indices: ValueRange{c0, c0}, |
| 1179 | val: dp); |
| 1180 | |
| 1181 | auto vectorType = VectorType::get(shape: {16, 16}, elementType: opType); |
| 1182 | int64_t srcRank = |
| 1183 | (dyn_cast<ShapedType>(Val: resultBuffer.getType())).getRank(); |
| 1184 | Value padding = ub::PoisonOp::create(builder&: rewriter, location: loc, result: opType); |
| 1185 | auto map = AffineMap::getMinorIdentityMap(dims: srcRank, results: vectorType.getRank(), |
| 1186 | context: rewriter.getContext()); |
| 1187 | SmallVector<bool> inBounds(vectorType.getRank(), true); |
| 1188 | |
| 1189 | Value vecRow = vector::TransferReadOp::create( |
| 1190 | builder&: rewriter, location: loc, vectorType, source: resultBuffer, indices: ValueRange{c0, c0}, padding, |
| 1191 | permutationMap: map, inBounds); |
| 1192 | |
| 1193 | Value resultOp = contractionUsersAfterYield(v: contractOp.getResult()); |
| 1194 | if (auto vecType = dyn_cast<VectorType>(Val: resultOp.getType())) |
| 1195 | vecRow = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: vecType, source: vecRow); |
| 1196 | |
| 1197 | rewriter.replaceAllUsesWith(from: resultOp, to: vecRow); |
| 1198 | return success(); |
| 1199 | } |
| 1200 | |
| 1201 | // Case 2: The acc are passed as iter args through the reduction loop. |
| 1202 | // We support, reduction loop depth until 2. TODO: Support for n-depth |
| 1203 | // reduction loop. |
| 1204 | // TODOs: Re-factor 2a and 2b. |
| 1205 | SmallVector<scf::ForOp> loopLists; |
| 1206 | Operation *current = contractOp; |
| 1207 | while (true) { |
| 1208 | Operation *parent = current->getParentOfType<scf::ForOp>(); |
| 1209 | |
| 1210 | if (!parent) { |
| 1211 | // The accumulator initialization can be hoisted above an enclosing |
| 1212 | // parallel region (scf.parallel/scf.forall) when the register tile |
| 1213 | // matches the problem size and the M/N register loops fold away. In |
| 1214 | // that case the reduction loop(s) collected so far are still valid to |
| 1215 | // rewrite, so stop climbing instead of bailing out. |
| 1216 | if (!loopLists.empty()) |
| 1217 | break; |
| 1218 | return rewriter.notifyMatchFailure( |
| 1219 | arg&: contractOp, |
| 1220 | msg: "Accumulator read and contract op not within scf.for op" ); |
| 1221 | } |
| 1222 | |
| 1223 | loopLists.push_back(Elt: dyn_cast<scf::ForOp>(Val: parent)); |
| 1224 | |
| 1225 | if (resultBlock == parent->getBlock()) { |
| 1226 | break; |
| 1227 | } |
| 1228 | |
| 1229 | current = parent; |
| 1230 | } |
| 1231 | |
| 1232 | if (loopLists.size() > 2 || loopLists.size() == 0) |
| 1233 | return rewriter.notifyMatchFailure( |
| 1234 | arg&: contractOp, msg: "Rewrite is supported until reduction loop depth of 2." ); |
| 1235 | |
| 1236 | auto srcIndxLhs = getSrcIndxValue(rewriter, loc: contractOp.getLoc(), |
| 1237 | operand: contractOp.getLhs(), isNotAcc: false); |
| 1238 | if (failed(Result: srcIndxLhs)) |
| 1239 | return rewriter.notifyMatchFailure(arg&: contractOp, |
| 1240 | msg: "Failed to get the LHS src." ); |
| 1241 | auto [srcBuffLhs, indicesLhs] = *srcIndxLhs; |
| 1242 | |
| 1243 | auto srcIndxRhs = getSrcIndxValue(rewriter, loc: contractOp.getLoc(), |
| 1244 | operand: contractOp.getRhs(), isNotAcc: false); |
| 1245 | if (failed(Result: srcIndxRhs)) |
| 1246 | return rewriter.notifyMatchFailure(arg&: contractOp, |
| 1247 | msg: "Failed to get the RHS src." ); |
| 1248 | auto [srcBuffRhs, indicesRhs] = *srcIndxRhs; |
| 1249 | Operation *vectorOpLhs; |
| 1250 | llvm::TypeSwitch<Operation *>(contractOp.getLhs().getDefiningOp()) |
| 1251 | .Case<TransferReadOp, LoadOp>(caseFn: [&](auto readOp) { |
| 1252 | vectorOpLhs = readOp.getBase().getDefiningOp(); |
| 1253 | }); |
| 1254 | |
| 1255 | Operation *vectorOpRhs; |
| 1256 | llvm::TypeSwitch<Operation *>(contractOp.getRhs().getDefiningOp()) |
| 1257 | .Case<TransferReadOp, LoadOp>(caseFn: [&](auto readOp) { |
| 1258 | vectorOpRhs = readOp.getBase().getDefiningOp(); |
| 1259 | }); |
| 1260 | |
| 1261 | if (!vectorOpLhs || !vectorOpRhs) |
| 1262 | return rewriter.notifyMatchFailure( |
| 1263 | arg&: contractOp, msg: "Failed to find LHS or RHS read source operation" ); |
| 1264 | |
| 1265 | // Retrive all the contaction operation within the loop. |
| 1266 | SmallVector<vector::ContractionOp> ops; |
| 1267 | // Accumulators whose initial value passed into the rewritten reduction |
| 1268 | // loops is neither read from memory nor zero, e.g. an iter_arg of an |
| 1269 | // enclosing loop. Paired with the index of their contraction in `ops`. |
| 1270 | SmallVector<std::pair<size_t, Value>> carriedAccs; |
| 1271 | for (Operation &op : loopLists[0].getBody()->getOperations()) { |
| 1272 | |
| 1273 | if (auto contract = dyn_cast<vector::ContractionOp>(Val&: op)) { |
| 1274 | if (isVnni && !(isVnniOperandRead(operand: contract.getLhs(), vnni: blockingFactor) && |
| 1275 | isVnniOperandRead(operand: contract.getRhs(), vnni: blockingFactor))) |
| 1276 | return rewriter.notifyMatchFailure( |
| 1277 | arg&: contractOp, msg: "The LHS or RHS src of an associated contract " |
| 1278 | "operation does not read whole VNNI groups." ); |
| 1279 | |
| 1280 | LogicalResult validate = validateContractOps( |
| 1281 | rewriter, contractOp: contract, blockingFactor: dimValue, srcBuffLhs, srcBuffRhs, srcValidate: true); |
| 1282 | |
| 1283 | if (failed(Result: validate)) |
| 1284 | return rewriter.notifyMatchFailure( |
| 1285 | arg&: contractOp, |
| 1286 | msg: "The associated contract operations doesn't satisfy " |
| 1287 | "the re-write conditions either the dimensions are " |
| 1288 | "wrong or MemRef source are different or many users." ); |
| 1289 | |
| 1290 | Operation *contractReadOp = |
| 1291 | traceToVectorReadLikeParentOperation(v: contract.getAcc()); |
| 1292 | |
| 1293 | if (isAccZeroVectorConstant != isZeroVectorConstant(op: contractReadOp)) |
| 1294 | return rewriter.notifyMatchFailure( |
| 1295 | arg&: contractOp, msg: "The input acc should be a zero constant or " |
| 1296 | "transfer_read for all contracts" ); |
| 1297 | |
| 1298 | Value accInit = getLoopNestInit(acc: contract.getAcc(), loops: loopLists); |
| 1299 | Operation *accInitOp = accInit.getDefiningOp(); |
| 1300 | bool carried = !isa_and_present<TransferReadOp, LoadOp>(Val: accInitOp) && |
| 1301 | !isZeroVectorConstant(op: accInitOp); |
| 1302 | if (carried) { |
| 1303 | if (loopLists.back().getRegion().isAncestor( |
| 1304 | other: accInit.getParentRegion())) |
| 1305 | return rewriter.notifyMatchFailure( |
| 1306 | arg&: contractOp, msg: "The accumulator should enter the reduction loops " |
| 1307 | "through their init args." ); |
| 1308 | carriedAccs.emplace_back(Args: ops.size(), Args&: accInit); |
| 1309 | } |
| 1310 | |
| 1311 | ops.push_back(Elt: contract); |
| 1312 | } |
| 1313 | } |
| 1314 | |
| 1315 | // The accumulators read from memory are added back to all the results at |
| 1316 | // once, so they can't be mixed with carried ones. |
| 1317 | if (!isAccZeroVectorConstant && !carriedAccs.empty() && |
| 1318 | carriedAccs.size() != ops.size()) |
| 1319 | return rewriter.notifyMatchFailure( |
| 1320 | arg&: contractOp, msg: "The accumulators read from memory can't be mixed with " |
| 1321 | "carried in accumulators." ); |
| 1322 | |
| 1323 | if (!isVnni) { |
| 1324 | unsigned int pairCount = 0; |
| 1325 | for (size_t j = 0; j < ops.size(); j++) { |
| 1326 | for (size_t i = j; i < ops.size(); i++) { |
| 1327 | if (i != j && validatePairVectorContract(contractOp: ops[j], pairContOp: ops[i], rhsHasMultipleNonUnitDims: true, nonUnitDimValue: 16)) |
| 1328 | pairCount = pairCount + 2; |
| 1329 | } |
| 1330 | } |
| 1331 | |
| 1332 | if (pairCount != ops.size()) |
| 1333 | return rewriter.notifyMatchFailure( |
| 1334 | arg&: contractOp, msg: "Coudn't find the pair vector contract " ); |
| 1335 | } |
| 1336 | |
| 1337 | scf::ForOp innerLoop; |
| 1338 | scf::ForOp outerLoop; |
| 1339 | |
| 1340 | // For VNNI layout, accumulator tiles stay in their original layout. |
| 1341 | // Load and store them back in place when possible. |
| 1342 | // Otherwise, start from zero tile and accumulate only the final result. |
| 1343 | std::optional<SmallVector<Operation *>> inPlaceAccReads, inPlaceAccWrites; |
| 1344 | if (isVnni) |
| 1345 | std::tie(args&: inPlaceAccReads, args&: inPlaceAccWrites) = |
| 1346 | getInPlaceAccAccesses(ops, /*writeOnly=*/isAccZeroVectorConstant); |
| 1347 | auto createInitTiles = [&](scf::ForOp loop) { |
| 1348 | if (inPlaceAccReads) |
| 1349 | return createAccTileLoads(rewriter, loc: loop.getLoc(), opType, loop, |
| 1350 | reads: *inPlaceAccReads); |
| 1351 | return createTileZeros(rewriter, loc: loop.getLoc(), opType, outerLoop: loop, size: ops.size()); |
| 1352 | }; |
| 1353 | |
| 1354 | scf::ForOp newLoop; |
| 1355 | // Case 2a: Reduction loop depth is 2. |
| 1356 | if (loopLists.size() == 2) { |
| 1357 | outerLoop = loopLists[1]; |
| 1358 | innerLoop = loopLists[0]; |
| 1359 | |
| 1360 | LogicalResult validateOuterLoopStep = |
| 1361 | validateLoopStep(rewriter, step: outerLoop.getStep(), value: 1); |
| 1362 | if (failed(Result: validateOuterLoopStep)) |
| 1363 | return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Invalid loop step." ); |
| 1364 | |
| 1365 | int64_t stepValue = 16; |
| 1366 | if (!isVnni) |
| 1367 | stepValue = stepValue * blockingFactor; |
| 1368 | LogicalResult validateInnerLoopStep = |
| 1369 | validateLoopStep(rewriter, step: innerLoop.getStep(), value: stepValue); |
| 1370 | if (failed(Result: validateInnerLoopStep)) |
| 1371 | return rewriter.notifyMatchFailure( |
| 1372 | arg&: contractOp, msg: "Invalid loop step. The step should be 32 for BF16 and " |
| 1373 | "64 for Int8/F8." ); |
| 1374 | |
| 1375 | SmallVector<Value> loopItrArgs = createInitTiles(outerLoop); |
| 1376 | rewriter.setInsertionPoint(outerLoop); |
| 1377 | |
| 1378 | if (isVnni) { |
| 1379 | newLoop = scf::ForOp::create( |
| 1380 | builder&: rewriter, location: outerLoop.getLoc(), lowerBound: outerLoop.getLowerBound(), |
| 1381 | upperBound: outerLoop.getUpperBound(), step: outerLoop.getStep(), initArgs: loopItrArgs, |
| 1382 | odsArg4: [&](OpBuilder &rewriterOuterLoop, Location locOuterLoop, |
| 1383 | Value ivOuterLoop, ValueRange iterArgsOuterLoop) { |
| 1384 | auto newInnerLoop = createLoops( |
| 1385 | rewriter, loc: innerLoop.getLoc(), lowerBound: innerLoop.getLowerBound(), |
| 1386 | upperBound: innerLoop.getUpperBound(), step: innerLoop.getStep(), |
| 1387 | loopItrArgs: iterArgsOuterLoop, ipType, opType, blockingFactor, isVnni, |
| 1388 | vectorOpLhs, vectorOpRhs, contractOp, outerLoop, innerLoop, |
| 1389 | ops, ivOuterLoop, packedBuffer: nullptr, pack: true, innerLoopIndex: nullptr, isInnerLoopUBLarger: false, isInnerLoopUBHasOddQuot: false); |
| 1390 | |
| 1391 | scf::YieldOp::create(builder&: rewriterOuterLoop, location: locOuterLoop, |
| 1392 | results: newInnerLoop.getResults()); |
| 1393 | }); |
| 1394 | |
| 1395 | } else { |
| 1396 | |
| 1397 | bool isInnerLoopUBLarger = false; |
| 1398 | bool isInnerLoopUBHasOddQuot = false; |
| 1399 | |
| 1400 | int64_t ubVal = 16 * blockingFactor; |
| 1401 | Value ub = innerLoop.getUpperBound(); |
| 1402 | if (auto constOp = ub.getDefiningOp<arith::ConstantOp>()) { |
| 1403 | if (auto intAttr = dyn_cast<IntegerAttr>(Val: constOp.getValue())) { |
| 1404 | ubVal = intAttr.getInt(); |
| 1405 | } |
| 1406 | } |
| 1407 | |
| 1408 | isInnerLoopUBLarger = ubVal > 16 * blockingFactor; |
| 1409 | isInnerLoopUBHasOddQuot = |
| 1410 | (((ubVal / (16 * blockingFactor)) % 2) == 1) && isInnerLoopUBLarger; |
| 1411 | |
| 1412 | rewriter.setInsertionPoint(outerLoop); |
| 1413 | |
| 1414 | auto c0 = |
| 1415 | arith::ConstantIndexOp::create(builder&: rewriter, location: outerLoop.getLoc(), value: 0); |
| 1416 | auto c1 = |
| 1417 | arith::ConstantIndexOp::create(builder&: rewriter, location: outerLoop.getLoc(), value: 1); |
| 1418 | auto spillLoopBound = arith::ConstantIndexOp::create( |
| 1419 | builder&: rewriter, location: outerLoop.getLoc(), value: 16 * blockingFactor); |
| 1420 | |
| 1421 | Value spillOuterLoop = arith::SubIOp::create( |
| 1422 | builder&: rewriter, location: outerLoop.getLoc(), lhs: outerLoop.getUpperBound(), rhs: c1); |
| 1423 | Value spillInnerLoop = |
| 1424 | arith::SubIOp::create(builder&: rewriter, location: innerLoop.getLoc(), |
| 1425 | lhs: innerLoop.getUpperBound(), rhs: spillLoopBound); |
| 1426 | auto bufferType = |
| 1427 | MemRefType::get(shape: {2, 32, (blockingFactor * 16)}, elementType: ipType); |
| 1428 | auto packedBuffer = |
| 1429 | memref::AllocaOp::create(builder&: rewriter, location: outerLoop.getLoc(), memrefType: bufferType); |
| 1430 | |
| 1431 | // First Shuffling outside the reduction loops |
| 1432 | IRMapping rhsMapping; |
| 1433 | rhsMapping.map( |
| 1434 | from: vectorOpRhs->getOperand( |
| 1435 | idx: getIndexPosition(operand: contractOp.getRhs(), loop: outerLoop) + 1), |
| 1436 | to: outerLoop.getLowerBound()); |
| 1437 | rhsMapping.map( |
| 1438 | from: vectorOpRhs->getOperand( |
| 1439 | idx: getIndexPosition(operand: contractOp.getRhs(), loop: innerLoop) + 1), |
| 1440 | to: innerLoop.getLowerBound()); |
| 1441 | auto rhsClone = rewriter.clone(op&: *vectorOpRhs, mapper&: rhsMapping); |
| 1442 | |
| 1443 | Value quotient_batch = arith::DivUIOp::create( |
| 1444 | builder&: rewriter, location: outerLoop.getLoc(), lhs: outerLoop.getLowerBound(), |
| 1445 | rhs: outerLoop.getStep()); |
| 1446 | Value quotient_k = arith::DivUIOp::create(builder&: rewriter, location: outerLoop.getLoc(), |
| 1447 | lhs: innerLoop.getLowerBound(), |
| 1448 | rhs: innerLoop.getStep()); |
| 1449 | |
| 1450 | Value quotient_add = arith::AddIOp::create(builder&: rewriter, location: outerLoop.getLoc(), |
| 1451 | lhs: quotient_batch, rhs: quotient_k); |
| 1452 | Value c2 = |
| 1453 | arith::ConstantIndexOp::create(builder&: rewriter, location: outerLoop.getLoc(), value: 2); |
| 1454 | Value rem = arith::RemUIOp::create(builder&: rewriter, location: outerLoop.getLoc(), |
| 1455 | lhs: quotient_add, rhs: c2); |
| 1456 | |
| 1457 | performShuffle(rewriter, loc: outerLoop.getLoc(), matB: rhsClone->getResult(idx: 0), |
| 1458 | ipType, offset: blockingFactor, packedBuffer, indxToStoreInBuffer: rem); |
| 1459 | |
| 1460 | // First Set of Loops |
| 1461 | auto newLoopNonSpill = scf::ForOp::create( |
| 1462 | builder&: rewriter, location: outerLoop.getLoc(), lowerBound: outerLoop.getLowerBound(), |
| 1463 | upperBound: spillOuterLoop, step: outerLoop.getStep(), initArgs: loopItrArgs, |
| 1464 | odsArg4: [&](OpBuilder &rewriterOuterLoop, Location locOuterLoop, |
| 1465 | Value ivOuterLoop, ValueRange iterArgsOuterLoop) { |
| 1466 | auto newInnerLoop1 = createLoops( |
| 1467 | rewriter, loc: innerLoop.getLoc(), lowerBound: innerLoop.getLowerBound(), |
| 1468 | upperBound: spillInnerLoop, step: innerLoop.getStep(), loopItrArgs: iterArgsOuterLoop, |
| 1469 | ipType, opType, blockingFactor, isVnni, vectorOpLhs, |
| 1470 | vectorOpRhs, contractOp, outerLoop, innerLoop, ops, |
| 1471 | ivOuterLoop, packedBuffer, pack: true, innerLoopIndex: spillLoopBound, |
| 1472 | isInnerLoopUBLarger, isInnerLoopUBHasOddQuot); |
| 1473 | |
| 1474 | auto newInnerLoop = createLoops( |
| 1475 | rewriter, loc: innerLoop.getLoc(), lowerBound: spillInnerLoop, |
| 1476 | upperBound: innerLoop.getUpperBound(), step: innerLoop.getStep(), |
| 1477 | loopItrArgs: newInnerLoop1.getResults(), ipType, opType, blockingFactor, |
| 1478 | isVnni, vectorOpLhs, vectorOpRhs, contractOp, outerLoop, |
| 1479 | innerLoop, ops, ivOuterLoop, packedBuffer, pack: true, innerLoopIndex: c0, |
| 1480 | isInnerLoopUBLarger, isInnerLoopUBHasOddQuot); |
| 1481 | |
| 1482 | scf::YieldOp::create(builder&: rewriterOuterLoop, location: locOuterLoop, |
| 1483 | results: newInnerLoop.getResults()); |
| 1484 | }); |
| 1485 | |
| 1486 | // Last set of Loops |
| 1487 | newLoop = scf::ForOp::create( |
| 1488 | builder&: rewriter, location: outerLoop.getLoc(), lowerBound: spillOuterLoop, |
| 1489 | upperBound: outerLoop.getUpperBound(), step: outerLoop.getStep(), |
| 1490 | initArgs: newLoopNonSpill.getResults(), |
| 1491 | odsArg4: [&](OpBuilder &rewriterOuterLoop, Location locOuterLoop, |
| 1492 | Value ivOuterLoop, ValueRange iterArgsOuterLoop) { |
| 1493 | auto newInnerLoop1 = createLoops( |
| 1494 | rewriter, loc: innerLoop.getLoc(), lowerBound: innerLoop.getLowerBound(), |
| 1495 | upperBound: spillInnerLoop, step: innerLoop.getStep(), loopItrArgs: iterArgsOuterLoop, |
| 1496 | ipType, opType, blockingFactor, isVnni, vectorOpLhs, |
| 1497 | vectorOpRhs, contractOp, outerLoop, innerLoop, ops, |
| 1498 | ivOuterLoop, packedBuffer, pack: true, innerLoopIndex: spillLoopBound, |
| 1499 | isInnerLoopUBLarger, isInnerLoopUBHasOddQuot); |
| 1500 | |
| 1501 | auto newInnerLoop = createLoops( |
| 1502 | rewriter, loc: innerLoop.getLoc(), lowerBound: spillInnerLoop, |
| 1503 | upperBound: innerLoop.getUpperBound(), step: innerLoop.getStep(), |
| 1504 | loopItrArgs: newInnerLoop1.getResults(), ipType, opType, blockingFactor, |
| 1505 | isVnni, vectorOpLhs, vectorOpRhs, contractOp, outerLoop, |
| 1506 | innerLoop, ops, ivOuterLoop, packedBuffer, pack: false, innerLoopIndex: c0, |
| 1507 | isInnerLoopUBLarger, isInnerLoopUBHasOddQuot); |
| 1508 | |
| 1509 | scf::YieldOp::create(builder&: rewriterOuterLoop, location: locOuterLoop, |
| 1510 | results: newInnerLoop.getResults()); |
| 1511 | }); |
| 1512 | } |
| 1513 | } |
| 1514 | |
| 1515 | // Case 2b: Reduction loop depth is 1. |
| 1516 | if (loopLists.size() == 1) { |
| 1517 | |
| 1518 | innerLoop = loopLists[0]; |
| 1519 | int64_t stepValue = 16; |
| 1520 | if (!isVnni) |
| 1521 | stepValue = stepValue * blockingFactor; |
| 1522 | |
| 1523 | LogicalResult validateInnerLoopStep = |
| 1524 | validateLoopStep(rewriter, step: innerLoop.getStep(), value: stepValue); |
| 1525 | if (failed(Result: validateInnerLoopStep)) |
| 1526 | return rewriter.notifyMatchFailure( |
| 1527 | arg&: contractOp, |
| 1528 | msg: "Invalid loop step. The step should be 32 for BF16 and " |
| 1529 | "64 for Int8/F8 or 1 if it is rduction loop other than K." ); |
| 1530 | |
| 1531 | SmallVector<Value> loopItrArgs = createInitTiles(innerLoop); |
| 1532 | rewriter.setInsertionPoint(innerLoop); |
| 1533 | |
| 1534 | if (isVnni) { |
| 1535 | newLoop = createLoops( |
| 1536 | rewriter, loc: innerLoop.getLoc(), lowerBound: innerLoop.getLowerBound(), |
| 1537 | upperBound: innerLoop.getUpperBound(), step: innerLoop.getStep(), loopItrArgs, ipType, |
| 1538 | opType, blockingFactor, isVnni, vectorOpLhs, vectorOpRhs, |
| 1539 | contractOp, outerLoop: nullptr, innerLoop, ops, ivOuterLoop: nullptr, packedBuffer: nullptr, pack: true, |
| 1540 | innerLoopIndex: nullptr, isInnerLoopUBLarger: false, isInnerLoopUBHasOddQuot: false); |
| 1541 | |
| 1542 | } else { |
| 1543 | |
| 1544 | bool isInnerLoopUBLarger = false; |
| 1545 | bool isInnerLoopUBHasOddQuot = false; |
| 1546 | |
| 1547 | int64_t ubVal = 16 * blockingFactor; |
| 1548 | Value ub = innerLoop.getUpperBound(); |
| 1549 | if (auto constOp = ub.getDefiningOp<arith::ConstantOp>()) { |
| 1550 | if (auto intAttr = dyn_cast<IntegerAttr>(Val: constOp.getValue())) { |
| 1551 | ubVal = intAttr.getInt(); |
| 1552 | } |
| 1553 | } |
| 1554 | |
| 1555 | isInnerLoopUBLarger = ubVal > 16 * blockingFactor; |
| 1556 | isInnerLoopUBHasOddQuot = |
| 1557 | (((ubVal / (16 * blockingFactor)) % 2) == 1) && isInnerLoopUBLarger; |
| 1558 | |
| 1559 | rewriter.setInsertionPoint(innerLoop); |
| 1560 | |
| 1561 | auto c0 = |
| 1562 | arith::ConstantIndexOp::create(builder&: rewriter, location: innerLoop.getLoc(), value: 0); |
| 1563 | int64_t offset = 16 * blockingFactor; |
| 1564 | if (auto cst = |
| 1565 | innerLoop.getStep().getDefiningOp<arith::ConstantIndexOp>()) |
| 1566 | offset = cst.value(); |
| 1567 | |
| 1568 | auto spillLoopBound = arith::ConstantIndexOp::create( |
| 1569 | builder&: rewriter, location: innerLoop.getLoc(), value: offset); |
| 1570 | Value spillInnerLoop = |
| 1571 | arith::SubIOp::create(builder&: rewriter, location: innerLoop.getLoc(), |
| 1572 | lhs: innerLoop.getUpperBound(), rhs: spillLoopBound); |
| 1573 | |
| 1574 | auto bufferType = |
| 1575 | MemRefType::get(shape: {2, 32, (blockingFactor * 16)}, elementType: ipType); |
| 1576 | auto packedBuffer = |
| 1577 | memref::AllocaOp::create(builder&: rewriter, location: innerLoop.getLoc(), memrefType: bufferType); |
| 1578 | |
| 1579 | // First Shuffling outside the reduction loops |
| 1580 | IRMapping rhsMapping; |
| 1581 | rhsMapping.map( |
| 1582 | from: vectorOpRhs->getOperand( |
| 1583 | idx: getIndexPosition(operand: contractOp.getRhs(), loop: innerLoop) + 1), |
| 1584 | to: innerLoop.getLowerBound()); |
| 1585 | auto rhsClone = rewriter.clone(op&: *vectorOpRhs, mapper&: rhsMapping); |
| 1586 | |
| 1587 | Value quotient_k = arith::DivUIOp::create(builder&: rewriter, location: innerLoop.getLoc(), |
| 1588 | lhs: innerLoop.getLowerBound(), |
| 1589 | rhs: innerLoop.getStep()); |
| 1590 | Value c2 = |
| 1591 | arith::ConstantIndexOp::create(builder&: rewriter, location: innerLoop.getLoc(), value: 2); |
| 1592 | Value rem = arith::RemUIOp::create(builder&: rewriter, location: innerLoop.getLoc(), |
| 1593 | lhs: quotient_k, rhs: c2); |
| 1594 | |
| 1595 | performShuffle(rewriter, loc: innerLoop.getLoc(), matB: rhsClone->getResult(idx: 0), |
| 1596 | ipType, offset: blockingFactor, packedBuffer, indxToStoreInBuffer: rem); |
| 1597 | |
| 1598 | auto newLoopNonSpill = createLoops( |
| 1599 | rewriter, loc: innerLoop.getLoc(), lowerBound: innerLoop.getLowerBound(), |
| 1600 | upperBound: spillInnerLoop, step: innerLoop.getStep(), loopItrArgs, ipType, opType, |
| 1601 | blockingFactor, isVnni, vectorOpLhs, vectorOpRhs, contractOp, |
| 1602 | outerLoop: nullptr, innerLoop, ops, ivOuterLoop: nullptr, packedBuffer, pack: true, |
| 1603 | innerLoopIndex: spillLoopBound, isInnerLoopUBLarger, isInnerLoopUBHasOddQuot); |
| 1604 | |
| 1605 | newLoop = createLoops(rewriter, loc: innerLoop.getLoc(), lowerBound: spillInnerLoop, |
| 1606 | upperBound: innerLoop.getUpperBound(), step: innerLoop.getStep(), |
| 1607 | loopItrArgs: newLoopNonSpill.getResults(), ipType, opType, |
| 1608 | blockingFactor, isVnni, vectorOpLhs, vectorOpRhs, |
| 1609 | contractOp, outerLoop: nullptr, innerLoop, ops, ivOuterLoop: nullptr, |
| 1610 | packedBuffer, pack: false, innerLoopIndex: c0, isInnerLoopUBLarger, |
| 1611 | isInnerLoopUBHasOddQuot); |
| 1612 | } |
| 1613 | |
| 1614 | // This helps the final store back to the acc uses the same code for |
| 1615 | // the both reduction loop depth 1 or 2. |
| 1616 | outerLoop = innerLoop; |
| 1617 | } |
| 1618 | |
| 1619 | // Copy the amx tile accumulation results to a MemRef buffer, add the |
| 1620 | // initial accumulation value, and store back to the C-Matrix |
| 1621 | Location loc = outerLoop.getLoc(); |
| 1622 | |
| 1623 | if (inPlaceAccWrites) { |
| 1624 | // The tiles already include the initial accumulator or were |
| 1625 | // zero-initialized: store each one where its contraction result was |
| 1626 | // written. |
| 1627 | for (auto [i, write] : llvm::enumerate(First&: *inPlaceAccWrites)) { |
| 1628 | auto [base, indices] = *getAccTileAccess(op: write); |
| 1629 | rewriter.setInsertionPoint(write); |
| 1630 | amx::TileStoreOp::create(builder&: rewriter, location: write->getLoc(), base, indices, |
| 1631 | val: newLoop.getResult(i)); |
| 1632 | rewriter.eraseOp(op: write); |
| 1633 | } |
| 1634 | return success(); |
| 1635 | } |
| 1636 | |
| 1637 | // Carried-in accumulators are added to the results instead. |
| 1638 | bool addAccFromMemory = !isAccZeroVectorConstant && carriedAccs.empty(); |
| 1639 | |
| 1640 | // Direct tile stores weren't possible, so we need a temporary buffer to |
| 1641 | // move tile register contents through memory. |
| 1642 | unsigned int M, N; |
| 1643 | Value srcBuffAcc; |
| 1644 | SmallVector<Value> indicesAcc; |
| 1645 | |
| 1646 | if (addAccFromMemory) { |
| 1647 | // We accumulate onto a register tile in memory, hence we let the |
| 1648 | // temporary buffer match its shape. |
| 1649 | llvm::TypeSwitch<Operation *>(accReadOp).Case<TransferReadOp, LoadOp>( |
| 1650 | caseFn: [&](auto readOp) { |
| 1651 | srcBuffAcc = readOp.getOperand(0); |
| 1652 | |
| 1653 | auto indices = readOp.getIndices(); |
| 1654 | indicesAcc.reserve(N: indices.size()); |
| 1655 | |
| 1656 | llvm::transform(indices, std::back_inserter(x&: indicesAcc), |
| 1657 | [&](OpFoldResult ofr) { |
| 1658 | return mlir::getValueOrCreateConstantIndexOp( |
| 1659 | b&: rewriter, loc, ofr); |
| 1660 | }); |
| 1661 | }); |
| 1662 | auto outputShapes = cast<MemRefType>(Val: srcBuffAcc.getType()).getShape(); |
| 1663 | M = outputShapes[outputShapes.size() - 2]; |
| 1664 | N = outputShapes[outputShapes.size() - 1]; |
| 1665 | } else { |
| 1666 | // Otherwise we can just pick a suitable shape to hold all result tiles. |
| 1667 | // For the VNNI-packed case, we pass the result from each tile register to |
| 1668 | // memory individually. For the online packing case, we ensure two tiles |
| 1669 | // are neighboring in the N-dimension to facilitate the shuffling. |
| 1670 | M = isVnni ? ops.size() * 16 : ops.size() / 2 * 16; |
| 1671 | N = isVnni ? 16 : 2 * 16; |
| 1672 | } |
| 1673 | |
| 1674 | SmallVector<Value> dps = newLoop.getResults(); |
| 1675 | auto bufferType = MemRefType::get(shape: {M, N}, elementType: opType); |
| 1676 | auto resultBuffer = memref::AllocaOp::create(builder&: rewriter, location: loc, memrefType: bufferType); |
| 1677 | |
| 1678 | // Store the amx tiled-dot product output into an MxN memref. |
| 1679 | for (unsigned int i = 0, k = 0; i < M; i = i + 16) { |
| 1680 | for (unsigned int j = 0; j < N; j = j + 16) { |
| 1681 | Value indexOp_i = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: i); |
| 1682 | Value indexOp_j = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: j); |
| 1683 | amx::TileStoreOp::create(builder&: rewriter, location: loc, base: resultBuffer, |
| 1684 | indices: ValueRange{indexOp_i, indexOp_j}, val: dps[k]); |
| 1685 | k++; |
| 1686 | } |
| 1687 | } |
| 1688 | |
| 1689 | auto c0 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 1690 | auto c16 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 16); |
| 1691 | auto one = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1); |
| 1692 | auto mBound = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: M); |
| 1693 | |
| 1694 | if (!isVnni || addAccFromMemory) { |
| 1695 | // Create a loop that iterates over the MxN memref, retrieves two rows + |
| 1696 | // shuffles them, adds up the C element values and stores them to the temp |
| 1697 | // buffer. |
| 1698 | scf::ForOp::create( |
| 1699 | builder&: rewriter, location: loc, lowerBound: c0, upperBound: mBound, step: one, initArgs: ValueRange{}, |
| 1700 | odsArg4: [&](OpBuilder &nestedBuilder, Location loc, Value iv, |
| 1701 | ValueRange iterArgs) { |
| 1702 | auto row = vector::LoadOp::create(builder&: rewriter, location: loc, |
| 1703 | resultType: VectorType::get(shape: 16, elementType: opType), |
| 1704 | base: resultBuffer, indices: ValueRange{iv, c0}); |
| 1705 | |
| 1706 | auto row2 = vector::LoadOp::create( |
| 1707 | builder&: rewriter, location: loc, resultType: VectorType::get(shape: 16, elementType: opType), base: resultBuffer, |
| 1708 | indices: ValueRange{iv, c16}); |
| 1709 | |
| 1710 | Value writeVal1 = row; |
| 1711 | Value writeVal2 = row2; |
| 1712 | |
| 1713 | if (!isVnni) { |
| 1714 | writeVal1 = vector::ShuffleOp::create( |
| 1715 | builder&: rewriter, location: loc, result: VectorType::get(shape: 16, elementType: opType), v1: row, v2: row2, |
| 1716 | mask: ArrayRef<int64_t>{0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, |
| 1717 | 21, 22, 23}); |
| 1718 | |
| 1719 | writeVal2 = vector::ShuffleOp::create( |
| 1720 | builder&: rewriter, location: loc, result: VectorType::get(shape: 16, elementType: opType), v1: row, v2: row2, |
| 1721 | mask: ArrayRef<int64_t>{8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, |
| 1722 | 15, 28, 29, 30, 31}); |
| 1723 | } |
| 1724 | |
| 1725 | if (addAccFromMemory) { |
| 1726 | indicesAcc[indicesAcc.size() - 2] = iv; |
| 1727 | indicesAcc[indicesAcc.size() - 1] = c0; |
| 1728 | |
| 1729 | Value valueCRow1 = vector::LoadOp::create( |
| 1730 | builder&: rewriter, location: loc, resultType: VectorType::get(shape: 16, elementType: opType), base: srcBuffAcc, |
| 1731 | indices: indicesAcc); |
| 1732 | indicesAcc[indicesAcc.size() - 1] = c16; |
| 1733 | |
| 1734 | Value valueCRow2 = vector::LoadOp::create( |
| 1735 | builder&: rewriter, location: loc, resultType: VectorType::get(shape: 16, elementType: opType), base: srcBuffAcc, |
| 1736 | indices: indicesAcc); |
| 1737 | |
| 1738 | if (ipType.isBF16() || ipType.isF8E5M2() || ipType.isF8E4M3FN()) { |
| 1739 | writeVal1 = |
| 1740 | arith::AddFOp::create(builder&: rewriter, location: loc, lhs: writeVal1, rhs: valueCRow1); |
| 1741 | |
| 1742 | writeVal2 = |
| 1743 | arith::AddFOp::create(builder&: rewriter, location: loc, lhs: writeVal2, rhs: valueCRow2); |
| 1744 | } |
| 1745 | |
| 1746 | if (ipType.isSignlessInteger(width: 8)) { |
| 1747 | writeVal1 = |
| 1748 | arith::AddIOp::create(builder&: rewriter, location: loc, lhs: writeVal1, rhs: valueCRow1); |
| 1749 | |
| 1750 | writeVal2 = |
| 1751 | arith::AddIOp::create(builder&: rewriter, location: loc, lhs: writeVal2, rhs: valueCRow2); |
| 1752 | } |
| 1753 | } |
| 1754 | |
| 1755 | vector::StoreOp::create(builder&: rewriter, location: loc, valueToStore: writeVal1, base: resultBuffer, |
| 1756 | indices: ValueRange{iv, c0}); |
| 1757 | vector::StoreOp::create(builder&: rewriter, location: loc, valueToStore: writeVal2, base: resultBuffer, |
| 1758 | indices: ValueRange{iv, c16}); |
| 1759 | |
| 1760 | scf::YieldOp::create(builder&: nestedBuilder, location: loc); |
| 1761 | }); |
| 1762 | } |
| 1763 | |
| 1764 | SmallVector<Value> writeResults; |
| 1765 | for (unsigned int i = 0; i < M; i = i + 16) { |
| 1766 | for (unsigned int j = 0; j < N; j = j + 16) { |
| 1767 | Value indexOp_i = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: i); |
| 1768 | Value indexOp_j = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: j); |
| 1769 | |
| 1770 | auto vectorType = VectorType::get(shape: {16, 16}, elementType: opType); |
| 1771 | |
| 1772 | int64_t srcRank = |
| 1773 | (dyn_cast<ShapedType>(Val: resultBuffer.getType())).getRank(); |
| 1774 | Value padding = ub::PoisonOp::create(builder&: rewriter, location: loc, result: opType); |
| 1775 | auto map = AffineMap::getMinorIdentityMap(dims: srcRank, results: vectorType.getRank(), |
| 1776 | context: rewriter.getContext()); |
| 1777 | SmallVector<bool> inBounds(vectorType.getRank(), true); |
| 1778 | |
| 1779 | auto vec1 = vector::TransferReadOp::create( |
| 1780 | builder&: rewriter, location: loc, vectorType, source: resultBuffer, |
| 1781 | indices: ValueRange{indexOp_i, indexOp_j}, padding, permutationMap: map, inBounds); |
| 1782 | writeResults.push_back(Elt: vec1); |
| 1783 | } |
| 1784 | } |
| 1785 | |
| 1786 | for (auto [idx, acc] : carriedAccs) { |
| 1787 | Type resultType = writeResults[idx].getType(); |
| 1788 | if (acc.getType() != resultType) |
| 1789 | acc = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: resultType, source: acc); |
| 1790 | if (isa<FloatType>(Val: opType)) |
| 1791 | writeResults[idx] = |
| 1792 | arith::AddFOp::create(builder&: rewriter, location: loc, lhs: writeResults[idx], rhs: acc); |
| 1793 | else |
| 1794 | writeResults[idx] = |
| 1795 | arith::AddIOp::create(builder&: rewriter, location: loc, lhs: writeResults[idx], rhs: acc); |
| 1796 | } |
| 1797 | |
| 1798 | // Replace use of vector.contract with dot-products. |
| 1799 | for (size_t i = 0; i < ops.size(); i++) { |
| 1800 | vector::ContractionOp contOp = ops[i]; |
| 1801 | Value vecRow = writeResults[i]; |
| 1802 | |
| 1803 | Value resultWriteOp = contractionUsersAfterYield(v: contOp.getResult()); |
| 1804 | if (auto vecType = dyn_cast<VectorType>(Val: resultWriteOp.getType())) |
| 1805 | vecRow = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: vecType, |
| 1806 | source: writeResults[i]); |
| 1807 | |
| 1808 | rewriter.replaceAllUsesWith(from: resultWriteOp, to: vecRow); |
| 1809 | } |
| 1810 | |
| 1811 | return success(); |
| 1812 | } |
| 1813 | }; |
| 1814 | } // namespace |
| 1815 | |
| 1816 | void x86::populateVectorContractToAMXDotProductPatterns( |
| 1817 | RewritePatternSet &patterns) { |
| 1818 | patterns.add<VectorContractToAMXDotProduct>(arg: patterns.getContext()); |
| 1819 | } |
| 1820 | |