| 1 | //===- PrepareVectorContractLoops.cpp -------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 10 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 11 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 12 | #include "mlir/Dialect/Utils/StaticValueUtils.h" |
| 13 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 14 | #include "mlir/Dialect/X86/Transforms.h" |
| 15 | #include "mlir/IR/PatternMatch.h" |
| 16 | #include "llvm/ADT/SmallVectorExtras.h" |
| 17 | #include "llvm/Support/DebugLog.h" |
| 18 | |
| 19 | #define DEBUG_TYPE "x86-prepare-vector-contract-loops" |
| 20 | |
| 21 | using namespace mlir; |
| 22 | |
| 23 | static vector::ContractionOp getCandidate(scf::ForOp loop) { |
| 24 | if (loop.getNumRegionIterArgs() != 1) { |
| 25 | LDBG() << "reject loop at " << loop.getLoc() << ": expected one iter_arg" ; |
| 26 | return {}; |
| 27 | } |
| 28 | |
| 29 | auto yield = cast<scf::YieldOp>(Val: loop.getBody()->getTerminator()); |
| 30 | for (Operation &op : loop.getBody()->without_terminator()) { |
| 31 | auto contract = dyn_cast<vector::ContractionOp>(Val: &op); |
| 32 | if (!contract) |
| 33 | continue; |
| 34 | if (contract.getKind() != vector::CombiningKind::ADD || |
| 35 | contract.getAcc() != loop.getRegionIterArgs().front() || |
| 36 | !contract.getAcc().hasOneUse() || !contract.getResult().hasOneUse() || |
| 37 | yield->getOperand(idx: 0) != contract.getResult()) { |
| 38 | LDBG() << "reject contraction at " << contract.getLoc() |
| 39 | << ": not a single loop-carried accumulation" ; |
| 40 | continue; |
| 41 | } |
| 42 | |
| 43 | auto iterators = contract.getIteratorTypesArray(); |
| 44 | auto maps = contract.getIndexingMapsArray(); |
| 45 | MLIRContext *ctx = contract.getContext(); |
| 46 | auto m = getAffineDimExpr(position: 0, context: ctx); |
| 47 | auto n = getAffineDimExpr(position: 1, context: ctx); |
| 48 | auto k = getAffineDimExpr(position: 2, context: ctx); |
| 49 | auto par = vector::IteratorType::parallel; |
| 50 | auto red = vector::IteratorType::reduction; |
| 51 | if (llvm::equal(LRange&: iterators, RRange: ArrayRef{par, par, red}) && |
| 52 | maps[0] == AffineMap::get(dimCount: 3, symbolCount: 0, results: {m, k}, context: ctx) && |
| 53 | maps[1] == AffineMap::get(dimCount: 3, symbolCount: 0, results: {k, n}, context: ctx) && |
| 54 | maps[2] == AffineMap::get(dimCount: 3, symbolCount: 0, results: {m, n}, context: ctx)) |
| 55 | return contract; |
| 56 | LDBG() << "reject contraction at " << contract.getLoc() |
| 57 | << ": noncanonical matmul layout or iterators" ; |
| 58 | } |
| 59 | return {}; |
| 60 | } |
| 61 | |
| 62 | // Look through reshapes/broadcasts without changing the row-major element |
| 63 | // order. A broadcast is safe to skip if it adds unit dimensions only. |
| 64 | static Value skipReshapes(Value v) { |
| 65 | while (isa_and_present<vector::ShapeCastOp, vector::BroadcastOp>( |
| 66 | Val: v.getDefiningOp())) { |
| 67 | Value source = v.getDefiningOp()->getOperand(idx: 0); |
| 68 | auto sourceType = dyn_cast<VectorType>(Val: source.getType()); |
| 69 | if (!sourceType || sourceType.getNumElements() != |
| 70 | cast<VectorType>(Val: v.getType()).getNumElements()) |
| 71 | break; |
| 72 | v = source; |
| 73 | } |
| 74 | return v; |
| 75 | } |
| 76 | |
| 77 | // Find the source and VNNI lane j of a split: v[i] = src[factor * i + j]. |
| 78 | // A split step of width s keeps every s-th element. Match either form below |
| 79 | // (unit dimensions may also be present): |
| 80 | // |
| 81 | // %even, %odd = vector.deinterleave %src // s = 2 |
| 82 | // : vector<4x8xbf16> -> vector<4x4xbf16> |
| 83 | // |
| 84 | // or: |
| 85 | // |
| 86 | // %packed = vector.shape_cast %src |
| 87 | // : vector<4x8xbf16> to vector<4x4x2xbf16> |
| 88 | // %split = vector.transpose %packed, [2, 0, 1] |
| 89 | // : vector<4x4x2xbf16> to vector<2x4x4xbf16> |
| 90 | // %even = vector.extract %split[0] |
| 91 | // : vector<4x4xbf16> from vector<2x4x4xbf16> |
| 92 | // %odd = vector.extract %split[1] |
| 93 | // : vector<4x4xbf16> from vector<2x4x4xbf16> |
| 94 | // |
| 95 | // If s < factor, the split's input must itself come from splits of src |
| 96 | // totaling factor / s. For example, for factor = 4, a 4-way transpose/extract |
| 97 | // split of src matches, as do two levels of binary splits: |
| 98 | // |
| 99 | // %even, %odd = vector.deinterleave %src // lanes {0, 2} and {1, 3} |
| 100 | // %l1, %l3 = vector.deinterleave %odd // %l3[i] = src[4 * i + 3] |
| 101 | // |
| 102 | // Splits closer to src give the low lane bits; result j of the split that |
| 103 | // produces v gives the high bits. |
| 104 | static std::optional<std::pair<Value, int64_t>> matchSplit(Value v, |
| 105 | int64_t factor) { |
| 106 | v = skipReshapes(v); |
| 107 | if (factor == 1) |
| 108 | return std::pair(v, int64_t(0)); |
| 109 | |
| 110 | auto matchSource = |
| 111 | [&](Value source, int64_t ways, |
| 112 | int64_t j) -> std::optional<std::pair<Value, int64_t>> { |
| 113 | if (factor % ways != 0) |
| 114 | return std::nullopt; |
| 115 | auto lane = matchSplit(v: source, factor: factor / ways); |
| 116 | if (!lane) |
| 117 | return std::nullopt; |
| 118 | lane->second += factor / ways * j; |
| 119 | return lane; |
| 120 | }; |
| 121 | if (auto deinterleave = v.getDefiningOp<vector::DeinterleaveOp>()) |
| 122 | return matchSource(deinterleave.getSource(), 2, |
| 123 | cast<OpResult>(Val&: v).getResultNumber()); |
| 124 | |
| 125 | auto = v.getDefiningOp<vector::ExtractOp>(); |
| 126 | auto transpose = |
| 127 | extract ? extract.getSource().getDefiningOp<vector::TransposeOp>() |
| 128 | : vector::TransposeOp(); |
| 129 | auto type = dyn_cast<VectorType>(Val: v.getType()); |
| 130 | if (!transpose || !type || extract.hasDynamicPosition() || |
| 131 | transpose.getSourceVectorType().isScalable()) |
| 132 | return std::nullopt; |
| 133 | |
| 134 | ArrayRef<int64_t> shape = transpose.getSourceVectorType().getShape(); |
| 135 | auto isNonUnit = [&](int64_t dim) { return shape[dim] != 1; }; |
| 136 | SmallVector<int64_t> order = |
| 137 | llvm::filter_to_vector(C: llvm::seq<int64_t>(Begin: 0, End: shape.size()), Pred&: isNonUnit); |
| 138 | if (order.empty()) |
| 139 | return std::nullopt; |
| 140 | int64_t ways = shape[order.back()]; |
| 141 | |
| 142 | std::rotate(first: order.begin(), middle: std::prev(x: order.end()), last: order.end()); |
| 143 | ArrayRef<int64_t> perm = transpose.getPermutation(); |
| 144 | if (llvm::filter_to_vector(C&: perm, Pred&: isNonUnit) != order) |
| 145 | return std::nullopt; |
| 146 | |
| 147 | // The split dimension is now first among the non-unit dimensions. Keeping |
| 148 | // exactly 1 / ways of the elements means the extract indexes that dimension; |
| 149 | // any other dimensions it indexes must be unit dimensions. |
| 150 | if (ways * type.getNumElements() != |
| 151 | transpose.getSourceVectorType().getNumElements()) |
| 152 | return std::nullopt; |
| 153 | |
| 154 | int64_t j = |
| 155 | extract |
| 156 | .getStaticPosition()[llvm::find(Range&: perm, Val: order.front()) - perm.begin()]; |
| 157 | if (j < 0 || j >= ways) |
| 158 | return std::nullopt; |
| 159 | |
| 160 | return matchSource(transpose.getVector(), ways, j); |
| 161 | } |
| 162 | |
| 163 | // Find the inputs in VNNI-lane order for a join. A join step of width s |
| 164 | // interleaves s inputs. Match either form below: |
| 165 | // |
| 166 | // %joined = vector.interleave %even, %odd // s = 2 |
| 167 | // : vector<4x1xbf16> -> vector<4x2xbf16> |
| 168 | // |
| 169 | // or s slice inserts in any order (shown for s = 2): |
| 170 | // |
| 171 | // %poison = ub.poison : vector<4x2xbf16> |
| 172 | // %partial = vector.insert_strided_slice %odd, %poison |
| 173 | // offsets = [0, 1], strides = [1, 1] |
| 174 | // : vector<4x1xbf16> into vector<4x2xbf16> |
| 175 | // %joined = vector.insert_strided_slice %even, %partial |
| 176 | // offsets = [0, 0], strides = [1, 1] |
| 177 | // : vector<4x1xbf16> into vector<4x2xbf16> |
| 178 | // |
| 179 | // If s < factor, each input must itself be a join of width factor / s |
| 180 | // (possibly built from smaller joins). |
| 181 | static std::optional<SmallVector<Value, 4>> matchJoin(Value v, int64_t factor) { |
| 182 | v = skipReshapes(v); |
| 183 | if (factor == 1) |
| 184 | return SmallVector<Value, 4>{v}; |
| 185 | |
| 186 | SmallVector<Value, 4> parts; |
| 187 | if (auto interleave = v.getDefiningOp<vector::InterleaveOp>()) { |
| 188 | parts = {interleave.getLhs(), interleave.getRhs()}; |
| 189 | } else if (auto insert = v.getDefiningOp<vector::InsertStridedSliceOp>()) { |
| 190 | int64_t ways = insert.getDestVectorType().getShape().back(); |
| 191 | if (ways < 2 || factor % ways != 0) |
| 192 | return std::nullopt; |
| 193 | parts.resize(N: ways); |
| 194 | // All slices must be complete and disjoint so the original destination |
| 195 | // contributes no elements to the result. |
| 196 | for (int64_t i = 0; i < ways; ++i) { |
| 197 | auto insert = v.getDefiningOp<vector::InsertStridedSliceOp>(); |
| 198 | if (!insert) |
| 199 | return std::nullopt; |
| 200 | ArrayRef<int64_t> shape = insert.getDestVectorType().getShape(); |
| 201 | SmallVector<int64_t> slice(shape.drop_back()); |
| 202 | slice.push_back(Elt: 1); |
| 203 | int64_t j = |
| 204 | extractFromIntegerArrayAttr<int64_t>(attr: insert.getOffsets()).back(); |
| 205 | if (insert.getSourceVectorType().getShape() != ArrayRef(slice) || |
| 206 | parts[j]) |
| 207 | return std::nullopt; |
| 208 | parts[j] = insert.getValueToStore(); |
| 209 | v = insert.getDest(); |
| 210 | } |
| 211 | } else { |
| 212 | return std::nullopt; |
| 213 | } |
| 214 | |
| 215 | int64_t ways = parts.size(); |
| 216 | if (factor % ways != 0) |
| 217 | return std::nullopt; |
| 218 | SmallVector<Value, 4> lanes(factor); |
| 219 | for (auto [j, part] : llvm::enumerate(First&: parts)) { |
| 220 | auto inner = matchJoin(v: part, factor: factor / ways); |
| 221 | if (!inner) |
| 222 | return std::nullopt; |
| 223 | for (auto [i, lane] : llvm::enumerate(First&: *inner)) |
| 224 | lanes[ways * i + j] = lane; |
| 225 | } |
| 226 | return lanes; |
| 227 | } |
| 228 | |
| 229 | // Find the packed RHS so the contraction can use it without unpacking. |
| 230 | // For a K x N RHS and VNNI factor f, require for each VNNI lane j: |
| 231 | // rhs[f * r + j][c] = src[r][c][j], |
| 232 | // where 0 <= j < f and src is viewed as K/f x N x f. |
| 233 | // Match transpose/shape_cast unpacking (shown for f = 2): |
| 234 | // |
| 235 | // %packed = vector.shape_cast %src |
| 236 | // : vector<4x6xbf16> to vector<4x3x2xbf16> |
| 237 | // %transposed = vector.transpose %packed, [0, 2, 1] |
| 238 | // : vector<4x3x2xbf16> to vector<4x2x3xbf16> |
| 239 | // %rhs = vector.shape_cast %transposed |
| 240 | // : vector<4x2x3xbf16> to vector<8x3xbf16> |
| 241 | // |
| 242 | // or split/transpose/join/transpose unpacking (shown for f = 2): |
| 243 | // |
| 244 | // %even, %odd = vector.deinterleave %src |
| 245 | // : vector<4x6xbf16> -> vector<4x3xbf16> |
| 246 | // %even_t = vector.transpose %even, [1, 0] |
| 247 | // : vector<4x3xbf16> to vector<3x4xbf16> |
| 248 | // %odd_t = vector.transpose %odd, [1, 0] |
| 249 | // : vector<4x3xbf16> to vector<3x4xbf16> |
| 250 | // %joined = vector.interleave %even_t, %odd_t |
| 251 | // : vector<3x4xbf16> -> vector<3x8xbf16> |
| 252 | // %rhs = vector.transpose %joined, [1, 0] |
| 253 | // : vector<3x8xbf16> to vector<8x3xbf16> |
| 254 | // |
| 255 | // For f = 4, splits and joins may each be one 4-way transpose/extract or |
| 256 | // slice-insert step, or two levels of binary steps. With binary steps, the |
| 257 | // second-level splits produce VNNI lanes 0/2 and 1/3: |
| 258 | // |
| 259 | // %even, %odd = vector.deinterleave %src // lanes {0, 2} and {1, 3} |
| 260 | // %l0, %l2 = vector.deinterleave %even |
| 261 | // %l1, %l3 = vector.deinterleave %odd |
| 262 | // // Transpose each lane %lj to %tj. |
| 263 | // %even_t = vector.interleave %t0, %t2 |
| 264 | // %odd_t = vector.interleave %t1, %t3 |
| 265 | // %joined = vector.interleave %even_t, %odd_t |
| 266 | // |
| 267 | // Any binary split or join may instead use the transpose/extract or |
| 268 | // slice-insert form. |
| 269 | static Value matchUnpackedRhs(Value rhs, int64_t factor) { |
| 270 | auto type = cast<VectorType>(Val: rhs.getType()); |
| 271 | int64_t k = type.getDimSize(idx: 0) / factor, n = type.getDimSize(idx: 1); |
| 272 | auto transpose = skipReshapes(v: rhs).getDefiningOp<vector::TransposeOp>(); |
| 273 | if (!transpose) |
| 274 | return {}; |
| 275 | |
| 276 | ArrayRef<int64_t> shape = transpose.getSourceVectorType().getShape(); |
| 277 | ArrayRef<int64_t> perm = transpose.getPermutation(); |
| 278 | if (shape.size() >= 3) { |
| 279 | size_t leading = shape.size() - 3; |
| 280 | auto expected = llvm::to_vector(Range: llvm::seq<int64_t>(Begin: 0, End: shape.size())); |
| 281 | std::swap(a&: expected[leading + 1], b&: expected[leading + 2]); |
| 282 | bool isUnpacking = |
| 283 | llvm::all_of(Range: shape.take_front(N: leading), |
| 284 | P: [](int64_t dim) { return dim == 1; }) && |
| 285 | shape.drop_front(N: leading) == ArrayRef<int64_t>{k, n, factor} && |
| 286 | perm == ArrayRef(expected); |
| 287 | return isUnpacking ? skipReshapes(v: transpose.getVector()) : Value(); |
| 288 | } |
| 289 | |
| 290 | const int64_t swap[] = {1, 0}; |
| 291 | if (perm != ArrayRef(swap) || shape != ArrayRef<int64_t>{n, factor * k}) |
| 292 | return {}; |
| 293 | |
| 294 | auto lanes = matchJoin(v: transpose.getVector(), factor); |
| 295 | if (!lanes) |
| 296 | return {}; |
| 297 | |
| 298 | Value src; |
| 299 | for (auto [j, lane] : llvm::enumerate(First&: *lanes)) { |
| 300 | auto laneTranspose = |
| 301 | skipReshapes(v: lane).getDefiningOp<vector::TransposeOp>(); |
| 302 | if (!laneTranspose || laneTranspose.getPermutation() != ArrayRef(swap) || |
| 303 | laneTranspose.getSourceVectorType().getShape() != |
| 304 | ArrayRef<int64_t>{k, n}) |
| 305 | return {}; |
| 306 | |
| 307 | auto source = matchSplit(v: laneTranspose.getVector(), factor); |
| 308 | if (!source || source->second != int64_t(j) || |
| 309 | (src && source->first != src)) |
| 310 | return {}; |
| 311 | |
| 312 | src = source->first; |
| 313 | } |
| 314 | return src; |
| 315 | } |
| 316 | |
| 317 | static bool isSimpleReadOutsideLoop(vector::TransferReadOp read, |
| 318 | scf::ForOp loop) { |
| 319 | return read && read->getParentOp() == loop && |
| 320 | isa<MemRefType>(Val: read.getBase().getType()) && |
| 321 | loop.isDefinedOutsideOfLoop(value: read.getBase()) && |
| 322 | read.getPermutationMap().isMinorIdentity() && !read.getMask() && |
| 323 | !read.hasOutOfBoundsDim(); |
| 324 | } |
| 325 | |
| 326 | namespace { |
| 327 | struct ExpandedRead { |
| 328 | vector::TransferReadOp read; |
| 329 | MemRefType type; |
| 330 | SmallVector<ReassociationIndices> reassociation; |
| 331 | int64_t lastIndex; |
| 332 | // Static view of a dynamic last dimension, or null. |
| 333 | MemRefType view; |
| 334 | }; |
| 335 | } // namespace |
| 336 | |
| 337 | static std::optional<ExpandedRead> |
| 338 | getExpandableRead(vector::TransferReadOp read, unsigned factor, |
| 339 | scf::ForOp loop) { |
| 340 | if (!isSimpleReadOutsideLoop(read, loop)) |
| 341 | return std::nullopt; |
| 342 | auto type = cast<MemRefType>(Val: read.getBase().getType()); |
| 343 | if (type.getElementType() != read.getVectorType().getElementType() || |
| 344 | type.getRank() < 2) |
| 345 | return std::nullopt; |
| 346 | auto lastIndex = getConstantIntValue(ofr: read.getIndices().back()); |
| 347 | if (!lastIndex || *lastIndex < 0 || *lastIndex % factor != 0) |
| 348 | return std::nullopt; |
| 349 | |
| 350 | MemRefType view; |
| 351 | if (type.isDynamicDim(idx: type.getRank() - 1)) { |
| 352 | SmallVector<int64_t> strides; |
| 353 | int64_t offset; |
| 354 | if (read.getVectorType().isScalable() || |
| 355 | failed(Result: type.getStridesAndOffset(strides, offset))) |
| 356 | return std::nullopt; |
| 357 | // The full dimension has a dynamic size, but this in-bounds read ends at |
| 358 | // a known index. Create a view whose last dimension ends there and give |
| 359 | // the divisibility check below a static size to work with. |
| 360 | SmallVector<int64_t> shape(type.getShape()); |
| 361 | shape.back() = *lastIndex + read.getVectorType().getShape().back(); |
| 362 | view = type = MemRefType::get( |
| 363 | shape, elementType: type.getElementType(), |
| 364 | layout: StridedLayoutAttr::get(context: type.getContext(), offset, strides), |
| 365 | memorySpace: type.getMemorySpace()); |
| 366 | } |
| 367 | if (type.getShape().back() % factor != 0) |
| 368 | return std::nullopt; |
| 369 | |
| 370 | SmallVector<int64_t> shape(type.getShape()); |
| 371 | shape.back() /= factor; |
| 372 | shape.push_back(Elt: factor); |
| 373 | SmallVector<ReassociationIndices> reassociation; |
| 374 | for (int64_t i = 0; i < type.getRank(); ++i) |
| 375 | reassociation.push_back(Elt: i == type.getRank() - 1 |
| 376 | ? ReassociationIndices{i, i + 1} |
| 377 | : ReassociationIndices{i}); |
| 378 | FailureOr<MemRefType> expanded = |
| 379 | memref::ExpandShapeOp::computeExpandedType(srcType: type, resultShape: shape, reassociation); |
| 380 | if (failed(Result: expanded)) |
| 381 | return std::nullopt; |
| 382 | return ExpandedRead{.read: read, .type: *expanded, .reassociation: reassociation, .lastIndex: *lastIndex / factor, |
| 383 | .view: view}; |
| 384 | } |
| 385 | |
| 386 | static Value buildExpandedRead(PatternRewriter &rewriter, Location loc, |
| 387 | ExpandedRead &info, VectorType resultType) { |
| 388 | OpBuilder::InsertionGuard guard(rewriter); |
| 389 | rewriter.setInsertionPoint(info.read->getParentOp()); |
| 390 | Value base = info.read.getBase(); |
| 391 | if (info.view) { |
| 392 | // Unlike memref.subview, memref.reinterpret_cast is valid even when |
| 393 | // the loop runs zero times (which makes the view out of bounds). |
| 394 | auto metadata = |
| 395 | memref::ExtractStridedMetadataOp::create(builder&: rewriter, location: loc, source: base); |
| 396 | SmallVector<OpFoldResult> sizes = metadata.getConstifiedMixedSizes(); |
| 397 | sizes.back() = rewriter.getIndexAttr(value: info.view.getShape().back()); |
| 398 | base = memref::ReinterpretCastOp::create( |
| 399 | builder&: rewriter, location: loc, resultType: info.view, source: base, offset: metadata.getConstifiedMixedOffset(), |
| 400 | sizes, strides: metadata.getConstifiedMixedStrides()); |
| 401 | } |
| 402 | Value expanded = memref::ExpandShapeOp::create(builder&: rewriter, location: loc, resultType: info.type, src: base, |
| 403 | reassociation: info.reassociation); |
| 404 | Value lastIndex = |
| 405 | arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: info.lastIndex); |
| 406 | Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 407 | rewriter.setInsertionPoint(info.read); |
| 408 | SmallVector<Value> indices(info.read.getIndices()); |
| 409 | indices.back() = lastIndex; |
| 410 | indices.push_back(Elt: zero); |
| 411 | auto packed = vector::TransferReadOp::create( |
| 412 | builder&: rewriter, location: loc, vectorType: resultType, source: expanded, indices, padding: info.read.getPadding()); |
| 413 | packed.setInBoundsAttr( |
| 414 | rewriter.getBoolArrayAttr(values: SmallVector<bool>(resultType.getRank(), true))); |
| 415 | return packed; |
| 416 | } |
| 417 | |
| 418 | namespace { |
| 419 | // Fold RHS unpacking into a VNNI contraction: |
| 420 | // - Match a packed RHS read unpacked for a flat matmul. |
| 421 | // - Split the LHS memref's K dimension and reread it in VNNI layout. |
| 422 | // A dynamic K dimension is first narrowed to the read's static extent. |
| 423 | // - Contract the packed reads directly over K and the VNNI dimension. |
| 424 | // For bf16, 32x32 LHS and RHS become 32x16x2 LHS and 16x32x2 RHS. |
| 425 | struct FoldVNNIUnpacking : OpRewritePattern<vector::ContractionOp> { |
| 426 | using OpRewritePattern::OpRewritePattern; |
| 427 | |
| 428 | LogicalResult matchAndRewrite(vector::ContractionOp contract, |
| 429 | PatternRewriter &rewriter) const override { |
| 430 | auto loop = dyn_cast<scf::ForOp>(Val: contract->getParentOp()); |
| 431 | if (!loop) |
| 432 | return rewriter.notifyMatchFailure(arg&: contract, |
| 433 | msg: "not directly in a for loop" ); |
| 434 | if (getCandidate(loop) != contract) |
| 435 | return rewriter.notifyMatchFailure( |
| 436 | arg&: contract, msg: "not the loop's canonical contraction" ); |
| 437 | |
| 438 | auto lhsType = contract.getLhsType(); |
| 439 | auto rhsType = contract.getRhsType(); |
| 440 | Type element = lhsType.getElementType(); |
| 441 | assert(lhsType.getRank() == 2 && rhsType.getRank() == 2 && |
| 442 | lhsType.getDimSize(1) == rhsType.getDimSize(0) && |
| 443 | element == rhsType.getElementType() && |
| 444 | "expected canonical matmul contraction" ); |
| 445 | if (lhsType.isScalable() || rhsType.isScalable()) |
| 446 | return rewriter.notifyMatchFailure(arg&: contract, msg: "scalable operand vectors" ); |
| 447 | if (!(element.isBF16() || element.isInteger(width: 8))) |
| 448 | return rewriter.notifyMatchFailure(arg&: contract, |
| 449 | msg: "expected bf16 or i8 operands" ); |
| 450 | unsigned factor = 32 / element.getIntOrFloatBitWidth(); |
| 451 | if (lhsType.getDimSize(idx: 1) % factor != 0) |
| 452 | return rewriter.notifyMatchFailure( |
| 453 | arg&: contract, msg: "K dimension is not divisible by VNNI factor" ); |
| 454 | |
| 455 | Value packedSource = matchUnpackedRhs(rhs: contract.getRhs(), factor); |
| 456 | if (!packedSource) |
| 457 | return rewriter.notifyMatchFailure(arg&: contract, |
| 458 | msg: "RHS does not unpack a packed vector" ); |
| 459 | VectorType rhsPackedType = |
| 460 | VectorType::get(shape: {lhsType.getDimSize(idx: 1) / factor, rhsType.getDimSize(idx: 1), |
| 461 | int64_t(factor)}, |
| 462 | elementType: element); |
| 463 | auto lhsRead = contract.getLhs().getDefiningOp<vector::TransferReadOp>(); |
| 464 | auto lhs = getExpandableRead(read: lhsRead, factor, loop); |
| 465 | auto rhsRead = packedSource.getDefiningOp<vector::TransferReadOp>(); |
| 466 | if (!lhs) |
| 467 | return rewriter.notifyMatchFailure( |
| 468 | arg&: contract, msg: "LHS read cannot be expanded into VNNI layout" ); |
| 469 | if (!isSimpleReadOutsideLoop(read: rhsRead, loop)) |
| 470 | return rewriter.notifyMatchFailure(arg&: contract, |
| 471 | msg: "RHS is not a simple transfer read" ); |
| 472 | std::optional<ExpandedRead> splitRhs; |
| 473 | if (rhsRead.getVectorType() != rhsPackedType) { |
| 474 | auto flatType = rhsRead.getVectorType(); |
| 475 | if (flatType.getRank() != 2 || |
| 476 | flatType.getDimSize(idx: 0) != rhsPackedType.getDimSize(idx: 0) || |
| 477 | flatType.getDimSize(idx: 1) % factor != 0 || |
| 478 | flatType.getDimSize(idx: 1) / factor != rhsPackedType.getDimSize(idx: 1)) |
| 479 | return rewriter.notifyMatchFailure( |
| 480 | arg&: contract, msg: "flat RHS shape is not VNNI compatible" ); |
| 481 | if (!(splitRhs = getExpandableRead(read: rhsRead, factor, loop))) |
| 482 | return rewriter.notifyMatchFailure( |
| 483 | arg&: contract, msg: "RHS read cannot be expanded into VNNI layout" ); |
| 484 | } else { |
| 485 | if (getConstantIntValue(ofr: rhsRead.getIndices().back()) != 0) |
| 486 | return rewriter.notifyMatchFailure( |
| 487 | arg&: contract, msg: "packed RHS has nonzero inner offset" ); |
| 488 | } |
| 489 | |
| 490 | Location loc = contract.getLoc(); |
| 491 | VectorType lhsPackedType = |
| 492 | VectorType::get(shape: {lhsType.getDimSize(idx: 0), lhsType.getDimSize(idx: 1) / factor, |
| 493 | int64_t(factor)}, |
| 494 | elementType: element); |
| 495 | Value packedLhs = buildExpandedRead(rewriter, loc, info&: *lhs, resultType: lhsPackedType); |
| 496 | Value packedRhs = |
| 497 | splitRhs ? buildExpandedRead(rewriter, loc, info&: *splitRhs, resultType: rhsPackedType) |
| 498 | : rhsRead.getResult(); |
| 499 | rewriter.setInsertionPoint(contract); |
| 500 | auto ctx = rewriter.getContext(); |
| 501 | auto m = rewriter.getAffineDimExpr(position: 0); |
| 502 | auto n = rewriter.getAffineDimExpr(position: 1); |
| 503 | auto k = rewriter.getAffineDimExpr(position: 2); |
| 504 | auto v = rewriter.getAffineDimExpr(position: 3); |
| 505 | auto vnniMaps = |
| 506 | rewriter.getAffineMapArrayAttr(values: {AffineMap::get(dimCount: 4, symbolCount: 0, results: {m, k, v}, context: ctx), |
| 507 | AffineMap::get(dimCount: 4, symbolCount: 0, results: {k, n, v}, context: ctx), |
| 508 | AffineMap::get(dimCount: 4, symbolCount: 0, results: {m, n}, context: ctx)}); |
| 509 | auto iterators = rewriter.getArrayAttr( |
| 510 | value: {vector::IteratorTypeAttr::get(context: ctx, value: vector::IteratorType::parallel), |
| 511 | vector::IteratorTypeAttr::get(context: ctx, value: vector::IteratorType::parallel), |
| 512 | vector::IteratorTypeAttr::get(context: ctx, value: vector::IteratorType::reduction), |
| 513 | vector::IteratorTypeAttr::get(context: ctx, value: vector::IteratorType::reduction)}); |
| 514 | rewriter.replaceOpWithNewOp<vector::ContractionOp>( |
| 515 | op: contract, args&: packedLhs, args&: packedRhs, args: contract.getAcc(), args&: vnniMaps, args&: iterators, |
| 516 | args: contract.getKind(), args: contract.getFastmath()); |
| 517 | LDBG() << "Replaced contract with VNNI-compatible contraction" ; |
| 518 | return success(); |
| 519 | } |
| 520 | }; |
| 521 | } // namespace |
| 522 | |
| 523 | void x86::populatePrepareVectorContractLoopsPatterns( |
| 524 | RewritePatternSet &patterns) { |
| 525 | patterns.add<FoldVNNIUnpacking>(arg: patterns.getContext()); |
| 526 | } |
| 527 | |