1//===- X86Utils.cpp - MLIR Utilities for X86Ops -------------------------===//
2//
3// Part of the MLIR Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/X86/Utils/X86Utils.h"
10
11#include "mlir/Dialect/Linalg/IR/Linalg.h"
12#include "mlir/Dialect/Linalg/IR/LinalgInterfaces.h"
13#include "mlir/Dialect/SCF/IR/SCF.h"
14#include "mlir/Dialect/Vector/IR/VectorOps.h"
15#include "mlir/IR/BuiltinTypes.h"
16#include "mlir/IR/TypeUtilities.h"
17#include "mlir/IR/Types.h"
18
19#include "llvm/ADT/TypeSwitch.h"
20#include "llvm/Support/Casting.h"
21
22#include "llvm/ADT/ArrayRef.h"
23#include <cassert>
24
25namespace mlir {
26namespace x86 {
27
28static FailureOr<SmallVector<mlir::utils::IteratorType>>
29inferIteratorsFromOutMap(AffineMap map) {
30 if (!map.isProjectedPermutation())
31 return failure();
32 SmallVector<mlir::utils::IteratorType> iterators(
33 map.getNumDims(), mlir::utils::IteratorType::reduction);
34 for (auto expr : map.getResults())
35 if (auto dim = dyn_cast<AffineDimExpr>(Val&: expr))
36 iterators[dim.getPosition()] = mlir::utils::IteratorType::parallel;
37 return iterators;
38}
39
40// Returns true if the operation is in VNNI layout.
41// Optionally, the check can be constrained to a specific VNNI blocking factor.
42bool isInVnniLayout(Operation *op, ArrayRef<AffineMap> indexingMaps,
43 std::optional<unsigned> blockingFactor) {
44 // Narrow down type operations - VNNI only applies to contractions.
45 FailureOr<linalg::ContractionDimensions> dims =
46 linalg::inferContractionDims(indexingMaps);
47 if (failed(Result: dims))
48 return false;
49
50 auto matA = op->getOperand(idx: 0);
51 auto matB = op->getOperand(idx: 1);
52 auto typeA = dyn_cast<ShapedType>(Val: matA.getType());
53 auto typeB = dyn_cast<ShapedType>(Val: matB.getType());
54 unsigned rankA = typeA.getRank();
55 unsigned rankB = typeB.getRank();
56 // VNNI format requires at least 1 parallel and 2 reduction dimensions.
57 if (rankA < 3 || rankB < 3)
58 return false;
59
60 // At least two reduction dimensions are expected:
61 // one for the VNNI factor and one for the K dimension
62 if (dims->k.size() < 2)
63 return false;
64
65 // Validate affine maps - VNNI computation should be defined by the two
66 // innermost reduction iterators.
67 // The input matrix dimensions layout must match the following:
68 // - matrix A - [...][K/vnniFactor][vnniFactor]
69 // - matrix B - [...][K/vnniFactor][N][vnniFactor]
70 auto maybeIters = inferIteratorsFromOutMap(map: indexingMaps[2] /* outs */);
71 if (failed(Result: maybeIters))
72 return false;
73 SmallVector<mlir::utils::IteratorType> iteratorTypes = *maybeIters;
74 AffineMap mapA = indexingMaps[0];
75 AffineMap mapB = indexingMaps[1];
76
77 auto vnniDimA = dyn_cast<AffineDimExpr>(Val: mapA.getResult(idx: rankA - 1));
78 auto vnniDimB = dyn_cast<AffineDimExpr>(Val: mapB.getResult(idx: rankB - 1));
79 if (!vnniDimA || !vnniDimB || vnniDimA != vnniDimB ||
80 iteratorTypes[vnniDimA.getPosition()] !=
81 mlir::utils::IteratorType::reduction)
82 return false;
83 auto redDimA = dyn_cast<AffineDimExpr>(Val: mapA.getResult(idx: rankA - 2));
84 auto redDimB = dyn_cast<AffineDimExpr>(Val: mapB.getResult(idx: rankB - 3));
85 if (!redDimA || !redDimB || redDimA != redDimB ||
86 iteratorTypes[redDimA.getPosition()] !=
87 mlir::utils::IteratorType::reduction)
88 return false;
89 auto parallelDimB = dyn_cast<AffineDimExpr>(Val: mapB.getResult(idx: rankB - 2));
90 if (!parallelDimB || iteratorTypes[parallelDimB.getPosition()] !=
91 mlir::utils::IteratorType::parallel)
92 return false;
93
94 // VNNI factor must be:
95 // - the innermost inputs' dimension
96 // - statically known
97 // - multiple of 2 or equal to the specified factor
98 auto vnniDimSize = typeB.getShape().back();
99 if (vnniDimSize == ShapedType::kDynamic || vnniDimSize == 0 ||
100 vnniDimSize % 2 != 0)
101 return false;
102 if (typeA.getShape().back() != vnniDimSize)
103 return false;
104 if (blockingFactor && vnniDimSize != *blockingFactor)
105 return false;
106
107 // The split reduction dimension size should also match.
108 if (typeA.getShape().end()[-2] != typeB.getShape().end()[-3])
109 return false;
110
111 return true;
112}
113
114struct ShuffleMasks {
115 llvm::ArrayRef<int64_t> maskLo;
116 llvm::ArrayRef<int64_t> maskHi;
117};
118
119inline ShuffleMasks getShuffleMasks(int64_t nonUnitDimAcc, bool isInt8Avx2) {
120 // We only support these two layouts for now.
121 assert((nonUnitDimAcc == 8 || nonUnitDimAcc == 16) &&
122 "Unsupported nonUnitDimAcc value");
123
124 // Do interleaving between two <8xf32> targeting AVX2.
125 static constexpr int64_t maskLo8[] = {0, 8, 1, 9, 2, 10, 3, 11};
126 static constexpr int64_t maskHi8[] = {4, 12, 5, 13, 6, 14, 7, 15};
127
128 // Do interleaving between two <8xi32> targeting AVX2.
129 static constexpr int64_t maskLo8_avx2_int8[] = {0, 1, 2, 3, 8, 9, 10, 11};
130 static constexpr int64_t maskHi8_avx2_int8[] = {4, 5, 6, 7, 12, 13, 14, 15};
131
132 // Shuffle two <16xf32/i32> as below targeting AVX512.
133 static constexpr int64_t maskLo16[] = {0, 1, 2, 3, 16, 17, 18, 19,
134 4, 5, 6, 7, 20, 21, 22, 23};
135 static constexpr int64_t maskHi16[] = {8, 9, 10, 11, 24, 25, 26, 27,
136 12, 13, 14, 15, 28, 29, 30, 31};
137
138 if (nonUnitDimAcc == 16)
139 return {.maskLo: maskLo16, .maskHi: maskHi16};
140
141 if (isInt8Avx2)
142 return {.maskLo: maskLo8_avx2_int8, .maskHi: maskHi8_avx2_int8};
143
144 return {.maskLo: maskLo8, .maskHi: maskHi8};
145}
146
147// Recursively follows single-use values through scf.yield operations
148// and returns the first non-yield user result in the contraction chain.
149Value contractionUsersAfterYield(Value v) {
150 if (!v || v.getNumUses() != 1)
151 return nullptr;
152
153 OpOperand &use = *v.use_begin();
154 Operation *user = use.getOwner();
155
156 if (!isa<scf::YieldOp>(Val: user))
157 return v;
158
159 auto yield = cast<scf::YieldOp>(Val: user);
160 Operation *parent = yield->getParentOp();
161 unsigned idx = use.getOperandNumber();
162
163 return contractionUsersAfterYield(v: parent->getResult(idx));
164}
165
166// This function walks backward from a value to locate its originating
167// vector read-like operation (`vector.transfer_read` or `vector.load`).
168// It follows simple forwarding through unary ops and across `scf.for`
169// loop iter-arguments, while stopping if layout-transforming ops such
170// as `shape_cast` or `shuffle` are encountered. The traversal returns
171// the read-like defining operation or `nullptr` if no valid source
172// is found.
173Operation *traceToVectorReadLikeParentOperation(Value v) {
174 while (true) {
175 // Case 1: Value defined by an operation
176 if (Operation *defOp = v.getDefiningOp()) {
177 if (isa<vector::TransferReadOp, vector::LoadOp, arith::ConstantOp>(
178 Val: defOp)) {
179
180 if (auto constOp = dyn_cast<arith::ConstantOp>(Val: defOp)) {
181 if (auto denseAttr =
182 dyn_cast<DenseElementsAttr>(Val: constOp.getValue())) {
183 if (!denseAttr.isSplat())
184 return nullptr;
185
186 Attribute splat = denseAttr.getSplatValue<Attribute>();
187
188 if (auto floatAttr = dyn_cast<FloatAttr>(Val&: splat))
189 return floatAttr.getValue().isZero() ? defOp : nullptr;
190
191 if (auto intAttr = dyn_cast<IntegerAttr>(Val&: splat))
192 return intAttr.getValue().isZero() ? defOp : nullptr;
193 }
194
195 return nullptr;
196 }
197 return defOp;
198 }
199
200 return nullptr;
201 }
202
203 // Case 2: BlockArgument (scf.for iter_arg)
204 if (auto barg = dyn_cast<BlockArgument>(Val&: v)) {
205 auto *parentOp = barg.getOwner()->getParentOp();
206
207 if (auto forOp = dyn_cast<scf::ForOp>(Val: parentOp)) {
208 unsigned argNum = barg.getArgNumber();
209
210 // arg0 = induction variable (not an iter_arg)
211 if (argNum == 0)
212 return nullptr;
213
214 unsigned iterIdx = argNum - 1;
215 v = forOp.getInitArgs()[iterIdx];
216 continue;
217 }
218
219 return nullptr;
220 }
221
222 return nullptr;
223 }
224}
225
226// This function recursively traces a value through its uses to find
227// a downstream vector write-like operation (`vector.transfer_write`
228// or `vector.store`). It transparently follows values across `scf.for`
229// and `scf.yield` boundaries while stopping if layout-altering ops
230// like `shuffle` are encountered. The traversal returns
231// the matching write-like user. Returns `nullptr` if none is found or
232// the value has multiple users.
233Operation *traceToVectorWriteLikeUserOperation(Value v) {
234
235 if (v.getNumUses() > 1)
236 return nullptr;
237
238 for (OpOperand &use : v.getUses()) {
239 Operation *user = use.getOwner();
240
241 // --- TERMINAL OPS ---
242 if (isa<vector::TransferWriteOp>(Val: user) || isa<vector::StoreOp>(Val: user))
243 return user;
244
245 if (isa<vector::ShuffleOp>(Val: user))
246 return nullptr;
247
248 // --- SCF YIELD ---
249 if (auto yield = dyn_cast<scf::YieldOp>(Val: user)) {
250 Operation *parent = yield->getParentOp();
251 unsigned idx = use.getOperandNumber();
252 if (auto *res =
253 traceToVectorWriteLikeUserOperation(v: parent->getResult(idx)))
254 return res;
255 continue;
256 }
257
258 // --- SCF FOR ---
259 if (auto forOp = dyn_cast<scf::ForOp>(Val: user)) {
260 OpResult loopResult = forOp.getTiedLoopResult(opOperand: &use);
261 if (!loopResult)
262 continue;
263 if (auto *res = traceToVectorWriteLikeUserOperation(v: loopResult))
264 return res;
265 continue;
266 }
267
268 // --- GENERIC CASE ---
269 for (Value res : user->getResults()) {
270 if (auto *found = traceToVectorWriteLikeUserOperation(v: res))
271 return found;
272 }
273 }
274
275 return nullptr;
276}
277
278// This function packs the accumulator of two flat BF16 vector.contract
279// operations into VNNI packed and are then replaced in their respective
280// contraction ops, enabling post-read layout or packing transformations.
281// TODO: replace all use with the packed value along with contration
282// and for op.
283LogicalResult shuffleAfterReadLikeOp(PatternRewriter &rewriter, Operation *opA,
284 Operation *opB,
285 vector::ContractionOp contractA,
286 vector::ContractionOp contractB,
287 int64_t nonUnitDimAcc, VectorType accTy) {
288
289 if (!isa<vector::TransferReadOp, vector::LoadOp>(Val: opA) ||
290 !isa<vector::TransferReadOp, vector::LoadOp>(Val: opB)) {
291 return failure();
292 }
293
294 Operation *insertAfter = opA->isBeforeInBlock(other: opB) ? opB : opA;
295
296 rewriter.setInsertionPointAfter(insertAfter);
297 Location loc = insertAfter->getLoc();
298
299 auto elemTy = accTy.getElementType();
300 auto flatTy = VectorType::get(shape: nonUnitDimAcc, elementType: elemTy);
301
302 auto castA =
303 vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: flatTy, source: opA->getResult(idx: 0));
304 auto castB =
305 vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: flatTy, source: opB->getResult(idx: 0));
306
307 auto masks = getShuffleMasks(
308 nonUnitDimAcc, isInt8Avx2: (elemTy.isSignlessInteger(width: 32) && nonUnitDimAcc == 8));
309
310 auto shuffleLo = vector::ShuffleOp::create(builder&: rewriter, location: loc, result: flatTy, v1: castA,
311 v2: castB, mask: masks.maskLo);
312 auto shuffleHi = vector::ShuffleOp::create(builder&: rewriter, location: loc, result: flatTy, v1: castA,
313 v2: castB, mask: masks.maskHi);
314
315 auto newAccA = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: accTy, source: shuffleLo);
316 auto newAccB = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: accTy, source: shuffleHi);
317
318 rewriter.replaceUsesWithIf(
319 from: opA->getResult(idx: 0), to: newAccA.getResult(), functor: [&](OpOperand &use) {
320 return isa<vector::ContractionOp, scf::ForOp>(Val: use.getOwner());
321 });
322
323 rewriter.replaceUsesWithIf(
324 from: opB->getResult(idx: 0), to: newAccB.getResult(), functor: [&](OpOperand &use) {
325 return isa<vector::ContractionOp, scf::ForOp>(Val: use.getOwner());
326 });
327
328 return success();
329}
330
331// This function shuffles the vectors written by vector.contract operation
332// as a flat layout structure before they are stored.
333LogicalResult shuffleBeforeWriteLikeOp(PatternRewriter &rewriter,
334 Value contractARes, Value contractBRes,
335 int64_t nonUnitDimAcc,
336 VectorType accTy) {
337
338 Value vecA = contractionUsersAfterYield(v: contractARes);
339 Value vecB = contractionUsersAfterYield(v: contractBRes);
340
341 if (!vecA || !vecB)
342 return failure();
343
344 Operation *resultWriteOpA = *vecA.getUsers().begin();
345 Operation *resultWriteOpB = *vecB.getUsers().begin();
346
347 // Decide insertion point and location
348 Operation *insertBefore = resultWriteOpA->isBeforeInBlock(other: resultWriteOpB)
349 ? resultWriteOpA
350 : resultWriteOpB;
351
352 rewriter.setInsertionPoint(insertBefore);
353 Location loc = insertBefore->getLoc();
354
355 auto elemTy = accTy.getElementType();
356 auto flatTy = VectorType::get(shape: nonUnitDimAcc, elementType: elemTy);
357
358 // Flatten vectors
359 auto castA = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: flatTy, source: vecA);
360 auto castB = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: flatTy, source: vecB);
361
362 // TODO: derive shuffle masks instead of hard-coding
363 auto masks = getShuffleMasks(
364 nonUnitDimAcc, isInt8Avx2: (elemTy.isSignlessInteger(width: 32) && nonUnitDimAcc == 8));
365
366 auto shuffledLo = vector::ShuffleOp::create(builder&: rewriter, location: loc, result: flatTy, v1: castA,
367 v2: castB, mask: masks.maskLo);
368 auto shuffledHi = vector::ShuffleOp::create(builder&: rewriter, location: loc, result: flatTy, v1: castA,
369 v2: castB, mask: masks.maskHi);
370
371 // Cast back to accumulator type
372 auto newVecA = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: accTy, source: shuffledLo);
373 auto newVecB = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: accTy, source: shuffledHi);
374
375 rewriter.modifyOpInPlace(root: resultWriteOpA, callable: [&]() {
376 resultWriteOpA->replaceUsesOfWith(from: vecA, to: newVecA);
377 });
378 rewriter.modifyOpInPlace(root: resultWriteOpB, callable: [&]() {
379 resultWriteOpB->replaceUsesOfWith(from: vecB, to: newVecB);
380 });
381
382 return success();
383}
384
385// Return true if vector.contract operations matches on below conditions:
386// (1) - the unitDim operand Lhs or Rhs should be same,
387// (2) - the defining source memref should be same for nonUnitDim
388// operation,
389// (3) - the nonUnit dim offset difference between the
390// vector.contracts should be 8 or 16.
391bool validatePairVectorContract(vector::ContractionOp contractOp,
392 vector::ContractionOp pairContOp,
393 bool rhsHasMultipleNonUnitDims,
394 int64_t nonUnitDimValue) {
395 if (contractOp == pairContOp)
396 return false;
397
398 if (rhsHasMultipleNonUnitDims &&
399 !(contractOp.getLhs() == pairContOp.getLhs()))
400 return false;
401
402 if (!rhsHasMultipleNonUnitDims &&
403 !(contractOp.getRhs() == pairContOp.getRhs()))
404 return false;
405
406 auto nonUnitOperand =
407 rhsHasMultipleNonUnitDims ? contractOp.getRhs() : contractOp.getLhs();
408 auto nonUnitOperandPairContOp =
409 rhsHasMultipleNonUnitDims ? pairContOp.getRhs() : pairContOp.getLhs();
410
411 Value srcBuff;
412 SmallVector<OpFoldResult> indexVals;
413 llvm::TypeSwitch<Operation *>(nonUnitOperand.getDefiningOp())
414 .Case<vector::TransferReadOp, vector::LoadOp>(caseFn: [&](auto readOp) {
415 srcBuff = readOp.getOperand(0);
416 indexVals = SmallVector<OpFoldResult>(readOp.getIndices().begin(),
417 readOp.getIndices().end());
418 })
419 .Case<vector::ShapeCastOp>(caseFn: [&](vector::ShapeCastOp op) {
420 srcBuff = op.getSource();
421 indexVals.clear();
422 });
423
424 Value srcBuffPairContOp;
425 SmallVector<OpFoldResult> indexValsPairContOp;
426 llvm::TypeSwitch<Operation *>(nonUnitOperandPairContOp.getDefiningOp())
427 .Case<vector::TransferReadOp, vector::LoadOp>(caseFn: [&](auto readOp) {
428 srcBuffPairContOp = readOp.getOperand(0);
429 indexValsPairContOp = SmallVector<OpFoldResult>(
430 readOp.getIndices().begin(), readOp.getIndices().end());
431 })
432 .Case<vector::ShapeCastOp>(caseFn: [&](vector::ShapeCastOp op) {
433 srcBuffPairContOp = op.getSource();
434 indexVals.clear();
435 });
436
437 if (!srcBuff || !srcBuffPairContOp)
438 return false;
439
440 auto shuffleLw = srcBuff.getDefiningOp<vector::ShuffleOp>();
441 auto shuffleHw = srcBuffPairContOp.getDefiningOp<vector::ShuffleOp>();
442
443 if (shuffleLw && shuffleHw)
444 return shuffleLw.getV1() == shuffleHw.getV1() &&
445 shuffleLw.getV2() == shuffleHw.getV2();
446
447 if (srcBuff != srcBuffPairContOp)
448 return false;
449
450 bool oneConstantOffset = false;
451 for (size_t i = 0; i < indexVals.size(); i++) {
452
453 if (indexVals[i] == indexValsPairContOp[i])
454 continue;
455
456 auto v0 = getConstantIntValue(ofr: indexVals[i]);
457 auto v1 = getConstantIntValue(ofr: indexValsPairContOp[i]);
458
459 if (!v0 || !v1)
460 return false;
461
462 if ((*v1 - *v0) != nonUnitDimValue)
463 return false;
464
465 oneConstantOffset = true;
466 }
467
468 return oneConstantOffset;
469}
470
471} // namespace x86
472} // namespace mlir
473