1//===- VectorDistribute.cpp - patterns to do vector distribution ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Affine/IR/AffineOps.h"
10#include "mlir/Dialect/Arith/IR/Arith.h"
11#include "mlir/Dialect/GPU/IR/GPUDialect.h"
12#include "mlir/Dialect/GPU/Utils/DistributionUtils.h"
13#include "mlir/Dialect/MemRef/IR/MemRef.h"
14#include "mlir/Dialect/SCF/IR/SCF.h"
15#include "mlir/Dialect/UB/IR/UBOps.h"
16#include "mlir/Dialect/Vector/IR/VectorOps.h"
17#include "mlir/Dialect/Vector/Transforms/VectorDistribution.h"
18#include "mlir/IR/AffineExpr.h"
19#include "mlir/IR/Attributes.h"
20#include "mlir/IR/BuiltinTypes.h"
21#include "mlir/Interfaces/LoopLikeInterface.h"
22#include "mlir/Interfaces/SideEffectInterfaces.h"
23#include "mlir/Transforms/RegionUtils.h"
24#include "llvm/ADT/SetVector.h"
25#include "llvm/ADT/SmallBitVector.h"
26#include "llvm/ADT/SmallVectorExtras.h"
27#include "llvm/Support/FormatVariadic.h"
28#include <utility>
29
30using namespace mlir;
31using namespace mlir::vector;
32using namespace mlir::gpu;
33
34/// Currently the distribution map is implicit based on the vector shape. In the
35/// future it will be part of the op.
36/// Example:
37/// ```
38/// %0 = gpu.warp_execute_on_lane_0(%arg0) -> (vector<1x16x2xf32>) {
39/// ...
40/// gpu.yield %3 : vector<32x16x64xf32>
41/// }
42/// ```
43/// Would have an implicit map of:
44/// `(d0, d1, d2) -> (d0, d2)`
45static AffineMap calculateImplicitMap(VectorType sequentialType,
46 VectorType distributedType) {
47 SmallVector<AffineExpr> perm;
48 perm.reserve(N: 1);
49 // Check which dimensions of the sequential type are different than the
50 // dimensions of the distributed type to know the distributed dimensions. Then
51 // associate each distributed dimension to an ID in order.
52 for (unsigned i = 0, e = sequentialType.getRank(); i < e; i++) {
53 if (sequentialType.getDimSize(idx: i) != distributedType.getDimSize(idx: i))
54 perm.push_back(Elt: getAffineDimExpr(position: i, context: distributedType.getContext()));
55 }
56 auto map = AffineMap::get(dimCount: sequentialType.getRank(), symbolCount: 0, results: perm,
57 context: distributedType.getContext());
58 return map;
59}
60
61/// Given a sequential and distributed vector type, returns the distributed
62/// dimension. This function expects that only a single dimension is
63/// distributed.
64static int getDistributedDim(VectorType sequentialType,
65 VectorType distributedType) {
66 assert(sequentialType.getRank() == distributedType.getRank() &&
67 "sequential and distributed vector types must have the same rank");
68 int64_t distributedDim = -1;
69 for (int64_t i = 0; i < sequentialType.getRank(); ++i) {
70 if (distributedType.getDimSize(idx: i) != sequentialType.getDimSize(idx: i)) {
71 // Keep this assert here in case WarpExecuteOnLane0Op gets extended to
72 // support distributing multiple dimensions in the future.
73 assert(distributedDim == -1 && "found multiple distributed dims");
74 distributedDim = i;
75 }
76 }
77 return distributedDim;
78}
79
80namespace {
81
82/// Helper struct to create the load / store operations that permit transit
83/// through the parallel / sequential and the sequential / parallel boundaries
84/// when performing `rewriteWarpOpToScfFor`.
85///
86/// The vector distribution dimension is inferred from the vector types.
87struct DistributedLoadStoreHelper {
88 DistributedLoadStoreHelper(Value sequentialVal, Value distributedVal,
89 Value laneId, Value zero)
90 : sequentialVal(sequentialVal), distributedVal(distributedVal),
91 laneId(laneId), zero(zero) {
92 sequentialVectorType = dyn_cast<VectorType>(Val: sequentialVal.getType());
93 distributedVectorType = dyn_cast<VectorType>(Val: distributedVal.getType());
94 if (sequentialVectorType && distributedVectorType)
95 distributionMap =
96 calculateImplicitMap(sequentialType: sequentialVectorType, distributedType: distributedVectorType);
97 }
98
99 Value buildDistributedOffset(RewriterBase &b, Location loc, int64_t index) {
100 int64_t distributedSize = distributedVectorType.getDimSize(idx: index);
101 AffineExpr tid = getAffineSymbolExpr(position: 0, context: b.getContext());
102 return b.createOrFold<affine::AffineApplyOp>(location: loc, args: tid * distributedSize,
103 args: ArrayRef<Value>{laneId});
104 }
105
106 /// Create a store during the process of distributing the
107 /// `vector.warp_execute_on_thread_0` op.
108 /// Vector distribution assumes the following convention regarding the
109 /// temporary buffers that are created to transition values. This **must**
110 /// be properly specified in the `options.warpAllocationFn`:
111 /// 1. scalars of type T transit through a memref<1xT>.
112 /// 2. vectors of type V<shapexT> transit through a memref<shapexT>
113 Operation *buildStore(RewriterBase &b, Location loc, Value val,
114 Value buffer) {
115 assert((val == distributedVal || val == sequentialVal) &&
116 "Must store either the preregistered distributed or the "
117 "preregistered sequential value.");
118 // Scalar case can directly use memref.store.
119 if (!isa<VectorType>(Val: val.getType()))
120 return memref::StoreOp::create(builder&: b, location: loc, valueToStore: val, memref: buffer, indices: zero);
121
122 // Vector case must use vector::TransferWriteOp which will later lower to
123 // vector.store of memref.store depending on further lowerings.
124 int64_t rank = sequentialVectorType.getRank();
125 SmallVector<Value> indices(rank, zero);
126 if (val == distributedVal) {
127 for (auto dimExpr : distributionMap.getResults()) {
128 int64_t index = cast<AffineDimExpr>(Val&: dimExpr).getPosition();
129 indices[index] = buildDistributedOffset(b, loc, index);
130 }
131 }
132 SmallVector<bool> inBounds(indices.size(), true);
133 return vector::TransferWriteOp::create(
134 builder&: b, location: loc, vector: val, dest: buffer, indices,
135 inBounds: ArrayRef<bool>(inBounds.begin(), inBounds.end()));
136 }
137
138 /// Create a load during the process of distributing the
139 /// `vector.warp_execute_on_thread_0` op.
140 /// Vector distribution assumes the following convention regarding the
141 /// temporary buffers that are created to transition values. This **must**
142 /// be properly specified in the `options.warpAllocationFn`:
143 /// 1. scalars of type T transit through a memref<1xT>.
144 /// 2. vectors of type V<shapexT> transit through a memref<shapexT>
145 ///
146 /// When broadcastMode is true, the load is not distributed to account for
147 /// the broadcast semantics of the `gpu.warp_execute_on_lane_0` op.
148 ///
149 /// Example:
150 ///
151 /// ```
152 /// %r = gpu.warp_execute_on_lane_0(...) -> (f32) {
153 /// gpu.yield %cst : f32
154 /// }
155 /// // Both types are f32. The constant %cst is broadcasted to all lanes.
156 /// ```
157 /// This behavior described in more detail in the documentation of the op.
158 Value buildLoad(RewriterBase &b, Location loc, Type type, Value buffer) {
159
160 // Scalar case can directly use memref.store.
161 if (!isa<VectorType>(Val: type))
162 return memref::LoadOp::create(builder&: b, location: loc, memref: buffer, indices: zero);
163
164 // Other cases must be vector atm.
165 // Vector case must use vector::TransferReadOp which will later lower to
166 // vector.read of memref.read depending on further lowerings.
167 assert((type == distributedVectorType || type == sequentialVectorType) &&
168 "Must store either the preregistered distributed or the "
169 "preregistered sequential type.");
170 SmallVector<Value> indices(sequentialVectorType.getRank(), zero);
171 if (type == distributedVectorType) {
172 for (auto dimExpr : distributionMap.getResults()) {
173 int64_t index = cast<AffineDimExpr>(Val&: dimExpr).getPosition();
174 indices[index] = buildDistributedOffset(b, loc, index);
175 }
176 }
177 SmallVector<bool> inBounds(indices.size(), true);
178 return vector::TransferReadOp::create(
179 builder&: b, location: loc, vectorType: cast<VectorType>(Val&: type), source: buffer, indices,
180 /*padding=*/std::nullopt,
181 inBounds: ArrayRef<bool>(inBounds.begin(), inBounds.end()));
182 }
183
184 Value sequentialVal, distributedVal, laneId, zero;
185 VectorType sequentialVectorType, distributedVectorType;
186 AffineMap distributionMap;
187};
188
189} // namespace
190
191// Clones `op` into a new operation that takes `operands` and returns
192// `resultTypes`.
193static Operation *cloneOpWithOperandsAndTypes(RewriterBase &rewriter,
194 Location loc, Operation *op,
195 ArrayRef<Value> operands,
196 ArrayRef<Type> resultTypes) {
197 OperationState res(loc, op->getName().getStringRef(), operands, resultTypes,
198 op->getDiscardableAttrDictionary().getValue());
199 res.propertiesAttr = op->getPropertiesAsAttribute();
200 return rewriter.create(state: res);
201}
202
203/// Reuse an args buffer from `buffers` to transit a value of `targetType`, or
204/// return null if none fits.
205static Value tryReuseArgBuffer(SmallVectorImpl<std::pair<Value, bool>> &buffers,
206 Type targetType, Location loc,
207 RewriterBase &rewriter) {
208 auto targetVecType = dyn_cast<VectorType>(Val&: targetType);
209 if (!targetVecType || !targetVecType.hasStaticShape())
210 return Value();
211
212 for (auto &entry : buffers) {
213 if (entry.second)
214 continue;
215 Value candidate = entry.first;
216 auto candidateMemref = dyn_cast<MemRefType>(Val: candidate.getType());
217 if (!candidateMemref || !candidateMemref.hasStaticShape())
218 continue;
219 if (candidateMemref.getElementType() != targetVecType.getElementType())
220 continue;
221 // Skip rank mismatch up-front to avoid guessing a reshape.
222 if (candidateMemref.getRank() != targetVecType.getRank())
223 continue;
224 // The candidate must be at least as large as the target in every
225 // dimension, otherwise the leading subview below would run out of bounds.
226 // This also guarantees that the candidate has enough elements in total.
227 if (!llvm::all_of(Range: llvm::seq<unsigned>(Begin: 0, End: targetVecType.getRank()),
228 P: [&](unsigned i) {
229 return candidateMemref.getDimSize(idx: i) >=
230 targetVecType.getDimSize(idx: i);
231 }))
232 continue;
233
234 // All checks passed: claim the buffer.
235 entry.second = true;
236
237 // Same shape: reuse directly.
238 if (candidateMemref.getShape() == targetVecType.getShape())
239 return candidate;
240
241 // Larger candidate: take a leading subview of the target shape.
242 int64_t candidateRank = candidateMemref.getRank();
243 SmallVector<OpFoldResult> offsets(candidateRank, rewriter.getIndexAttr(value: 0));
244 SmallVector<OpFoldResult> strides(candidateRank, rewriter.getIndexAttr(value: 1));
245 SmallVector<OpFoldResult> sizes;
246 sizes.reserve(N: candidateRank);
247 for (int64_t dim : targetVecType.getShape())
248 sizes.push_back(Elt: rewriter.getIndexAttr(value: dim));
249
250 return memref::SubViewOp::create(builder&: rewriter, location: loc, source: candidate, offsets, sizes,
251 strides);
252 }
253 return Value();
254}
255
256/// Return true if `block` is part of a CFG cycle, which
257/// `LoopLikeOpInterface` does not cover.
258static bool isBlockInCFGLoop(Block *block) {
259 llvm::SmallSetVector<Block *, 4> visited;
260 SmallVector<Block *> worklist(block->getSuccessors().begin(),
261 block->getSuccessors().end());
262 while (!worklist.empty()) {
263 Block *current = worklist.pop_back_val();
264 if (current == block)
265 return true;
266 if (!visited.insert(X: current))
267 continue;
268 for (Block *succ : current->getSuccessors())
269 worklist.push_back(Elt: succ);
270 }
271 return false;
272}
273
274/// Return true if `block` or any of its ancestor blocks is part of a CFG
275/// cycle, which `LoopLikeOpInterface` does not cover.
276static bool isInCFGLoop(Block *block) {
277 while (block) {
278 if (isBlockInCFGLoop(block))
279 return true;
280 Operation *parent = block->getParentOp();
281 block = parent ? parent->getBlock() : nullptr;
282 }
283 return false;
284}
285
286namespace {
287/// Rewrite a WarpExecuteOnLane0Op into a predicated scf.if op where the single
288/// thread `laneId` executes the entirety of the computation.
289///
290/// After the transformation:
291/// - the IR within the scf.if op can be thought of as executing sequentially
292/// (from the point of view of threads along `laneId`).
293/// - the IR outside of the scf.if op can be thought of as executing in
294/// parallel (from the point of view of threads along `laneId`).
295///
296/// Values that need to transit through the parallel / sequential and the
297/// sequential / parallel boundaries do so via reads and writes to a temporary
298/// memory location.
299///
300/// The transformation proceeds in multiple steps:
301/// 1. Create the scf.if op.
302/// 2. Insert appropriate (alloc, write)-pairs before the scf.if and reads
303/// within the scf.if to transit the values captured from above.
304/// 3. Synchronize before the scf.if to ensure all writes inserted in 2. are
305/// consistent within the scf.if.
306/// 4. Move the body of the WarpExecuteOnLane0Op inside the scf.if.
307/// 5. Insert appropriate writes within scf.if and reads after the scf.if to
308/// transit the values returned by the op.
309/// 6. Synchronize after the scf.if to ensure all writes inserted in 5. are
310/// consistent after the scf.if.
311/// 7. Perform late cleanups.
312///
313/// All this assumes the vector distribution occurs along the most minor
314/// distributed vector dimension.
315struct WarpOpToScfIfPattern : public WarpDistributionPattern {
316 WarpOpToScfIfPattern(MLIRContext *context,
317 const WarpExecuteOnLane0LoweringOptions &options,
318 PatternBenefit benefit = 1)
319 : WarpDistributionPattern(context, benefit), options(options) {}
320
321 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
322 PatternRewriter &rewriter) const override {
323 assert(warpOp.getBodyRegion().hasOneBlock() &&
324 "expected WarpOp with single block");
325 Block *warpOpBody = &warpOp.getBodyRegion().front();
326 Location loc = warpOp.getLoc();
327
328 // Passed all checks. Start rewriting.
329 OpBuilder::InsertionGuard g(rewriter);
330 rewriter.setInsertionPoint(warpOp);
331
332 // Step 1: Create scf.if op.
333 Value c0 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
334 Value isLane0 = arith::CmpIOp::create(
335 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq, lhs: warpOp.getLaneid(), rhs: c0);
336 auto ifOp = scf::IfOp::create(builder&: rewriter, location: loc, cond: isLane0,
337 /*withElseRegion=*/false);
338 rewriter.eraseOp(op: ifOp.thenBlock()->getTerminator());
339
340 // Step 2: insert appropriate (alloc, write)-pairs before the scf.if and
341 // reads within the scf.if to transit the values captured from above.
342 SmallVector<Value> bbArgReplacements;
343 // Pool of args buffers paired with a "consumed" flag, so Step 5 can try
344 // to reuse them for yielded values' transit buffers. Skip reuse inside a
345 // loop (structured or CFG): thread convergence is only guaranteed within
346 // a barrier, so the next iteration's arg stores could race with this
347 // iteration's result loads.
348 SmallVector<std::pair<Value, /*consumed=*/bool>> reusableArgBuffers;
349 bool canReuseArgBuffers = !warpOp->getParentOfType<LoopLikeOpInterface>() &&
350 !isInCFGLoop(block: warpOp->getBlock());
351 if (canReuseArgBuffers)
352 reusableArgBuffers.reserve(N: warpOp.getArgs().size());
353 for (const auto &it : llvm::enumerate(First: warpOp.getArgs())) {
354 Value sequentialVal = warpOpBody->getArgument(i: it.index());
355 Value distributedVal = it.value();
356 DistributedLoadStoreHelper helper(sequentialVal, distributedVal,
357 warpOp.getLaneid(), c0);
358
359 // Create buffer before the ifOp.
360 rewriter.setInsertionPoint(ifOp);
361 Value buffer = options.warpAllocationFn(loc, rewriter, warpOp,
362 sequentialVal.getType());
363 if (canReuseArgBuffers)
364 reusableArgBuffers.emplace_back(Args&: buffer, /*consumed=*/Args: false);
365 // Store distributed vector into buffer, before the ifOp.
366 helper.buildStore(b&: rewriter, loc, val: distributedVal, buffer);
367 // Load sequential vector from buffer, inside the ifOp.
368 rewriter.setInsertionPointToStart(ifOp.thenBlock());
369 bbArgReplacements.push_back(
370 Elt: helper.buildLoad(b&: rewriter, loc, type: sequentialVal.getType(), buffer));
371 }
372
373 // Step 3. Insert sync after all the stores and before all the loads.
374 if (!warpOp.getArgs().empty()) {
375 rewriter.setInsertionPoint(ifOp);
376 options.warpSynchronizationFn(loc, rewriter, warpOp);
377 }
378
379 // Step 4. Move body of warpOp to ifOp.
380 rewriter.mergeBlocks(source: warpOpBody, dest: ifOp.thenBlock(), argValues: bbArgReplacements);
381
382 // Step 5. Insert appropriate writes within scf.if and reads after the
383 // scf.if to transit the values returned by the op.
384 // The args buffers from Step 2 are unused by now, try to reuse previous
385 // buffers.
386 SmallVector<Value> replacements;
387 auto yieldOp = cast<gpu::YieldOp>(Val: ifOp.thenBlock()->getTerminator());
388 Location yieldLoc = yieldOp.getLoc();
389 for (const auto &it : llvm::enumerate(First: yieldOp.getOperands())) {
390 Value sequentialVal = it.value();
391 Value distributedVal = warpOp->getResult(idx: it.index());
392 DistributedLoadStoreHelper helper(sequentialVal, distributedVal,
393 warpOp.getLaneid(), c0);
394
395 // Reuse an args buffer if possible, otherwise allocate.
396 rewriter.setInsertionPoint(ifOp);
397 Value buffer = tryReuseArgBuffer(buffers&: reusableArgBuffers,
398 targetType: sequentialVal.getType(), loc, rewriter);
399 if (!buffer)
400 buffer = options.warpAllocationFn(loc, rewriter, warpOp,
401 sequentialVal.getType());
402
403 // Store yielded value into buffer, inside the ifOp, before the
404 // terminator.
405 rewriter.setInsertionPoint(yieldOp);
406 helper.buildStore(b&: rewriter, loc, val: sequentialVal, buffer);
407
408 // Load distributed value from buffer, after the warpOp.
409 rewriter.setInsertionPointAfter(ifOp);
410 // Result type and yielded value type are the same. This is a broadcast.
411 // E.g.:
412 // %r = gpu.warp_execute_on_lane_0(...) -> (f32) {
413 // gpu.yield %cst : f32
414 // }
415 // Both types are f32. The constant %cst is broadcasted to all lanes.
416 // This is described in more detail in the documentation of the op.
417 replacements.push_back(
418 Elt: helper.buildLoad(b&: rewriter, loc, type: distributedVal.getType(), buffer));
419 }
420
421 // Step 6. Insert sync after all the stores and before all the loads.
422 if (!yieldOp.getOperands().empty()) {
423 rewriter.setInsertionPointAfter(ifOp);
424 options.warpSynchronizationFn(loc, rewriter, warpOp);
425 }
426
427 // Step 7. Delete terminator and add empty scf.yield.
428 rewriter.eraseOp(op: yieldOp);
429 rewriter.setInsertionPointToEnd(ifOp.thenBlock());
430 scf::YieldOp::create(builder&: rewriter, location: yieldLoc);
431
432 // Compute replacements for WarpOp results.
433 rewriter.replaceOp(op: warpOp, newValues: replacements);
434
435 return success();
436 }
437
438private:
439 const WarpExecuteOnLane0LoweringOptions &options;
440};
441
442/// Return the distributed vector type based on the original type and the
443/// distribution map. The map is expected to have a dimension equal to the
444/// original type rank and should be a projection where the results are the
445/// distributed dimensions. If the number of results is zero there is no
446/// distribution (i.e. original type is returned).
447/// Otherwise, The number of results should be equal to the number
448/// of warp sizes which is currently limited to 1.
449/// Example: For a vector<16x32x64> distributed with a map(d0, d1, d2) -> (d1)
450/// and a warp size of 16 would distribute the second dimension (associated to
451/// d1) and return vector<16x2x64>
452static VectorType getDistributedType(VectorType originalType, AffineMap map,
453 int64_t warpSize) {
454 // If the map has zero results, return the original type.
455 if (map.getNumResults() == 0)
456 return originalType;
457 SmallVector<int64_t> targetShape(originalType.getShape());
458 for (unsigned i = 0, e = map.getNumResults(); i < e; i++) {
459 unsigned position = map.getDimPosition(idx: i);
460 if (targetShape[position] % warpSize != 0) {
461 if (warpSize % targetShape[position] != 0) {
462 return VectorType();
463 }
464 warpSize /= targetShape[position];
465 targetShape[position] = 1;
466 continue;
467 }
468 targetShape[position] = targetShape[position] / warpSize;
469 warpSize = 1;
470 break;
471 }
472 if (warpSize != 1) {
473 return VectorType();
474 }
475 VectorType targetType =
476 VectorType::get(shape: targetShape, elementType: originalType.getElementType());
477 return targetType;
478}
479
480/// Given a warpOp that contains ops with regions, the corresponding op's
481/// "inner" region and the distributionMapFn, get all values used by the op's
482/// region that are defined within the warpOp, but outside the inner region.
483/// Return the set of values, their types and their distributed types.
484std::tuple<llvm::SmallSetVector<Value, 32>, SmallVector<Type>,
485 SmallVector<Type>>
486getInnerRegionEscapingValues(WarpExecuteOnLane0Op warpOp, Region &innerRegion,
487 DistributionMapFn distributionMapFn) {
488 llvm::SmallSetVector<Value, 32> escapingValues;
489 SmallVector<Type> escapingValueTypes;
490 SmallVector<Type> escapingValueDistTypes; // to yield from the new warpOp
491 if (innerRegion.empty())
492 return {std::move(escapingValues), std::move(escapingValueTypes),
493 std::move(escapingValueDistTypes)};
494 mlir::visitUsedValuesDefinedAbove(regions: innerRegion, callback: [&](OpOperand *operand) {
495 Operation *parent = operand->get().getParentRegion()->getParentOp();
496 if (warpOp->isAncestor(other: parent)) {
497 if (!escapingValues.insert(X: operand->get()))
498 return;
499 Type distType = operand->get().getType();
500 if (auto vecType = dyn_cast<VectorType>(Val&: distType)) {
501 AffineMap map = distributionMapFn(operand->get());
502 distType = getDistributedType(originalType: vecType, map,
503 warpSize: map.isEmpty() ? 1 : warpOp.getWarpSize());
504 }
505 escapingValueTypes.push_back(Elt: operand->get().getType());
506 escapingValueDistTypes.push_back(Elt: distType);
507 }
508 });
509 return {std::move(escapingValues), std::move(escapingValueTypes),
510 std::move(escapingValueDistTypes)};
511}
512
513/// Distribute transfer_write ops based on the affine map returned by
514/// `distributionMapFn`. Writes of size more than `maxNumElementToExtract`
515/// will not be distributed (it should be less than the warp size).
516///
517/// Example:
518/// ```
519/// %0 = gpu.warp_execute_on_lane_0(%id){
520/// ...
521/// vector.transfer_write %v, %A[%c0] : vector<32xf32>, memref<128xf32>
522/// gpu.yield
523/// }
524/// ```
525/// To
526/// ```
527/// %r:3 = gpu.warp_execute_on_lane_0(%id) -> (vector<1xf32>) {
528/// ...
529/// gpu.yield %v : vector<32xf32>
530/// }
531/// vector.transfer_write %v, %A[%id] : vector<1xf32>, memref<128xf32>
532struct WarpOpTransferWrite : public WarpDistributionPattern {
533 WarpOpTransferWrite(MLIRContext *ctx, DistributionMapFn fn,
534 unsigned maxNumElementsToExtract, PatternBenefit b = 1)
535 : WarpDistributionPattern(ctx, b), distributionMapFn(std::move(fn)),
536 maxNumElementsToExtract(maxNumElementsToExtract) {}
537
538 /// Distribute the TransferWriteOp. Only 1D distributions and vector dims that
539 /// are multiples of the distribution ratio are supported at the moment.
540 LogicalResult tryDistributeOp(RewriterBase &rewriter,
541 vector::TransferWriteOp writeOp,
542 WarpExecuteOnLane0Op warpOp) const {
543 VectorType writtenVectorType = writeOp.getVectorType();
544
545 // 1. If the write is 0-D, we just clone it into a new WarpExecuteOnLane0Op
546 // to separate it from the rest.
547 if (writtenVectorType.getRank() == 0)
548 return failure();
549
550 // 2. Compute the distributed type.
551 AffineMap map = distributionMapFn(writeOp.getVector());
552 VectorType targetType =
553 getDistributedType(originalType: writtenVectorType, map, warpSize: warpOp.getWarpSize());
554 if (!targetType)
555 return failure();
556
557 // 2.5 Compute the distributed type for the new mask;
558 VectorType maskType;
559 if (writeOp.getMask()) {
560 // TODO: Distribution of masked writes with non-trivial permutation maps
561 // requires the distribution of the mask to elementwise match the
562 // distribution of the permuted written vector. Currently the details
563 // of which lane is responsible for which element is captured strictly
564 // by shape information on the warp op, and thus requires materializing
565 // the permutation in IR.
566 if (!writeOp.getPermutationMap().isMinorIdentity())
567 return failure();
568 maskType =
569 getDistributedType(originalType: writeOp.getMaskType(), map, warpSize: warpOp.getWarpSize());
570 }
571
572 // 3. clone the write into a new WarpExecuteOnLane0Op to separate it from
573 // the rest.
574 vector::TransferWriteOp newWriteOp =
575 cloneWriteOp(rewriter, warpOp, writeOp, targetType, maybeMaskType: maskType);
576
577 // 4. Reindex the write using the distribution map.
578 auto newWarpOp =
579 newWriteOp.getVector().getDefiningOp<WarpExecuteOnLane0Op>();
580
581 // Delinearize the lane id based on the way threads are divided across the
582 // vector. To get the number of threads per vector dimension, divide the
583 // sequential size by the distributed size along each dim.
584 rewriter.setInsertionPoint(newWriteOp);
585 SmallVector<OpFoldResult> delinearizedIdSizes;
586 for (auto [seqSize, distSize] :
587 llvm::zip_equal(t: writtenVectorType.getShape(), u: targetType.getShape())) {
588 assert(seqSize % distSize == 0 && "Invalid distributed vector shape");
589 delinearizedIdSizes.push_back(Elt: rewriter.getIndexAttr(value: seqSize / distSize));
590 }
591 SmallVector<Value> delinearized;
592 if (map.getNumResults() > 1) {
593 delinearized = mlir::affine::AffineDelinearizeIndexOp::create(
594 builder&: rewriter, location: newWarpOp.getLoc(), linear_index: newWarpOp.getLaneid(),
595 basis: delinearizedIdSizes)
596 .getResults();
597 } else {
598 // If there is only one map result, we can elide the delinearization
599 // op and use the lane id directly.
600 delinearized.append(NumInputs: targetType.getRank(), Elt: newWarpOp.getLaneid());
601 }
602
603 AffineMap indexMap = map.compose(map: newWriteOp.getPermutationMap());
604 Location loc = newWriteOp.getLoc();
605 SmallVector<Value> indices(newWriteOp.getIndices().begin(),
606 newWriteOp.getIndices().end());
607 for (auto it : llvm::zip(t: indexMap.getResults(), u: map.getResults())) {
608 AffineExpr d0, d1;
609 bindDims(ctx: newWarpOp.getContext(), exprs&: d0, exprs&: d1);
610 auto indexExpr = dyn_cast<AffineDimExpr>(Val: std::get<0>(t&: it));
611 if (!indexExpr)
612 continue;
613 unsigned indexPos = indexExpr.getPosition();
614 unsigned vectorPos = cast<AffineDimExpr>(Val: std::get<1>(t&: it)).getPosition();
615 Value laneId = delinearized[vectorPos];
616 auto scale =
617 rewriter.getAffineConstantExpr(constant: targetType.getDimSize(idx: vectorPos));
618 indices[indexPos] = affine::makeComposedAffineApply(
619 b&: rewriter, loc, e: d0 + scale * d1, operands: {indices[indexPos], laneId});
620 }
621 newWriteOp.getIndicesMutable().assign(values: indices);
622
623 return success();
624 }
625
626 /// Extract TransferWriteOps of vector<1x> into a separate warp op.
627 LogicalResult tryExtractOp(RewriterBase &rewriter,
628 vector::TransferWriteOp writeOp,
629 WarpExecuteOnLane0Op warpOp) const {
630 Location loc = writeOp.getLoc();
631 VectorType vecType = writeOp.getVectorType();
632
633 if (vecType.getNumElements() > maxNumElementsToExtract) {
634 return rewriter.notifyMatchFailure(
635 arg&: warpOp,
636 msg: llvm::formatv(
637 Fmt: "writes more elements ({0}) than allowed to extract ({1})",
638 Vals: vecType.getNumElements(), Vals: maxNumElementsToExtract));
639 }
640
641 // Do not process warp ops that contain only TransferWriteOps.
642 if (llvm::all_of(Range: warpOp.getOps(),
643 P: llvm::IsaPred<vector::TransferWriteOp, gpu::YieldOp>))
644 return failure();
645
646 SmallVector<Value> yieldValues = {writeOp.getVector()};
647 SmallVector<Type> retTypes = {vecType};
648 SmallVector<size_t> newRetIndices;
649 WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
650 rewriter, warpOp, newYieldedValues: yieldValues, newReturnTypes: retTypes, indices&: newRetIndices);
651 rewriter.setInsertionPointAfter(newWarpOp);
652
653 // Create a second warp op that contains only writeOp.
654 auto secondWarpOp = WarpExecuteOnLane0Op::create(builder&: rewriter, location: loc, resultTypes: TypeRange(),
655 laneid: newWarpOp.getLaneid(),
656 warpSize: newWarpOp.getWarpSize());
657 Block &body = secondWarpOp.getBodyRegion().front();
658 rewriter.setInsertionPointToStart(&body);
659 auto newWriteOp =
660 cast<vector::TransferWriteOp>(Val: rewriter.clone(op&: *writeOp.getOperation()));
661 newWriteOp.getValueToStoreMutable().assign(
662 value: newWarpOp.getResult(i: newRetIndices[0]));
663 rewriter.eraseOp(op: writeOp);
664 gpu::YieldOp::create(builder&: rewriter, location: newWarpOp.getLoc());
665 return success();
666 }
667
668 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
669 PatternRewriter &rewriter) const override {
670 gpu::YieldOp yield = warpOp.getTerminator();
671 Operation *lastNode = yield->getPrevNode();
672 auto writeOp = dyn_cast_or_null<vector::TransferWriteOp>(Val: lastNode);
673 if (!writeOp)
674 return failure();
675
676 Value maybeMask = writeOp.getMask();
677 if (!llvm::all_of(Range: writeOp->getOperands(), P: [&](Value value) {
678 return writeOp.getVector() == value ||
679 (maybeMask && maybeMask == value) ||
680 warpOp.isDefinedOutsideOfRegion(value);
681 }))
682 return failure();
683
684 if (succeeded(Result: tryDistributeOp(rewriter, writeOp, warpOp)))
685 return success();
686
687 // Masked writes not supported for extraction.
688 if (writeOp.getMask())
689 return failure();
690
691 if (succeeded(Result: tryExtractOp(rewriter, writeOp, warpOp)))
692 return success();
693
694 return failure();
695 }
696
697private:
698 /// Clone `writeOp` assumed to be nested under `warpOp` into a new warp
699 /// execute op with the proper return type. The new write op is updated to
700 /// write the result of the new warp execute op. The old `writeOp` is deleted.
701 vector::TransferWriteOp cloneWriteOp(RewriterBase &rewriter,
702 WarpExecuteOnLane0Op warpOp,
703 vector::TransferWriteOp writeOp,
704 VectorType targetType,
705 VectorType maybeMaskType) const {
706 assert(writeOp->getParentOp() == warpOp &&
707 "write must be nested immediately under warp");
708 OpBuilder::InsertionGuard g(rewriter);
709 SmallVector<size_t> newRetIndices;
710 WarpExecuteOnLane0Op newWarpOp;
711 if (maybeMaskType) {
712 newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
713 rewriter, warpOp, newYieldedValues: ValueRange{writeOp.getVector(), writeOp.getMask()},
714 newReturnTypes: TypeRange{targetType, maybeMaskType}, indices&: newRetIndices);
715 } else {
716 newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
717 rewriter, warpOp, newYieldedValues: ValueRange{{writeOp.getVector()}},
718 newReturnTypes: TypeRange{targetType}, indices&: newRetIndices);
719 }
720 rewriter.setInsertionPointAfter(newWarpOp);
721 auto newWriteOp =
722 cast<vector::TransferWriteOp>(Val: rewriter.clone(op&: *writeOp.getOperation()));
723 rewriter.eraseOp(op: writeOp);
724 newWriteOp.getValueToStoreMutable().assign(
725 value: newWarpOp.getResult(i: newRetIndices[0]));
726 if (maybeMaskType)
727 newWriteOp.getMaskMutable().assign(value: newWarpOp.getResult(i: newRetIndices[1]));
728 return newWriteOp;
729 }
730
731 DistributionMapFn distributionMapFn;
732 unsigned maxNumElementsToExtract = 1;
733};
734
735/// Sink out elementwise op feeding into a warp op yield.
736/// ```
737/// %0 = gpu.warp_execute_on_lane_0(%arg0) -> (vector<1xf32>) {
738/// ...
739/// %3 = arith.addf %1, %2 : vector<32xf32>
740/// gpu.yield %3 : vector<32xf32>
741/// }
742/// ```
743/// To
744/// ```
745/// %r:3 = gpu.warp_execute_on_lane_0(%arg0) -> (vector<1xf32>,
746/// vector<1xf32>, vector<1xf32>) {
747/// ...
748/// %4 = arith.addf %2, %3 : vector<32xf32>
749/// gpu.yield %4, %2, %3 : vector<32xf32>, vector<32xf32>,
750/// vector<32xf32>
751/// }
752/// %0 = arith.addf %r#1, %r#2 : vector<1xf32>
753struct WarpOpElementwise : public WarpDistributionPattern {
754 using Base::Base;
755 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
756 PatternRewriter &rewriter) const override {
757 OpOperand *yieldOperand = getWarpResult(warpOp, fn: [](Operation *op) {
758 return OpTrait::hasElementwiseMappableTraits(op);
759 });
760 if (!yieldOperand)
761 return failure();
762
763 Operation *elementWise = yieldOperand->get().getDefiningOp();
764 unsigned operandIndex = yieldOperand->getOperandNumber();
765 Value distributedVal = warpOp.getResult(i: operandIndex);
766 SmallVector<Value> yieldValues;
767 SmallVector<Type> retTypes;
768 Location loc = warpOp.getLoc();
769 for (OpOperand &operand : elementWise->getOpOperands()) {
770 Type targetType;
771 if (auto vecType = dyn_cast<VectorType>(Val: distributedVal.getType())) {
772 // If the result type is a vector, the operands must also be vectors.
773 auto operandType = cast<VectorType>(Val: operand.get().getType());
774 targetType =
775 VectorType::get(shape: vecType.getShape(), elementType: operandType.getElementType());
776 } else {
777 auto operandType = operand.get().getType();
778 assert(!isa<VectorType>(operandType) &&
779 "unexpected yield of vector from op with scalar result type");
780 targetType = operandType;
781 }
782 retTypes.push_back(Elt: targetType);
783 yieldValues.push_back(Elt: operand.get());
784 }
785 SmallVector<size_t> newRetIndices;
786 WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
787 rewriter, warpOp, newYieldedValues: yieldValues, newReturnTypes: retTypes, indices&: newRetIndices);
788 rewriter.setInsertionPointAfter(newWarpOp);
789 SmallVector<Value> newOperands(elementWise->getOperands().begin(),
790 elementWise->getOperands().end());
791 for (unsigned i : llvm::seq(Begin: unsigned(0), End: elementWise->getNumOperands())) {
792 newOperands[i] = newWarpOp.getResult(i: newRetIndices[i]);
793 }
794 OpBuilder::InsertionGuard g(rewriter);
795 rewriter.setInsertionPointAfter(newWarpOp);
796 Operation *newOp = cloneOpWithOperandsAndTypes(
797 rewriter, loc, op: elementWise, operands: newOperands,
798 resultTypes: {newWarpOp.getResult(i: operandIndex).getType()});
799 rewriter.replaceAllUsesWith(from: newWarpOp.getResult(i: operandIndex),
800 to: newOp->getResult(idx: 0));
801 return success();
802 }
803};
804
805/// Sink out splat constant op feeding into a warp op yield.
806/// ```
807/// %0 = gpu.warp_execute_on_lane_0(%arg0) -> (vector<1xf32>) {
808/// ...
809/// %cst = arith.constant dense<2.0> : vector<32xf32>
810/// gpu.yield %cst : vector<32xf32>
811/// }
812/// ```
813/// To
814/// ```
815/// gpu.warp_execute_on_lane_0(%arg0 {
816/// ...
817/// }
818/// %0 = arith.constant dense<2.0> : vector<1xf32>
819struct WarpOpConstant : public WarpDistributionPattern {
820 using Base::Base;
821 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
822 PatternRewriter &rewriter) const override {
823 OpOperand *yieldOperand =
824 getWarpResult(warpOp, fn: llvm::IsaPred<arith::ConstantOp>);
825 if (!yieldOperand)
826 return failure();
827 auto constantOp = yieldOperand->get().getDefiningOp<arith::ConstantOp>();
828 auto dense = dyn_cast<SplatElementsAttr>(Val: constantOp.getValue());
829 if (!dense)
830 return failure();
831 // Notify the rewriter that the warp op is changing (see the comment on
832 // the WarpOpTransferRead pattern).
833 rewriter.startOpModification(op: warpOp);
834 unsigned operandIndex = yieldOperand->getOperandNumber();
835 Attribute scalarAttr = dense.getSplatValue<Attribute>();
836 auto newAttr = DenseElementsAttr::get(
837 type: cast<ShapedType>(Val: warpOp.getResult(i: operandIndex).getType()), values: scalarAttr);
838 Location loc = warpOp.getLoc();
839 rewriter.setInsertionPointAfter(warpOp);
840 Value distConstant = arith::ConstantOp::create(builder&: rewriter, location: loc, value: newAttr);
841 rewriter.replaceAllUsesWith(from: warpOp.getResult(i: operandIndex), to: distConstant);
842 rewriter.finalizeOpModification(op: warpOp);
843 return success();
844 }
845};
846
847/// Sink out step op feeding into a warp op yield.
848/// Vector step op is treated similar to arith.constant, apart from
849/// the result that represents a sequence [0, vec_size).
850/// Due to the to vec_size == warp_size limitation,
851/// we can simply wrap the lane id into a vector (i.e., broadcast).
852/// Supporting vec_size != warp_size may involve preserving the step
853/// result and using additional arith ops (the exact details are TBD).
854/// ```
855/// %0 = gpu.warp_execute_on_lane_0(%arg0) -> (vector<1xindex>) {
856/// ...
857/// %cst = vector.step : vector<32xindex>
858/// gpu.yield %cst : vector<1xindex>
859/// }
860/// ```
861/// To
862/// ```
863/// gpu.warp_execute_on_lane_0(%arg0) {
864/// ...
865/// }
866/// %lane_id_vec = vector.broadcast %arg0 : index to vector<1xindex>
867struct WarpOpStep final : public WarpDistributionPattern {
868 using Base::Base;
869 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
870 PatternRewriter &rewriter) const override {
871 OpOperand *yieldOperand =
872 getWarpResult(warpOp, fn: llvm::IsaPred<vector::StepOp>);
873 if (!yieldOperand)
874 return failure();
875 const unsigned operandIdx = yieldOperand->getOperandNumber();
876 auto stepOp = yieldOperand->get().getDefiningOp<vector::StepOp>();
877 VectorType resTy = stepOp.getResult().getType();
878 if (resTy.getNumElements() != static_cast<int64_t>(warpOp.getWarpSize()))
879 return rewriter.notifyMatchFailure(
880 arg&: warpOp,
881 msg: llvm::formatv(Fmt: "Expected result size ({0}) to be of warp size ({1})",
882 Vals: resTy.getNumElements(), Vals: warpOp.getWarpSize()));
883 VectorType newVecTy =
884 cast<VectorType>(Val: warpOp.getResult(i: operandIdx).getType());
885 rewriter.setInsertionPointAfter(warpOp);
886 Value laneIdVec = vector::BroadcastOp::create(builder&: rewriter, location: warpOp.getLoc(),
887 result: newVecTy, source: warpOp.getLaneid());
888 rewriter.replaceAllUsesWith(from: warpOp.getResult(i: operandIdx), to: laneIdVec);
889 return success();
890 }
891};
892
893/// Sink out transfer_read op feeding into a warp op yield.
894/// ```
895/// %0 = gpu.warp_execute_on_lane_0(%arg0) -> (vector<1xf32>) {
896/// ...
897// %2 = vector.transfer_read %src[%c0], %cst : memref<1024xf32>,
898// vector<32xf32>
899/// gpu.yield %2 : vector<32xf32>
900/// }
901/// ```
902/// To
903/// ```
904/// %dead = gpu.warp_execute_on_lane_0(%arg0) -> (vector<1xf32>,
905/// vector<1xf32>, vector<1xf32>) {
906/// ...
907/// %2 = vector.transfer_read %src[%c0], %cst : memref<1024xf32>,
908/// vector<32xf32> gpu.yield %2 : vector<32xf32>
909/// }
910/// %0 = vector.transfer_read %src[%c0], %cst : memref<1024xf32>, vector<1xf32>
911struct WarpOpTransferRead : public WarpDistributionPattern {
912 using Base::Base;
913 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
914 PatternRewriter &rewriter) const override {
915 // Try to find a distributable yielded read. Note that this pattern can
916 // still fail at the end after distribution, in which case this might have
917 // missed another distributable read.
918 OpOperand *operand = getWarpResult(warpOp, fn: [](Operation *op) {
919 // Don't duplicate transfer_read ops when distributing.
920 return isa<vector::TransferReadOp>(Val: op) && op->hasOneUse();
921 });
922 if (!operand)
923 return rewriter.notifyMatchFailure(
924 arg&: warpOp, msg: "warp result is not a vector.transfer_read op");
925 auto read = operand->get().getDefiningOp<vector::TransferReadOp>();
926
927 // Source must be defined outside of the region.
928 if (!warpOp.isDefinedOutsideOfRegion(value: read.getBase()))
929 return rewriter.notifyMatchFailure(
930 arg&: read, msg: "source must be defined outside of the region");
931
932 unsigned operandIndex = operand->getOperandNumber();
933 Value distributedVal = warpOp.getResult(i: operandIndex);
934
935 SmallVector<Value, 4> indices(read.getIndices().begin(),
936 read.getIndices().end());
937 auto sequentialType = cast<VectorType>(Val: read.getResult().getType());
938 auto distributedType = cast<VectorType>(Val: distributedVal.getType());
939 AffineMap map = calculateImplicitMap(sequentialType, distributedType);
940 AffineMap indexMap = map.compose(map: read.getPermutationMap());
941
942 // Try to delinearize the lane ID to match the rank expected for
943 // distribution.
944 SmallVector<Value> delinearizedIds;
945 if (!delinearizeLaneId(builder&: rewriter, loc: read.getLoc(), originalShape: sequentialType.getShape(),
946 distributedShape: distributedType.getShape(), warpSize: warpOp.getWarpSize(),
947 laneId: warpOp.getLaneid(), delinearizedIds)) {
948 return rewriter.notifyMatchFailure(
949 arg&: read, msg: "cannot delinearize lane ID for distribution");
950 }
951 assert(!delinearizedIds.empty() || map.getNumResults() == 0);
952
953 // Distribute indices and the mask (if present).
954 OpBuilder::InsertionGuard g(rewriter);
955 SmallVector<Value> additionalResults(indices.begin(), indices.end());
956 SmallVector<Type> additionalResultTypes(indices.size(),
957 rewriter.getIndexType());
958 additionalResults.push_back(Elt: read.getPadding());
959 additionalResultTypes.push_back(Elt: read.getPadding().getType());
960
961 bool hasMask = false;
962 if (read.getMask()) {
963 hasMask = true;
964 // TODO: Distribution of masked reads with non-trivial permutation maps
965 // requires the distribution of the mask to elementwise match the
966 // distribution of the permuted written vector. Currently the details
967 // of which lane is responsible for which element is captured strictly
968 // by shape information on the warp op, and thus requires materializing
969 // the permutation in IR.
970 if (!mlir::compressUnusedDims(map: read.getPermutationMap()).isIdentity())
971 return rewriter.notifyMatchFailure(
972 arg&: read, msg: "non-trivial permutation maps not supported");
973 VectorType maskType =
974 getDistributedType(originalType: read.getMaskType(), map, warpSize: warpOp.getWarpSize());
975 additionalResults.push_back(Elt: read.getMask());
976 additionalResultTypes.push_back(Elt: maskType);
977 }
978
979 SmallVector<size_t> newRetIndices;
980 WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
981 rewriter, warpOp, newYieldedValues: additionalResults, newReturnTypes: additionalResultTypes,
982 indices&: newRetIndices);
983 distributedVal = newWarpOp.getResult(i: operandIndex);
984
985 // Distributed indices were appended first.
986 SmallVector<Value> newIndices;
987 for (int64_t i = 0, e = indices.size(); i < e; ++i)
988 newIndices.push_back(Elt: newWarpOp.getResult(i: newRetIndices[i]));
989
990 rewriter.setInsertionPointAfter(newWarpOp);
991 for (auto it : llvm::zip_equal(t: indexMap.getResults(), u: map.getResults())) {
992 AffineExpr d0, d1;
993 bindDims(ctx: read.getContext(), exprs&: d0, exprs&: d1);
994 auto indexExpr = dyn_cast<AffineDimExpr>(Val: std::get<0>(t&: it));
995 if (!indexExpr)
996 continue;
997 unsigned indexPos = indexExpr.getPosition();
998 unsigned vectorPos = cast<AffineDimExpr>(Val: std::get<1>(t&: it)).getPosition();
999 int64_t scale = distributedType.getDimSize(idx: vectorPos);
1000 newIndices[indexPos] = affine::makeComposedAffineApply(
1001 b&: rewriter, loc: read.getLoc(), e: d0 + scale * d1,
1002 operands: {newIndices[indexPos], delinearizedIds[vectorPos]});
1003 }
1004
1005 // Distributed padding value was appended right after the indices.
1006 Value newPadding = newWarpOp.getResult(i: newRetIndices[indices.size()]);
1007 // Distributed mask value was added at the end (if the op has a mask).
1008 Value newMask =
1009 hasMask ? newWarpOp.getResult(i: newRetIndices[newRetIndices.size() - 1])
1010 : Value();
1011 auto newRead = vector::TransferReadOp::create(
1012 builder&: rewriter, location: read.getLoc(), result: distributedVal.getType(), base: read.getBase(),
1013 indices: newIndices, permutation_map: read.getPermutationMapAttr(), padding: newPadding, mask: newMask,
1014 in_bounds: read.getInBoundsAttr());
1015
1016 rewriter.replaceAllUsesWith(from: distributedVal, to: newRead);
1017 return success();
1018 }
1019};
1020
1021/// Remove any result that has no use along with the matching yieldOp operand.
1022// TODO: Move this in WarpExecuteOnLane0Op canonicalization.
1023struct WarpOpDeadResult : public WarpDistributionPattern {
1024 using Base::Base;
1025 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
1026 PatternRewriter &rewriter) const override {
1027 SmallVector<Type> newResultTypes;
1028 newResultTypes.reserve(N: warpOp->getNumResults());
1029 SmallVector<Value> newYieldValues;
1030 newYieldValues.reserve(N: warpOp->getNumResults());
1031 DenseMap<Value, int64_t> dedupYieldOperandPositionMap;
1032 DenseMap<OpResult, int64_t> dedupResultPositionMap;
1033 gpu::YieldOp yield = warpOp.getTerminator();
1034
1035 // Some values may be yielded multiple times and correspond to multiple
1036 // results. Deduplicating occurs by taking each result with its matching
1037 // yielded value, and:
1038 // 1. recording the unique first position at which the value with uses is
1039 // yielded.
1040 // 2. recording for the result, the first position at which the dedup'ed
1041 // value is yielded.
1042 // 3. skipping from the new result types / new yielded values any result
1043 // that has no use or whose yielded value has already been seen.
1044 for (OpResult result : warpOp.getResults()) {
1045 if (result.use_empty())
1046 continue;
1047 Value yieldOperand = yield.getOperand(i: result.getResultNumber());
1048 auto it = dedupYieldOperandPositionMap.insert(
1049 KV: std::make_pair(x&: yieldOperand, y: newResultTypes.size()));
1050 dedupResultPositionMap.insert(KV: std::make_pair(x&: result, y&: it.first->second));
1051 if (!it.second)
1052 continue;
1053 newResultTypes.push_back(Elt: result.getType());
1054 newYieldValues.push_back(Elt: yieldOperand);
1055 }
1056 // No modification, exit early.
1057 if (yield.getNumOperands() == newYieldValues.size())
1058 return failure();
1059 // Move the body of the old warpOp to a new warpOp.
1060 WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndReplaceReturns(
1061 rewriter, warpOp, newYieldedValues: newYieldValues, newReturnTypes: newResultTypes);
1062
1063 // Simplify the new warp op after dropping dead results.
1064 newWarpOp.getBody()->walk(callback: [&](Operation *op) {
1065 if (isOpTriviallyDead(op))
1066 rewriter.eraseOp(op);
1067 });
1068
1069 // Replace results of the old warpOp by the new, deduplicated results.
1070 SmallVector<Value> newValues;
1071 newValues.reserve(N: warpOp->getNumResults());
1072 for (OpResult result : warpOp.getResults()) {
1073 if (result.use_empty())
1074 newValues.push_back(Elt: Value());
1075 else
1076 newValues.push_back(
1077 Elt: newWarpOp.getResult(i: dedupResultPositionMap.lookup(Val: result)));
1078 }
1079 rewriter.replaceOp(op: warpOp, newValues);
1080 return success();
1081 }
1082};
1083
1084// If an operand is directly yielded out of the region we can forward it
1085// directly and it doesn't need to go through the region.
1086struct WarpOpForwardOperand : public WarpDistributionPattern {
1087 using Base::Base;
1088 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
1089 PatternRewriter &rewriter) const override {
1090 gpu::YieldOp yield = warpOp.getTerminator();
1091 Value valForwarded;
1092 unsigned resultIndex;
1093 for (OpOperand &operand : yield->getOpOperands()) {
1094 Value result = warpOp.getResult(i: operand.getOperandNumber());
1095 if (result.use_empty())
1096 continue;
1097
1098 // Assume all the values coming from above are uniform.
1099 if (!warpOp.getBodyRegion().isAncestor(other: operand.get().getParentRegion())) {
1100 if (result.getType() != operand.get().getType())
1101 continue;
1102 valForwarded = operand.get();
1103 resultIndex = operand.getOperandNumber();
1104 break;
1105 }
1106 auto arg = dyn_cast<BlockArgument>(Val: operand.get());
1107 if (!arg || arg.getOwner()->getParentOp() != warpOp.getOperation())
1108 continue;
1109 Value warpOperand = warpOp.getArgs()[arg.getArgNumber()];
1110 if (result.getType() != warpOperand.getType())
1111 continue;
1112 valForwarded = warpOperand;
1113 resultIndex = operand.getOperandNumber();
1114 break;
1115 }
1116 if (!valForwarded)
1117 return failure();
1118 // Notify the rewriter that the warp op is changing (see the comment on
1119 // the WarpOpTransferRead pattern).
1120 rewriter.startOpModification(op: warpOp);
1121 rewriter.replaceAllUsesWith(from: warpOp.getResult(i: resultIndex), to: valForwarded);
1122 rewriter.finalizeOpModification(op: warpOp);
1123 return success();
1124 }
1125};
1126
1127struct WarpOpBroadcast : public WarpDistributionPattern {
1128 using Base::Base;
1129 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
1130 PatternRewriter &rewriter) const override {
1131 OpOperand *operand =
1132 getWarpResult(warpOp, fn: llvm::IsaPred<vector::BroadcastOp>);
1133 if (!operand)
1134 return failure();
1135 unsigned int operandNumber = operand->getOperandNumber();
1136 auto broadcastOp = operand->get().getDefiningOp<vector::BroadcastOp>();
1137 Location loc = broadcastOp.getLoc();
1138 auto destVecType =
1139 cast<VectorType>(Val: warpOp->getResultTypes()[operandNumber]);
1140 Value broadcastSrc = broadcastOp.getSource();
1141 Type broadcastSrcType = broadcastSrc.getType();
1142
1143 // Check that the broadcast actually spans a set of values uniformly across
1144 // all threads. In other words, check that each thread can reconstruct
1145 // their own broadcast.
1146 // For that we simply check that the broadcast we want to build makes sense.
1147 if (vector::isBroadcastableTo(srcType: broadcastSrcType, dstVectorType: destVecType) !=
1148 vector::BroadcastableToResult::Success)
1149 return failure();
1150 SmallVector<size_t> newRetIndices;
1151 WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
1152 rewriter, warpOp, newYieldedValues: {broadcastSrc}, newReturnTypes: {broadcastSrcType}, indices&: newRetIndices);
1153 rewriter.setInsertionPointAfter(newWarpOp);
1154 Value broadcasted = vector::BroadcastOp::create(
1155 builder&: rewriter, location: loc, result: destVecType, source: newWarpOp->getResult(idx: newRetIndices[0]));
1156 rewriter.replaceAllUsesWith(from: newWarpOp->getResult(idx: operandNumber),
1157 to: broadcasted);
1158 return success();
1159 }
1160};
1161
1162/// Pattern to move shape cast out of the warp op. shape cast is basically a
1163/// no-op for warp distribution; we need to handle the shape though.
1164struct WarpOpShapeCast : public WarpDistributionPattern {
1165 using Base::Base;
1166 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
1167 PatternRewriter &rewriter) const override {
1168 OpOperand *operand =
1169 getWarpResult(warpOp, fn: llvm::IsaPred<vector::ShapeCastOp>);
1170 if (!operand)
1171 return failure();
1172
1173 auto oldCastOp = operand->get().getDefiningOp<vector::ShapeCastOp>();
1174
1175 unsigned int operandNumber = operand->getOperandNumber();
1176 auto castDistributedType =
1177 cast<VectorType>(Val: warpOp->getResultTypes()[operandNumber]);
1178 VectorType castOriginalType = oldCastOp.getSourceVectorType();
1179 VectorType castResultType = castDistributedType;
1180
1181 FailureOr<VectorType> maybeSrcType =
1182 inferDistributedSrcType(distributedType: castDistributedType, srcType: castOriginalType);
1183 if (failed(Result: maybeSrcType))
1184 return failure();
1185 castDistributedType = *maybeSrcType;
1186
1187 SmallVector<size_t> newRetIndices;
1188 WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
1189 rewriter, warpOp, newYieldedValues: {oldCastOp.getSource()}, newReturnTypes: {castDistributedType},
1190 indices&: newRetIndices);
1191 rewriter.setInsertionPointAfter(newWarpOp);
1192 Value newCast = vector::ShapeCastOp::create(
1193 builder&: rewriter, location: oldCastOp.getLoc(), result: castResultType,
1194 source: newWarpOp->getResult(idx: newRetIndices[0]));
1195 rewriter.replaceAllUsesWith(from: newWarpOp->getResult(idx: operandNumber), to: newCast);
1196 return success();
1197 }
1198
1199private:
1200 static FailureOr<VectorType>
1201 inferDistributedSrcType(VectorType distributedType, VectorType srcType) {
1202 unsigned distributedRank = distributedType.getRank();
1203 unsigned srcRank = srcType.getRank();
1204 if (distributedRank == srcRank)
1205 // Nothing to do.
1206 return distributedType;
1207 if (distributedRank < srcRank) {
1208 // If the distributed type has a smaller rank than the original type,
1209 // prepend with unit dimensions to make the types the same length.
1210 SmallVector<int64_t> shape(srcRank - distributedRank, 1);
1211 llvm::append_range(C&: shape, R: distributedType.getShape());
1212 return VectorType::get(shape, elementType: distributedType.getElementType());
1213 }
1214 // Handle the expanding shape_cast's.
1215 //
1216 // If the casted-from type has one rank, we can assert that the element
1217 // count in that rank will match the full thread-level element count of
1218 // the yielded type.
1219 // Note that getNumElements() will correctly "flatten" the shape of the
1220 // specific shape_cast's distributed type (its distribution may be
1221 // different from the overall warp size, e.g. if the cast is applied to
1222 // a result of a gather).
1223 if (srcRank == 1)
1224 return VectorType::get(shape: distributedType.getNumElements(),
1225 elementType: srcType.getElementType());
1226 // Try to strip leading unit dimensions to match the ranks. We bail out
1227 // for more complex tile sizes, because those would require us to
1228 // determine the specific distribution parameters to threads, which is
1229 // unfeasible within this pattern.
1230 unsigned excessDims = distributedRank - srcRank;
1231 ArrayRef<int64_t> shape = distributedType.getShape();
1232 if (!llvm::all_of(Range: shape.take_front(N: excessDims),
1233 P: [](int64_t d) { return d == 1; }))
1234 return failure();
1235 return VectorType::get(shape: shape.drop_front(N: excessDims),
1236 elementType: distributedType.getElementType());
1237 }
1238};
1239
1240/// Sink out vector.create_mask / vector.constant_mask op feeding into a warp op
1241/// yield.
1242/// ```
1243/// %0 = ...
1244/// %1 = gpu.warp_execute_on_lane_0(%arg0) -> (vector<1xf32>) {
1245/// ...
1246/// %mask = vector.create_mask %0 : vector<32xi1>
1247/// // or %mask = vector.constant_mask[2] : vector<32xi1>
1248/// gpu.yield %mask : vector<32xi1>
1249/// }
1250/// ```
1251/// To
1252/// ```
1253/// %0 = ...
1254/// gpu.warp_execute_on_lane_0(%arg0) {
1255/// ...
1256/// }
1257/// %cmp = arith.cmpi ult, %laneid, %0
1258/// %ub = arith.select %cmp, %c0, %c1
1259/// %1 = vector.create_mask %ub : vector<1xi1>
1260template <typename OpType,
1261 typename = std::enable_if_t<llvm::is_one_of<
1262 OpType, vector::CreateMaskOp, vector::ConstantMaskOp>::value>>
1263struct WarpOpCreateMask : public WarpDistributionPattern {
1264 using Base::Base;
1265 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
1266 PatternRewriter &rewriter) const override {
1267 OpOperand *yieldOperand = getWarpResult(warpOp, fn: (llvm::IsaPred<OpType>));
1268 if (!yieldOperand)
1269 return failure();
1270
1271 Operation *mask = yieldOperand->get().getDefiningOp<OpType>();
1272
1273 // Early exit if any values needed for calculating the new mask indices
1274 // are defined inside the warp op.
1275 if (mask->getOperands().size() &&
1276 !llvm::all_of(mask->getOperands(), [&](Value value) {
1277 return warpOp.isDefinedOutsideOfRegion(value);
1278 }))
1279 return failure();
1280
1281 Location loc = mask->getLoc();
1282 unsigned operandIndex = yieldOperand->getOperandNumber();
1283
1284 auto distType = cast<VectorType>(Val: warpOp.getResult(i: operandIndex).getType());
1285 VectorType seqType = cast<VectorType>(Val: mask->getResult(idx: 0).getType());
1286 ArrayRef<int64_t> seqShape = seqType.getShape();
1287 ArrayRef<int64_t> distShape = distType.getShape();
1288 SmallVector<Value> materializedOperands;
1289 if constexpr (std::is_same_v<OpType, vector::CreateMaskOp>) {
1290 materializedOperands.append(in_start: mask->getOperands().begin(),
1291 in_end: mask->getOperands().end());
1292 } else {
1293 auto constantMaskOp = cast<vector::ConstantMaskOp>(Val: mask);
1294 auto dimSizes = constantMaskOp.getMaskDimSizesAttr().asArrayRef();
1295 for (auto dimSize : dimSizes)
1296 materializedOperands.push_back(
1297 Elt: arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: dimSize).getResult());
1298 }
1299
1300 rewriter.setInsertionPointAfter(warpOp);
1301
1302 // Delinearize the lane ID for constructing the distributed mask sizes.
1303 SmallVector<Value> delinearizedIds;
1304 if (!delinearizeLaneId(builder&: rewriter, loc, originalShape: seqShape, distributedShape: distShape,
1305 warpSize: warpOp.getWarpSize(), laneId: warpOp.getLaneid(),
1306 delinearizedIds))
1307 return rewriter.notifyMatchFailure(
1308 arg&: mask, msg: "cannot delinearize lane ID for distribution");
1309 assert(!delinearizedIds.empty());
1310
1311 // Notify the rewriter that the warp op is changing (see the comment on
1312 // the WarpOpTransferRead pattern).
1313 rewriter.startOpModification(op: warpOp);
1314
1315 AffineExpr s0, s1;
1316 bindSymbols(ctx: rewriter.getContext(), exprs&: s0, exprs&: s1);
1317 SmallVector<Value> newOperands;
1318 for (int i = 0, e = distShape.size(); i < e; ++i) {
1319 // Get `mask_dim_range_upper_limit[i] - lane_id[i] * dist_sizes[i]` to
1320 // find the distance from the largest mask index owned by this lane to the
1321 // original mask size. `vector.create_mask` implicitly clamps mask
1322 // operands to the range [0, mask_vector_size[i]], or in other words, the
1323 // mask sizes are always in the range [0, mask_vector_size[i]).
1324 Value maskDimIdx = affine::makeComposedAffineApply(
1325 b&: rewriter, loc, e: s1 - s0 * distShape[i],
1326 operands: {delinearizedIds[i], materializedOperands[i]});
1327 newOperands.push_back(Elt: maskDimIdx);
1328 }
1329
1330 auto newMask =
1331 vector::CreateMaskOp::create(builder&: rewriter, location: loc, resultType0: distType, mask_dim_sizes: newOperands);
1332 rewriter.replaceAllUsesWith(from: warpOp.getResult(i: operandIndex), to: newMask);
1333 rewriter.finalizeOpModification(op: warpOp);
1334 return success();
1335 }
1336};
1337
1338/// Sink out insert_strided_slice op feeding into a warp op yield.
1339/// ```
1340/// %0 = gpu.warp_execute_on_lane_0(%arg0) -> (vector<8x1xf32>) {
1341/// ...
1342/// %src = ... : vector<4x32xf32>
1343/// %dest = ... : vector<8x32xf32>
1344/// %insert = vector.insert_strided_slice %src, %dest, offsets = [0, 0],
1345/// strides = [1, 1] : vector<4x32xf32> into vector<8x32xf32>
1346/// gpu.yield %insert : vector<8x32xf32>
1347/// }
1348/// ```
1349/// To
1350/// ```
1351/// %0 = gpu.warp_execute_on_lane_0(%arg0) -> (vector<4x1xf32>,
1352/// vector<8x1xf32>) {
1353/// ...
1354/// %src = ... : vector<4x32xf32>
1355/// %dest = ... : vector<8x32xf32>
1356/// gpu.yield %src, %dest : vector<4x16xf32>, vector<8x16xf32>
1357/// }
1358/// %insert = vector.insert_strided_slice %0#0, %0#1,
1359/// offsets = [0, 0], strides = [1, 1] : vector<4x1xf32> into vector<8x1xf32>
1360/// ```
1361/// NOTE: Current support assumes that both src and dest vectors are distributed
1362/// to lanes and sinking the insert op does not require any cross lane
1363/// communication.
1364struct WarpOpInsertStridedSlice : public WarpDistributionPattern {
1365 using Base::Base;
1366 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
1367 PatternRewriter &rewriter) const override {
1368 OpOperand *operand =
1369 getWarpResult(warpOp, fn: llvm::IsaPred<vector::InsertStridedSliceOp>);
1370 if (!operand)
1371 return failure();
1372 unsigned int operandNumber = operand->getOperandNumber();
1373 auto insertOp =
1374 operand->get().getDefiningOp<vector::InsertStridedSliceOp>();
1375 auto distributedType =
1376 cast<VectorType>(Val: warpOp.getResult(i: operandNumber).getType());
1377 // Distributed type must be 2D or higher.
1378 // TODO: Support 1D distributed types.
1379 if (distributedType.getRank() < 2)
1380 return rewriter.notifyMatchFailure(
1381 arg&: insertOp, msg: "result vector type must be 2D or higher");
1382 // Find the distributed dimension of the dest vector. There should be
1383 // exactly one.
1384 auto yieldedType = cast<VectorType>(Val: operand->get().getType());
1385 int64_t destDistributedDim =
1386 getDistributedDim(sequentialType: yieldedType, distributedType);
1387 assert(destDistributedDim != -1 && "could not find distributed dimension");
1388
1389 VectorType srcType = insertOp.getSourceVectorType();
1390 VectorType destType = insertOp.getDestVectorType();
1391 // Currently we require that both source (kD) and dest (nD) vectors are
1392 // distributed. This requires that distributedDim (d) is contained in the
1393 // last k dims of the dest vector (d >= n - k).
1394 // TODO: Add support for case where source vector is not distributed.
1395 int64_t sourceDistributedDim =
1396 destDistributedDim - (destType.getRank() - srcType.getRank());
1397 if (sourceDistributedDim < 0)
1398 return rewriter.notifyMatchFailure(
1399 arg&: insertOp,
1400 msg: "distributed dimension must be in the last k dims of dest vector");
1401 // Distributed dimension must be fully inserted.
1402 if (srcType.getDimSize(idx: sourceDistributedDim) !=
1403 destType.getDimSize(idx: destDistributedDim))
1404 return rewriter.notifyMatchFailure(
1405 arg&: insertOp, msg: "distributed dimension must be fully inserted");
1406 SmallVector<int64_t> newSourceDistShape(
1407 insertOp.getSourceVectorType().getShape());
1408 newSourceDistShape[sourceDistributedDim] =
1409 distributedType.getDimSize(idx: destDistributedDim);
1410 auto newSourceTy =
1411 VectorType::get(shape: newSourceDistShape, elementType: distributedType.getElementType());
1412 VectorType newDestTy = distributedType;
1413 SmallVector<size_t> newRetIndices;
1414 WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
1415 rewriter, warpOp, newYieldedValues: {insertOp.getValueToStore(), insertOp.getDest()},
1416 newReturnTypes: {newSourceTy, newDestTy}, indices&: newRetIndices);
1417 rewriter.setInsertionPointAfter(newWarpOp);
1418 Value distributedSource = newWarpOp->getResult(idx: newRetIndices[0]);
1419 Value distributedDest = newWarpOp->getResult(idx: newRetIndices[1]);
1420 // Create a new insert strided slice op that inserts distributed source into
1421 // distributed dest.
1422 Value newInsert = vector::InsertStridedSliceOp::create(
1423 builder&: rewriter, location: insertOp.getLoc(), result: distributedDest.getType(),
1424 valueToStore: distributedSource, dest: distributedDest, offsets: insertOp.getOffsets(),
1425 strides: insertOp.getStrides());
1426 rewriter.replaceAllUsesWith(from: newWarpOp->getResult(idx: operandNumber), to: newInsert);
1427 return success();
1428 }
1429};
1430
1431/// Sink out extract_strided_slice op feeding into a warp op yield.
1432/// ```
1433/// %0 = gpu.warp_execute_on_lane_0(%arg0) -> (vector<16x1xf32>) {
1434/// ...
1435/// %src = ... : vector<64x32xf32>
1436/// %extract = vector.extract_strided_slice %src, offsets = [0], sizes = [16],
1437/// strides = [1] : vector<64x32xf32> to vector<16x32xf32>
1438/// gpu.yield %extract : vector<16x32xf32>
1439/// }
1440/// ```
1441/// To
1442/// ```
1443/// %0 = gpu.warp_execute_on_lane_0(%arg0) -> (vector<64x1xf32>) {
1444/// ...
1445/// %src = ... : vector<64x32xf32>
1446/// gpu.yield %src : vector<64x32xf32>
1447/// }
1448/// %extract = vector.extract_strided_slice %0, offsets = [0], sizes = [16],
1449/// strides = [1] : vector<64x1xf32> to vector<16x1xf32>
1450/// ```
1451/// NOTE: Current support assumes that the extraction happens only on non
1452/// distributed dimensions (does not require cross lane communication).
1453struct WarpOpExtractStridedSlice : public WarpDistributionPattern {
1454 using Base::Base;
1455 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
1456 PatternRewriter &rewriter) const override {
1457 OpOperand *operand =
1458 getWarpResult(warpOp, fn: llvm::IsaPred<vector::ExtractStridedSliceOp>);
1459 if (!operand)
1460 return failure();
1461 unsigned int operandNumber = operand->getOperandNumber();
1462 auto extractOp =
1463 operand->get().getDefiningOp<vector::ExtractStridedSliceOp>();
1464 auto distributedType =
1465 cast<VectorType>(Val: warpOp.getResult(i: operandNumber).getType());
1466 // Distributed type must be 2D or higher.
1467 // TODO: Support 1D distributed types.
1468 if (distributedType.getRank() < 2)
1469 return rewriter.notifyMatchFailure(
1470 arg&: extractOp, msg: "result vector type must be 2D or higher");
1471
1472 // Find the distributed dimension. There should be exactly one.
1473 auto yieldedType = cast<VectorType>(Val: operand->get().getType());
1474 int64_t distributedDim = getDistributedDim(sequentialType: yieldedType, distributedType);
1475 assert(distributedDim != -1 && "could not find distributed dimension");
1476
1477 int64_t numOfExtractedDims =
1478 static_cast<int64_t>(extractOp.getSizes().size());
1479 // If the distributed dim is included in the extracted dims, then we make
1480 // sure distributed dim is fully extracted. If distributed dim is not
1481 // included in extracted dims, it is guaranteed to be fully extracted (i.e.
1482 // distributed dim comes after all the extracted dims)
1483 // TODO: Partial extraction from distributed dimension require cross lane
1484 // communication.
1485 if (distributedDim < numOfExtractedDims) {
1486 int64_t distributedDimOffset =
1487 llvm::cast<IntegerAttr>(Val: extractOp.getOffsets()[distributedDim])
1488 .getInt();
1489 int64_t distributedDimSize =
1490 llvm::cast<IntegerAttr>(Val: extractOp.getSizes()[distributedDim])
1491 .getInt();
1492 if (distributedDimOffset != 0 ||
1493 distributedDimSize != yieldedType.getDimSize(idx: distributedDim))
1494 return rewriter.notifyMatchFailure(
1495 arg&: extractOp, msg: "distributed dimension must be fully extracted");
1496 }
1497 SmallVector<int64_t> newDistributedShape(
1498 extractOp.getSourceVectorType().getShape());
1499 newDistributedShape[distributedDim] =
1500 distributedType.getDimSize(idx: distributedDim);
1501 auto newDistributedType =
1502 VectorType::get(shape: newDistributedShape, elementType: distributedType.getElementType());
1503 SmallVector<size_t> newRetIndices;
1504 WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
1505 rewriter, warpOp, newYieldedValues: {extractOp.getSource()}, newReturnTypes: {newDistributedType},
1506 indices&: newRetIndices);
1507 rewriter.setInsertionPointAfter(newWarpOp);
1508 SmallVector<Attribute> distributedSizes = llvm::map_to_vector(
1509 C: extractOp.getSizes(), F: [](Attribute attr) { return attr; });
1510 // Update the distributed sizes to match the distributed type.
1511 if (distributedDim < static_cast<int64_t>(distributedSizes.size()))
1512 distributedSizes[distributedDim] = rewriter.getI64IntegerAttr(
1513 value: distributedType.getDimSize(idx: distributedDim));
1514
1515 // Create a new extract strided slice op that extracts from the
1516 // distributed vector.
1517 Value distributedVec = newWarpOp->getResult(idx: newRetIndices[0]);
1518 Value newExtract = vector::ExtractStridedSliceOp::create(
1519 builder&: rewriter, location: extractOp.getLoc(), resultType0: distributedType, source: distributedVec,
1520 offsets: extractOp.getOffsets(),
1521 sizes: ArrayAttr::get(context: rewriter.getContext(), value: distributedSizes),
1522 strides: extractOp.getStrides());
1523 rewriter.replaceAllUsesWith(from: newWarpOp->getResult(idx: operandNumber),
1524 to: newExtract);
1525 return success();
1526 }
1527};
1528
1529/// Pattern to move out vector.extract of single element vector. Those don't
1530/// need to be distributed and can just be propagated outside of the region.
1531struct WarpOpExtract : public WarpDistributionPattern {
1532 using Base::Base;
1533 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
1534 PatternRewriter &rewriter) const override {
1535 OpOperand *operand =
1536 getWarpResult(warpOp, fn: llvm::IsaPred<vector::ExtractOp>);
1537 if (!operand)
1538 return failure();
1539 unsigned int operandNumber = operand->getOperandNumber();
1540 auto extractOp = operand->get().getDefiningOp<vector::ExtractOp>();
1541 VectorType extractSrcType = extractOp.getSourceVectorType();
1542 Location loc = extractOp.getLoc();
1543
1544 // For 1-d or 0-d source cases, we rely on WarpOpExtractScalar pattern.
1545 if (extractSrcType.getRank() <= 1) {
1546 return failure();
1547 }
1548
1549 // All following cases are 2d or higher dimensional source vectors.
1550
1551 if (warpOp.getResult(i: operandNumber).getType() == operand->get().getType()) {
1552 // There is no distribution, this is a broadcast. Simply move the extract
1553 // out of the warp op.
1554 // TODO: This could be optimized. E.g., in case of a scalar result, let
1555 // one lane extract and shuffle the result to all other lanes (same as
1556 // the 1d case).
1557 SmallVector<size_t> newRetIndices;
1558 WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
1559 rewriter, warpOp, newYieldedValues: {extractOp.getSource()},
1560 newReturnTypes: {extractOp.getSourceVectorType()}, indices&: newRetIndices);
1561 rewriter.setInsertionPointAfter(newWarpOp);
1562 Value distributedVec = newWarpOp->getResult(idx: newRetIndices[0]);
1563 // Extract from distributed vector.
1564 Value newExtract = vector::ExtractOp::create(
1565 builder&: rewriter, location: loc, source: distributedVec, position: extractOp.getMixedPosition());
1566 rewriter.replaceAllUsesWith(from: newWarpOp->getResult(idx: operandNumber),
1567 to: newExtract);
1568 return success();
1569 }
1570
1571 // Find the distributed dimension. There should be exactly one.
1572 auto distributedType =
1573 cast<VectorType>(Val: warpOp.getResult(i: operandNumber).getType());
1574 auto yieldedType = cast<VectorType>(Val: operand->get().getType());
1575 int64_t distributedDim = getDistributedDim(sequentialType: yieldedType, distributedType);
1576 assert(distributedDim != -1 && "could not find distributed dimension");
1577 (void)distributedDim;
1578
1579 // Yield source vector from warp op.
1580 SmallVector<int64_t> newDistributedShape(extractSrcType.getShape());
1581 for (int i = 0; i < distributedType.getRank(); ++i)
1582 newDistributedShape[i + extractOp.getNumIndices()] =
1583 distributedType.getDimSize(idx: i);
1584 auto newDistributedType =
1585 VectorType::get(shape: newDistributedShape, elementType: distributedType.getElementType());
1586 SmallVector<size_t> newRetIndices;
1587 WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
1588 rewriter, warpOp, newYieldedValues: {extractOp.getSource()}, newReturnTypes: {newDistributedType},
1589 indices&: newRetIndices);
1590 rewriter.setInsertionPointAfter(newWarpOp);
1591 Value distributedVec = newWarpOp->getResult(idx: newRetIndices[0]);
1592 // Extract from distributed vector.
1593 Value newExtract = vector::ExtractOp::create(builder&: rewriter, location: loc, source: distributedVec,
1594 position: extractOp.getMixedPosition());
1595 rewriter.replaceAllUsesWith(from: newWarpOp->getResult(idx: operandNumber),
1596 to: newExtract);
1597 return success();
1598 }
1599};
1600
1601/// Pattern to move out vector.extract with a scalar result.
1602/// Only supports 1-D and 0-D sources for now.
1603struct WarpOpExtractScalar : public WarpDistributionPattern {
1604 WarpOpExtractScalar(MLIRContext *ctx, WarpShuffleFromIdxFn fn,
1605 PatternBenefit b = 1)
1606 : WarpDistributionPattern(ctx, b), warpShuffleFromIdxFn(std::move(fn)) {}
1607 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
1608 PatternRewriter &rewriter) const override {
1609 OpOperand *operand =
1610 getWarpResult(warpOp, fn: llvm::IsaPred<vector::ExtractOp>);
1611 if (!operand)
1612 return failure();
1613 unsigned int operandNumber = operand->getOperandNumber();
1614 auto extractOp = operand->get().getDefiningOp<vector::ExtractOp>();
1615 VectorType extractSrcType = extractOp.getSourceVectorType();
1616 // Only supports 1-D or 0-D sources for now.
1617 if (extractSrcType.getRank() > 1) {
1618 return rewriter.notifyMatchFailure(
1619 arg&: extractOp, msg: "only 0-D or 1-D source supported for now");
1620 }
1621 // TODO: Supported shuffle types should be parameterizable, similar to
1622 // `WarpShuffleFromIdxFn`.
1623 if (!extractSrcType.getElementType().isF32() &&
1624 !extractSrcType.getElementType().isInteger(width: 32))
1625 return rewriter.notifyMatchFailure(
1626 arg&: extractOp, msg: "only f32/i32 element types are supported");
1627 bool is0dOrVec1Extract = extractSrcType.getNumElements() == 1;
1628 Type elType = extractSrcType.getElementType();
1629 VectorType distributedVecType;
1630 if (!is0dOrVec1Extract) {
1631 assert(extractSrcType.getRank() == 1 &&
1632 "expected that extract src rank is 0 or 1");
1633 if (extractSrcType.getShape()[0] % warpOp.getWarpSize() != 0)
1634 return failure();
1635 int64_t elementsPerLane =
1636 extractSrcType.getShape()[0] / warpOp.getWarpSize();
1637 distributedVecType = VectorType::get(shape: {elementsPerLane}, elementType: elType);
1638 } else {
1639 distributedVecType = extractSrcType;
1640 }
1641 // Yield source vector and position (if present) from warp op.
1642 SmallVector<Value> additionalResults{extractOp.getSource()};
1643 SmallVector<Type> additionalResultTypes{distributedVecType};
1644 additionalResults.append(
1645 RHS: SmallVector<Value>(extractOp.getDynamicPosition()));
1646 additionalResultTypes.append(
1647 RHS: SmallVector<Type>(extractOp.getDynamicPosition().getTypes()));
1648
1649 Location loc = extractOp.getLoc();
1650 SmallVector<size_t> newRetIndices;
1651 WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
1652 rewriter, warpOp, newYieldedValues: additionalResults, newReturnTypes: additionalResultTypes,
1653 indices&: newRetIndices);
1654 rewriter.setInsertionPointAfter(newWarpOp);
1655 Value distributedVec = newWarpOp->getResult(idx: newRetIndices[0]);
1656
1657 // 0d extract: The new warp op broadcasts the source vector to all lanes.
1658 // All lanes extract the scalar.
1659 if (is0dOrVec1Extract) {
1660 Value newExtract;
1661 SmallVector<int64_t> indices(extractSrcType.getRank(), 0);
1662 newExtract =
1663 vector::ExtractOp::create(builder&: rewriter, location: loc, source: distributedVec, position: indices);
1664 rewriter.replaceAllUsesWith(from: newWarpOp->getResult(idx: operandNumber),
1665 to: newExtract);
1666 return success();
1667 }
1668
1669 int64_t staticPos = extractOp.getStaticPosition()[0];
1670 OpFoldResult pos = ShapedType::isDynamic(dValue: staticPos)
1671 ? (newWarpOp->getResult(idx: newRetIndices[1]))
1672 : OpFoldResult(rewriter.getIndexAttr(value: staticPos));
1673 // 1d extract: Distribute the source vector. One lane extracts and shuffles
1674 // the value to all other lanes.
1675 int64_t elementsPerLane = distributedVecType.getShape()[0];
1676 AffineExpr sym0 = getAffineSymbolExpr(position: 0, context: rewriter.getContext());
1677 // tid of extracting thread: pos / elementsPerLane
1678 Value broadcastFromTid = affine::makeComposedAffineApply(
1679 b&: rewriter, loc, e: sym0.ceilDiv(v: elementsPerLane), operands: pos);
1680 // Extract at position: pos % elementsPerLane
1681 Value newPos =
1682 elementsPerLane == 1
1683 ? arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0).getResult()
1684 : affine::makeComposedAffineApply(b&: rewriter, loc,
1685 e: sym0 % elementsPerLane, operands: pos);
1686 Value extracted =
1687 vector::ExtractOp::create(builder&: rewriter, location: loc, source: distributedVec, position: newPos);
1688
1689 // Shuffle the extracted value to all lanes.
1690 Value shuffled = warpShuffleFromIdxFn(
1691 loc, rewriter, extracted, broadcastFromTid, newWarpOp.getWarpSize());
1692 rewriter.replaceAllUsesWith(from: newWarpOp->getResult(idx: operandNumber), to: shuffled);
1693 return success();
1694 }
1695
1696private:
1697 WarpShuffleFromIdxFn warpShuffleFromIdxFn;
1698};
1699
1700/// Pattern to move out vector.insert with a scalar input.
1701/// Only supports 1-D and 0-D destinations for now.
1702struct WarpOpInsertScalar : public WarpDistributionPattern {
1703 using Base::Base;
1704 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
1705 PatternRewriter &rewriter) const override {
1706 OpOperand *operand = getWarpResult(warpOp, fn: llvm::IsaPred<vector::InsertOp>);
1707 if (!operand)
1708 return failure();
1709 unsigned int operandNumber = operand->getOperandNumber();
1710 auto insertOp = operand->get().getDefiningOp<vector::InsertOp>();
1711 VectorType vecType = insertOp.getDestVectorType();
1712 VectorType distrType =
1713 cast<VectorType>(Val: warpOp.getResult(i: operandNumber).getType());
1714
1715 // Only supports 1-D or 0-D destinations for now.
1716 if (vecType.getRank() > 1) {
1717 return rewriter.notifyMatchFailure(
1718 arg&: insertOp, msg: "only 0-D or 1-D source supported for now");
1719 }
1720
1721 // Yield destination vector, source scalar and position from warp op.
1722 SmallVector<Value> additionalResults{insertOp.getDest(),
1723 insertOp.getValueToStore()};
1724 SmallVector<Type> additionalResultTypes{
1725 distrType, insertOp.getValueToStore().getType()};
1726 additionalResults.append(RHS: SmallVector<Value>(insertOp.getDynamicPosition()));
1727 additionalResultTypes.append(
1728 RHS: SmallVector<Type>(insertOp.getDynamicPosition().getTypes()));
1729
1730 Location loc = insertOp.getLoc();
1731 SmallVector<size_t> newRetIndices;
1732 WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
1733 rewriter, warpOp, newYieldedValues: additionalResults, newReturnTypes: additionalResultTypes,
1734 indices&: newRetIndices);
1735 rewriter.setInsertionPointAfter(newWarpOp);
1736 Value distributedVec = newWarpOp->getResult(idx: newRetIndices[0]);
1737 Value newSource = newWarpOp->getResult(idx: newRetIndices[1]);
1738 rewriter.setInsertionPointAfter(newWarpOp);
1739
1740 OpFoldResult pos;
1741 if (vecType.getRank() != 0) {
1742 int64_t staticPos = insertOp.getStaticPosition()[0];
1743 pos = ShapedType::isDynamic(dValue: staticPos)
1744 ? (newWarpOp->getResult(idx: newRetIndices[2]))
1745 : OpFoldResult(rewriter.getIndexAttr(value: staticPos));
1746 }
1747
1748 // This condition is always true for 0-d vectors.
1749 if (vecType == distrType) {
1750 Value newInsert;
1751 SmallVector<OpFoldResult> indices;
1752 if (pos) {
1753 indices.push_back(Elt: pos);
1754 }
1755 newInsert = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: newSource,
1756 dest: distributedVec, position: indices);
1757 // Broadcast: Simply move the vector.insert op out.
1758 rewriter.replaceAllUsesWith(from: newWarpOp->getResult(idx: operandNumber),
1759 to: newInsert);
1760 return success();
1761 }
1762
1763 // This is a distribution. Only one lane should insert.
1764 int64_t elementsPerLane = distrType.getShape()[0];
1765 AffineExpr sym0 = getAffineSymbolExpr(position: 0, context: rewriter.getContext());
1766 // tid of extracting thread: pos / elementsPerLane
1767 Value insertingLane = affine::makeComposedAffineApply(
1768 b&: rewriter, loc, e: sym0.ceilDiv(v: elementsPerLane), operands: pos);
1769 // Insert position: pos % elementsPerLane
1770 OpFoldResult newPos = affine::makeComposedFoldedAffineApply(
1771 b&: rewriter, loc, expr: sym0 % elementsPerLane, operands: pos);
1772 Value isInsertingLane =
1773 arith::CmpIOp::create(builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq,
1774 lhs: newWarpOp.getLaneid(), rhs: insertingLane);
1775 Value newResult =
1776 scf::IfOp::create(
1777 builder&: rewriter, location: loc, cond: isInsertingLane,
1778 /*thenBuilder=*/
1779 [&](OpBuilder &builder, Location loc) {
1780 Value newInsert = vector::InsertOp::create(
1781 builder, location: loc, valueToStore: newSource, dest: distributedVec, position: newPos);
1782 scf::YieldOp::create(builder, location: loc, results: newInsert);
1783 },
1784 /*elseBuilder=*/
1785 [&](OpBuilder &builder, Location loc) {
1786 scf::YieldOp::create(builder, location: loc, results: distributedVec);
1787 })
1788 .getResult(i: 0);
1789 rewriter.replaceAllUsesWith(from: newWarpOp->getResult(idx: operandNumber), to: newResult);
1790 return success();
1791 }
1792};
1793
1794struct WarpOpInsert : public WarpDistributionPattern {
1795 using Base::Base;
1796 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
1797 PatternRewriter &rewriter) const override {
1798 OpOperand *operand = getWarpResult(warpOp, fn: llvm::IsaPred<vector::InsertOp>);
1799 if (!operand)
1800 return failure();
1801 unsigned int operandNumber = operand->getOperandNumber();
1802 auto insertOp = operand->get().getDefiningOp<vector::InsertOp>();
1803 Location loc = insertOp.getLoc();
1804
1805 // For 1-d or 0-d destination cases, we rely on WarpOpInsertScalar pattern.
1806 if (insertOp.getDestVectorType().getRank() <= 1) {
1807 return failure();
1808 }
1809
1810 // All following cases are 2d or higher dimensional source vectors.
1811
1812 if (warpOp.getResult(i: operandNumber).getType() == operand->get().getType()) {
1813 // There is no distribution, this is a broadcast. Simply move the insert
1814 // out of the warp op.
1815 SmallVector<size_t> newRetIndices;
1816 WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
1817 rewriter, warpOp, newYieldedValues: {insertOp.getValueToStore(), insertOp.getDest()},
1818 newReturnTypes: {insertOp.getValueToStoreType(), insertOp.getDestVectorType()},
1819 indices&: newRetIndices);
1820 rewriter.setInsertionPointAfter(newWarpOp);
1821 Value distributedSrc = newWarpOp->getResult(idx: newRetIndices[0]);
1822 Value distributedDest = newWarpOp->getResult(idx: newRetIndices[1]);
1823 Value newResult = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: distributedSrc,
1824 dest: distributedDest,
1825 position: insertOp.getMixedPosition());
1826 rewriter.replaceAllUsesWith(from: newWarpOp->getResult(idx: operandNumber),
1827 to: newResult);
1828 return success();
1829 }
1830
1831 // Find the distributed dimension. There should be exactly one.
1832 auto distrDestType =
1833 cast<VectorType>(Val: warpOp.getResult(i: operandNumber).getType());
1834 auto yieldedType = cast<VectorType>(Val: operand->get().getType());
1835 int64_t distrDestDim = -1;
1836 for (int64_t i = 0; i < yieldedType.getRank(); ++i) {
1837 if (distrDestType.getDimSize(idx: i) != yieldedType.getDimSize(idx: i)) {
1838 // Keep this assert here in case WarpExecuteOnLane0Op gets extended to
1839 // support distributing multiple dimensions in the future.
1840 assert(distrDestDim == -1 && "found multiple distributed dims");
1841 distrDestDim = i;
1842 }
1843 }
1844 assert(distrDestDim != -1 && "could not find distributed dimension");
1845
1846 // Compute the distributed source vector type.
1847 VectorType srcVecType = cast<VectorType>(Val: insertOp.getValueToStoreType());
1848 SmallVector<int64_t> distrSrcShape(srcVecType.getShape());
1849 // E.g.: vector.insert %s, %d [2] : vector<96xf32> into vector<128x96xf32>
1850 // Case 1: distrDestDim = 1 (dim of size 96). In that case, each lane will
1851 // insert a smaller vector<3xf32>.
1852 // Case 2: distrDestDim = 0 (dim of size 128) => distrSrcDim = -1. In that
1853 // case, one lane will insert the source vector<96xf32>. The other
1854 // lanes will not do anything.
1855 int64_t distrSrcDim = distrDestDim - insertOp.getNumIndices();
1856 if (distrSrcDim >= 0)
1857 distrSrcShape[distrSrcDim] = distrDestType.getDimSize(idx: distrDestDim);
1858 auto distrSrcType =
1859 VectorType::get(shape: distrSrcShape, elementType: distrDestType.getElementType());
1860
1861 // Yield source and dest vectors from warp op.
1862 SmallVector<size_t> newRetIndices;
1863 WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
1864 rewriter, warpOp, newYieldedValues: {insertOp.getValueToStore(), insertOp.getDest()},
1865 newReturnTypes: {distrSrcType, distrDestType}, indices&: newRetIndices);
1866 rewriter.setInsertionPointAfter(newWarpOp);
1867 Value distributedSrc = newWarpOp->getResult(idx: newRetIndices[0]);
1868 Value distributedDest = newWarpOp->getResult(idx: newRetIndices[1]);
1869
1870 // Insert into the distributed vector.
1871 Value newResult;
1872 if (distrSrcDim >= 0) {
1873 // Every lane inserts a small piece.
1874 newResult = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: distributedSrc,
1875 dest: distributedDest,
1876 position: insertOp.getMixedPosition());
1877 } else {
1878 // One lane inserts the entire source vector.
1879 int64_t elementsPerLane = distrDestType.getDimSize(idx: distrDestDim);
1880 SmallVector<OpFoldResult> pos = insertOp.getMixedPosition();
1881 SmallVector<int64_t> newPos = getAsIntegers(foldResults: pos);
1882 // tid of inserting lane: pos / elementsPerLane
1883 Value insertingLane = arith::ConstantIndexOp::create(
1884 builder&: rewriter, location: loc, value: newPos[distrDestDim] / elementsPerLane);
1885 Value isInsertingLane =
1886 arith::CmpIOp::create(builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq,
1887 lhs: newWarpOp.getLaneid(), rhs: insertingLane);
1888 // Insert position: pos % elementsPerLane
1889 newPos[distrDestDim] %= elementsPerLane;
1890 auto insertingBuilder = [&](OpBuilder &builder, Location loc) {
1891 Value newInsert = vector::InsertOp::create(builder, location: loc, valueToStore: distributedSrc,
1892 dest: distributedDest, position: newPos);
1893 scf::YieldOp::create(builder, location: loc, results: newInsert);
1894 };
1895 auto nonInsertingBuilder = [&](OpBuilder &builder, Location loc) {
1896 scf::YieldOp::create(builder, location: loc, results: distributedDest);
1897 };
1898 newResult = scf::IfOp::create(builder&: rewriter, location: loc, cond: isInsertingLane,
1899 /*thenBuilder=*/insertingBuilder,
1900 /*elseBuilder=*/nonInsertingBuilder)
1901 .getResult(i: 0);
1902 }
1903
1904 rewriter.replaceAllUsesWith(from: newWarpOp->getResult(idx: operandNumber), to: newResult);
1905 return success();
1906 }
1907};
1908
1909/// Sink scf.if out of WarpExecuteOnLane0Op. This can be done only if
1910/// the scf.if is the last operation in the region so that it doesn't
1911/// change the order of execution. This creates a new scf.if after the
1912/// WarpExecuteOnLane0Op. Each branch of the new scf.if is enclosed in
1913/// the "inner" WarpExecuteOnLane0Op. Example:
1914/// ```
1915/// gpu.warp_execute_on_lane_0(%laneid)[32] {
1916/// %payload = ... : vector<32xindex>
1917/// scf.if %pred {
1918/// vector.store %payload, %buffer[%idx] : memref<128xindex>,
1919/// vector<32xindex>
1920/// }
1921/// gpu.yield
1922/// }
1923/// ```
1924/// %r = gpu.warp_execute_on_lane_0(%laneid)[32] {
1925/// %payload = ... : vector<32xindex>
1926/// gpu.yield %payload : vector<32xindex>
1927/// }
1928/// scf.if %pred {
1929/// gpu.warp_execute_on_lane_0(%laneid)[32] args(%r : vector<1xindex>) {
1930/// ^bb0(%arg1: vector<32xindex>):
1931/// vector.store %arg1, %buffer[%idx] : memref<128xindex>, vector<32xindex>
1932/// }
1933/// }
1934/// ```
1935struct WarpOpScfIfOp : public WarpDistributionPattern {
1936 WarpOpScfIfOp(MLIRContext *ctx, DistributionMapFn fn, PatternBenefit b = 1)
1937 : WarpDistributionPattern(ctx, b), distributionMapFn(std::move(fn)) {}
1938 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
1939 PatternRewriter &rewriter) const override {
1940 gpu::YieldOp warpOpYield = warpOp.getTerminator();
1941 // Only pick up `IfOp` if it is the last op in the region.
1942 Operation *lastNode = warpOpYield->getPrevNode();
1943 auto ifOp = dyn_cast_or_null<scf::IfOp>(Val: lastNode);
1944 if (!ifOp)
1945 return failure();
1946
1947 // The current `WarpOp` can yield two types of values:
1948 // 1. Not results of `IfOp`:
1949 // Preserve them in the new `WarpOp`.
1950 // Collect their yield index to remap the usages.
1951 // 2. Results of `IfOp`:
1952 // They are not part of the new `WarpOp` results.
1953 // Map current warp's yield operand index to `IfOp` result idx.
1954 SmallVector<Value> nonIfYieldValues;
1955 SmallVector<unsigned> nonIfYieldIndices;
1956 llvm::SmallDenseMap<unsigned, unsigned> ifResultMapping;
1957 llvm::SmallDenseMap<unsigned, VectorType> ifResultDistTypes;
1958 for (OpOperand &yieldOperand : warpOpYield->getOpOperands()) {
1959 const unsigned yieldOperandIdx = yieldOperand.getOperandNumber();
1960 if (yieldOperand.get().getDefiningOp() != ifOp.getOperation()) {
1961 nonIfYieldValues.push_back(Elt: yieldOperand.get());
1962 nonIfYieldIndices.push_back(Elt: yieldOperandIdx);
1963 continue;
1964 }
1965 OpResult ifResult = cast<OpResult>(Val: yieldOperand.get());
1966 const unsigned ifResultIdx = ifResult.getResultNumber();
1967 ifResultMapping[yieldOperandIdx] = ifResultIdx;
1968 // If this `ifOp` result is vector type and it is yielded by the
1969 // `WarpOp`, we keep track the distributed type for this result.
1970 if (!isa<VectorType>(Val: ifResult.getType()))
1971 continue;
1972 VectorType distType =
1973 cast<VectorType>(Val: warpOp.getResult(i: yieldOperandIdx).getType());
1974 ifResultDistTypes[ifResultIdx] = distType;
1975 }
1976
1977 // Collect `WarpOp`-defined values used in `ifOp`, the new warp op returns
1978 // them
1979 auto [escapingValuesThen, escapingValueInputTypesThen,
1980 escapingValueDistTypesThen] =
1981 getInnerRegionEscapingValues(warpOp, innerRegion&: ifOp.getThenRegion(),
1982 distributionMapFn);
1983 auto [escapingValuesElse, escapingValueInputTypesElse,
1984 escapingValueDistTypesElse] =
1985 getInnerRegionEscapingValues(warpOp, innerRegion&: ifOp.getElseRegion(),
1986 distributionMapFn);
1987 if (llvm::is_contained(Range&: escapingValueDistTypesThen, Element: Type{}) ||
1988 llvm::is_contained(Range&: escapingValueDistTypesElse, Element: Type{}))
1989 return failure();
1990
1991 // The new `WarpOp` groups yields values in following order:
1992 // 1. Branch condition
1993 // 2. Escaping values then branch
1994 // 3. Escaping values else branch
1995 // 4. All non-`ifOp` yielded values.
1996 SmallVector<Value> newWarpOpYieldValues{ifOp.getCondition()};
1997 newWarpOpYieldValues.append(in_start: escapingValuesThen.begin(),
1998 in_end: escapingValuesThen.end());
1999 newWarpOpYieldValues.append(in_start: escapingValuesElse.begin(),
2000 in_end: escapingValuesElse.end());
2001 SmallVector<Type> newWarpOpDistTypes{ifOp.getCondition().getType()};
2002 newWarpOpDistTypes.append(in_start: escapingValueDistTypesThen.begin(),
2003 in_end: escapingValueDistTypesThen.end());
2004 newWarpOpDistTypes.append(in_start: escapingValueDistTypesElse.begin(),
2005 in_end: escapingValueDistTypesElse.end());
2006
2007 for (auto [idx, val] :
2008 llvm::zip_equal(t&: nonIfYieldIndices, u&: nonIfYieldValues)) {
2009 newWarpOpYieldValues.push_back(Elt: val);
2010 newWarpOpDistTypes.push_back(Elt: warpOp.getResult(i: idx).getType());
2011 }
2012 // Replace the old `WarpOp` with the new one that has additional yield
2013 // values and types.
2014 SmallVector<size_t> newIndices;
2015 WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
2016 rewriter, warpOp, newYieldedValues: newWarpOpYieldValues, newReturnTypes: newWarpOpDistTypes, indices&: newIndices);
2017 // `ifOp` returns the result of the inner warp op.
2018 SmallVector<Type> newIfOpDistResTypes;
2019 for (auto [i, res] : llvm::enumerate(First: ifOp.getResults())) {
2020 Type distType = cast<Value>(Val&: res).getType();
2021 if (auto vecType = dyn_cast<VectorType>(Val&: distType)) {
2022 AffineMap map = distributionMapFn(cast<Value>(Val&: res));
2023 // Fallback to affine map if the dist result was not previously recorded
2024 distType = ifResultDistTypes.count(Val: i)
2025 ? ifResultDistTypes[i]
2026 : getDistributedType(
2027 originalType: vecType, map,
2028 warpSize: map.isEmpty() ? 1 : newWarpOp.getWarpSize());
2029 }
2030 newIfOpDistResTypes.push_back(Elt: distType);
2031 }
2032 // Create a new `IfOp` outside the new `WarpOp` region.
2033 OpBuilder::InsertionGuard g(rewriter);
2034 rewriter.setInsertionPointAfter(newWarpOp);
2035 auto newIfOp = scf::IfOp::create(
2036 builder&: rewriter, location: ifOp.getLoc(), resultTypes: newIfOpDistResTypes,
2037 cond: newWarpOp.getResult(i: newIndices[0]), addThenBlock: static_cast<bool>(ifOp.thenBlock()),
2038 addElseBlock: static_cast<bool>(ifOp.elseBlock()));
2039 auto encloseRegionInWarpOp =
2040 [&](Block *oldIfBranch, Block *newIfBranch,
2041 llvm::SmallSetVector<Value, 32> &escapingValues,
2042 SmallVector<Type> &escapingValueInputTypes,
2043 size_t warpResRangeStart) {
2044 OpBuilder::InsertionGuard g(rewriter);
2045 if (!newIfBranch)
2046 return;
2047 rewriter.setInsertionPointToStart(newIfBranch);
2048 llvm::SmallDenseMap<Value, int64_t> escapeValToBlockArgIndex;
2049 SmallVector<Value> innerWarpInputVals;
2050 SmallVector<Type> innerWarpInputTypes;
2051 for (size_t i = 0; i < escapingValues.size();
2052 ++i, ++warpResRangeStart) {
2053 innerWarpInputVals.push_back(
2054 Elt: newWarpOp.getResult(i: newIndices[warpResRangeStart]));
2055 escapeValToBlockArgIndex[escapingValues[i]] =
2056 innerWarpInputTypes.size();
2057 innerWarpInputTypes.push_back(Elt: escapingValueInputTypes[i]);
2058 }
2059 auto innerWarp = WarpExecuteOnLane0Op::create(
2060 builder&: rewriter, location: newWarpOp.getLoc(), resultTypes: newIfOp.getResultTypes(),
2061 laneid: newWarpOp.getLaneid(), warpSize: newWarpOp.getWarpSize(),
2062 args: innerWarpInputVals, blockArgTypes: innerWarpInputTypes);
2063
2064 innerWarp.getWarpRegion().takeBody(other&: *oldIfBranch->getParent());
2065 innerWarp.getWarpRegion().addArguments(
2066 types: innerWarpInputTypes,
2067 locs: SmallVector<Location>(innerWarpInputTypes.size(), ifOp.getLoc()));
2068
2069 SmallVector<Value> yieldOperands;
2070 for (Value operand : oldIfBranch->getTerminator()->getOperands())
2071 yieldOperands.push_back(Elt: operand);
2072 rewriter.eraseOp(op: oldIfBranch->getTerminator());
2073
2074 rewriter.setInsertionPointToEnd(innerWarp.getBody());
2075 gpu::YieldOp::create(builder&: rewriter, location: innerWarp.getLoc(), values: yieldOperands);
2076 rewriter.setInsertionPointAfter(innerWarp);
2077 scf::YieldOp::create(builder&: rewriter, location: ifOp.getLoc(), results: innerWarp.getResults());
2078
2079 // Update any users of escaping values that were forwarded to the
2080 // inner `WarpOp`. These values are arguments of the inner `WarpOp`.
2081 innerWarp.walk(callback: [&](Operation *op) {
2082 SmallVector<std::pair<unsigned, Value>> replacements;
2083 for (OpOperand &operand : op->getOpOperands()) {
2084 auto it = escapeValToBlockArgIndex.find(Val: operand.get());
2085 if (it == escapeValToBlockArgIndex.end())
2086 continue;
2087 replacements.emplace_back(
2088 Args: operand.getOperandNumber(),
2089 Args: innerWarp.getBodyRegion().getArgument(i: it->second));
2090 }
2091 if (!replacements.empty()) {
2092 rewriter.modifyOpInPlace(root: op, callable: [&]() {
2093 for (auto [idx, newVal] : replacements)
2094 op->setOperand(idx, value: newVal);
2095 });
2096 }
2097 });
2098 mlir::vector::moveScalarUniformCode(op: innerWarp);
2099 };
2100 encloseRegionInWarpOp(&ifOp.getThenRegion().front(),
2101 &newIfOp.getThenRegion().front(), escapingValuesThen,
2102 escapingValueInputTypesThen, 1);
2103 if (!ifOp.getElseRegion().empty())
2104 encloseRegionInWarpOp(&ifOp.getElseRegion().front(),
2105 &newIfOp.getElseRegion().front(),
2106 escapingValuesElse, escapingValueInputTypesElse,
2107 1 + escapingValuesThen.size());
2108 // Update the users of `<- WarpOp.yield <- IfOp.yield` to use the new `IfOp`
2109 // result.
2110 for (auto [origIdx, newIdx] : ifResultMapping)
2111 rewriter.replaceAllUsesExcept(from: newWarpOp.getResult(i: origIdx),
2112 to: newIfOp.getResult(i: newIdx), exceptedUser: newIfOp);
2113
2114 // The original `ifOp` was left inside `newWarpOp` with empty then/else
2115 // regions (their blocks were moved into the inner WarpOps by takeBody).
2116 // Clear remaining uses and erase it to restore IR validity. Directly
2117 // update newWarpOp's yield operands instead of using replaceAllUsesWith,
2118 // to avoid triggering notifyOperandReplaced on the now-invalid ifOp.
2119 {
2120 OpBuilder::InsertionGuard guard(rewriter);
2121 rewriter.setInsertionPoint(ifOp);
2122 Operation *yield = newWarpOp.getTerminator();
2123 rewriter.modifyOpInPlace(root: yield, callable: [&]() {
2124 for (auto [origIdx, ifResultIdx] : ifResultMapping) {
2125 Value poison = ub::PoisonOp::create(
2126 builder&: rewriter, location: ifOp.getLoc(), result: ifOp.getResult(i: ifResultIdx).getType());
2127 yield->setOperand(idx: origIdx, value: poison);
2128 }
2129 });
2130 rewriter.eraseOp(op: ifOp);
2131 }
2132
2133 return success();
2134 }
2135
2136private:
2137 DistributionMapFn distributionMapFn;
2138};
2139
2140/// Sink scf.for region out of WarpExecuteOnLane0Op. This can be done only if
2141/// the scf.ForOp is the last operation in the region so that it doesn't
2142/// change the order of execution. This creates a new scf.for region after the
2143/// WarpExecuteOnLane0Op. The new scf.for region will contain a new
2144/// WarpExecuteOnLane0Op region. Example:
2145/// ```
2146/// %w = gpu.warp_execute_on_lane_0(%laneid) -> (vector<4xf32>) {
2147/// ...
2148/// %v1 = scf.for %arg3 = %c0 to %c128 step %c1 iter_args(%arg4 = %v)
2149/// -> (vector<128xf32>) {
2150/// ...
2151/// scf.yield %r : vector<128xf32>
2152/// }
2153/// gpu.yield %v1 : vector<128xf32>
2154/// }
2155/// ```
2156/// To:
2157/// %w0 = gpu.warp_execute_on_lane_0(%arg0) -> (vector<4xf32>) {
2158/// ...
2159/// gpu.yield %v : vector<128xf32>
2160/// }
2161/// %w = scf.for %arg3 = %c0 to %c128 step %c1 iter_args(%varg = %q0)
2162/// -> (vector<4xf32>) {
2163/// %iw = gpu.warp_execute_on_lane_0(%laneid)
2164/// args(%varg : vector<4xf32>) -> (vector<4xf32>) {
2165/// ^bb0(%arg: vector<128xf32>):
2166/// ...
2167/// gpu.yield %ir : vector<128xf32>
2168/// }
2169/// scf.yield %iw : vector<4xf32>
2170/// }
2171/// ```
2172struct WarpOpScfForOp : public WarpDistributionPattern {
2173
2174 WarpOpScfForOp(MLIRContext *ctx, DistributionMapFn fn, PatternBenefit b = 1)
2175 : WarpDistributionPattern(ctx, b), distributionMapFn(std::move(fn)) {}
2176 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
2177 PatternRewriter &rewriter) const override {
2178 gpu::YieldOp warpOpYield = warpOp.getTerminator();
2179 // Only pick up `ForOp` if it is the last op in the region.
2180 Operation *lastNode = warpOpYield->getPrevNode();
2181 auto forOp = dyn_cast_or_null<scf::ForOp>(Val: lastNode);
2182 if (!forOp)
2183 return failure();
2184 // Collect Values that come from the `WarpOp` but are outside the `ForOp`.
2185 // Those Values need to be returned by the new warp op.
2186 auto [escapingValues, escapingValueInputTypes, escapingValueDistTypes] =
2187 getInnerRegionEscapingValues(warpOp, innerRegion&: forOp.getBodyRegion(),
2188 distributionMapFn);
2189 if (llvm::is_contained(Range&: escapingValueDistTypes, Element: Type{}))
2190 return failure();
2191 // `WarpOp` can yield two types of values:
2192 // 1. Values that are not results of the `ForOp`:
2193 // These values must also be yielded by the new `WarpOp`. Also, we need
2194 // to record the index mapping for these values to replace them later.
2195 // 2. Values that are results of the `ForOp`:
2196 // In this case, we record the index mapping between the `WarpOp` result
2197 // index and matching `ForOp` result index.
2198 // Additionally, we keep track of the distributed types for all `ForOp`
2199 // vector results.
2200 SmallVector<Value> nonForYieldedValues;
2201 SmallVector<unsigned> nonForResultIndices;
2202 llvm::SmallDenseMap<unsigned, unsigned> forResultMapping;
2203 llvm::SmallDenseMap<unsigned, VectorType> forResultDistTypes;
2204 llvm::SmallBitVector forResultsMapped(forOp.getNumResults());
2205 for (OpOperand &yieldOperand : warpOpYield->getOpOperands()) {
2206 // Yielded value is not a result of the forOp.
2207 if (yieldOperand.get().getDefiningOp() != forOp.getOperation()) {
2208 nonForYieldedValues.push_back(Elt: yieldOperand.get());
2209 nonForResultIndices.push_back(Elt: yieldOperand.getOperandNumber());
2210 continue;
2211 }
2212 OpResult forResult = cast<OpResult>(Val: yieldOperand.get());
2213 unsigned int forResultNumber = forResult.getResultNumber();
2214 forResultMapping[yieldOperand.getOperandNumber()] = forResultNumber;
2215 forResultsMapped.set(forResultNumber);
2216 // If this `ForOp` result is vector type and it is yielded by the
2217 // `WarpOp`, we keep track the distributed type for this result.
2218 if (!isa<VectorType>(Val: forResult.getType()))
2219 continue;
2220 VectorType distType = cast<VectorType>(
2221 Val: warpOp.getResult(i: yieldOperand.getOperandNumber()).getType());
2222 forResultDistTypes[forResultNumber] = distType;
2223 }
2224
2225 // Newly created `WarpOp` will yield values in following order:
2226 // 1. Loop bounds.
2227 // 2. All init args of the `ForOp`.
2228 // 3. All escaping values.
2229 // 4. All non-`ForOp` yielded values.
2230 SmallVector<Value> newWarpOpYieldValues;
2231 SmallVector<Type> newWarpOpDistTypes;
2232 newWarpOpYieldValues.insert(
2233 I: newWarpOpYieldValues.end(),
2234 IL: {forOp.getLowerBound(), forOp.getUpperBound(), forOp.getStep()});
2235 newWarpOpDistTypes.insert(I: newWarpOpDistTypes.end(),
2236 IL: {forOp.getLowerBound().getType(),
2237 forOp.getUpperBound().getType(),
2238 forOp.getStep().getType()});
2239 for (auto [i, initArg] : llvm::enumerate(First: forOp.getInitArgs())) {
2240 newWarpOpYieldValues.push_back(Elt: initArg);
2241 // Compute the distributed type for this init arg.
2242 Type distType = initArg.getType();
2243 if (auto vecType = dyn_cast<VectorType>(Val&: distType)) {
2244 // If the `ForOp` result corresponds to this init arg is already yielded
2245 // we can get the distributed type from `forResultDistTypes` map.
2246 // Otherwise, we compute it using distributionMapFn.
2247 AffineMap map = distributionMapFn(initArg);
2248 distType =
2249 forResultDistTypes.count(Val: i)
2250 ? forResultDistTypes[i]
2251 : getDistributedType(originalType: vecType, map,
2252 warpSize: map.isEmpty() ? 1 : warpOp.getWarpSize());
2253 }
2254 newWarpOpDistTypes.push_back(Elt: distType);
2255 }
2256 // Insert escaping values and their distributed types.
2257 newWarpOpYieldValues.insert(I: newWarpOpYieldValues.end(),
2258 From: escapingValues.begin(), To: escapingValues.end());
2259 newWarpOpDistTypes.insert(I: newWarpOpDistTypes.end(),
2260 From: escapingValueDistTypes.begin(),
2261 To: escapingValueDistTypes.end());
2262 // Next, we insert all non-`ForOp` yielded values and their distributed
2263 // types.
2264 for (auto [i, v] :
2265 llvm::zip_equal(t&: nonForResultIndices, u&: nonForYieldedValues)) {
2266 newWarpOpYieldValues.push_back(Elt: v);
2267 newWarpOpDistTypes.push_back(Elt: warpOp.getResult(i).getType());
2268 }
2269 // Create the new `WarpOp` with the updated yield values and types.
2270 SmallVector<size_t> newIndices;
2271 WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
2272 rewriter, warpOp, newYieldedValues: newWarpOpYieldValues, newReturnTypes: newWarpOpDistTypes, indices&: newIndices);
2273
2274 // Next, we create a new `ForOp` with the init args yielded by the new
2275 // `WarpOp`.
2276 const unsigned initArgsStartIdx = 3; // After loop bounds.
2277 const unsigned escapingValuesStartIdx =
2278 initArgsStartIdx +
2279 forOp.getInitArgs().size(); // `ForOp` init args are positioned before
2280 // escaping values in the new `WarpOp`.
2281 SmallVector<Value> newForOpOperands;
2282 for (size_t i = initArgsStartIdx; i < escapingValuesStartIdx; ++i)
2283 newForOpOperands.push_back(Elt: newWarpOp.getResult(i: newIndices[i]));
2284
2285 // Create a new `ForOp` outside the new `WarpOp` region.
2286 OpBuilder::InsertionGuard g(rewriter);
2287 rewriter.setInsertionPointAfter(newWarpOp);
2288 auto newForOp = scf::ForOp::create(
2289 builder&: rewriter, location: forOp.getLoc(),
2290 /**LowerBound=**/ lowerBound: newWarpOp.getResult(i: newIndices[0]),
2291 /**UpperBound=**/ upperBound: newWarpOp.getResult(i: newIndices[1]),
2292 /**Step=**/ step: newWarpOp.getResult(i: newIndices[2]), initArgs: newForOpOperands,
2293 /*bodyBuilder=*/odsArg4: nullptr, unsignedCmp: forOp.getUnsignedCmp());
2294 // Next, we insert a new `WarpOp` (called inner `WarpOp`) inside the
2295 // newly created `ForOp`. This `WarpOp` will contain all ops that were
2296 // contained within the original `ForOp` body.
2297 rewriter.setInsertionPointToStart(newForOp.getBody());
2298
2299 SmallVector<Value> innerWarpInput(newForOp.getRegionIterArgs().begin(),
2300 newForOp.getRegionIterArgs().end());
2301 SmallVector<Type> innerWarpInputType(forOp.getResultTypes().begin(),
2302 forOp.getResultTypes().end());
2303 // Escaping values are forwarded to the inner `WarpOp` as its (additional)
2304 // arguments. We keep track of the mapping between these values and their
2305 // argument index in the inner `WarpOp` (to replace users later).
2306 llvm::SmallDenseMap<Value, int64_t> argIndexMapping;
2307 for (size_t i = escapingValuesStartIdx;
2308 i < escapingValuesStartIdx + escapingValues.size(); ++i) {
2309 innerWarpInput.push_back(Elt: newWarpOp.getResult(i: newIndices[i]));
2310 argIndexMapping[escapingValues[i - escapingValuesStartIdx]] =
2311 innerWarpInputType.size();
2312 innerWarpInputType.push_back(
2313 Elt: escapingValueInputTypes[i - escapingValuesStartIdx]);
2314 }
2315 // Create the inner `WarpOp` with the new input values and types.
2316 auto innerWarp = WarpExecuteOnLane0Op::create(
2317 builder&: rewriter, location: newWarpOp.getLoc(), resultTypes: newForOp.getResultTypes(),
2318 laneid: newWarpOp.getLaneid(), warpSize: newWarpOp.getWarpSize(), args: innerWarpInput,
2319 blockArgTypes: innerWarpInputType);
2320
2321 // Inline the `ForOp` body into the inner `WarpOp` body.
2322 SmallVector<Value> argMapping;
2323 argMapping.push_back(Elt: newForOp.getInductionVar());
2324 for (Value args : innerWarp.getBody()->getArguments())
2325 argMapping.push_back(Elt: args);
2326
2327 argMapping.resize(N: forOp.getBody()->getNumArguments());
2328 SmallVector<Value> yieldOperands;
2329 for (Value operand : forOp.getBody()->getTerminator()->getOperands()) {
2330 if (BlockArgument blockArg = dyn_cast<BlockArgument>(Val&: operand);
2331 blockArg && blockArg.getOwner() == forOp.getBody()) {
2332 yieldOperands.push_back(Elt: argMapping[blockArg.getArgNumber()]);
2333 continue;
2334 }
2335 yieldOperands.push_back(Elt: operand);
2336 }
2337
2338 rewriter.eraseOp(op: forOp.getBody()->getTerminator());
2339 rewriter.mergeBlocks(source: forOp.getBody(), dest: innerWarp.getBody(), argValues: argMapping);
2340
2341 // Insert a gpu `YieldOp` at the end of the inner `WarpOp` body that yields
2342 // original `ForOp` results.
2343 rewriter.setInsertionPointToEnd(innerWarp.getBody());
2344 gpu::YieldOp::create(builder&: rewriter, location: innerWarp.getLoc(), values: yieldOperands);
2345 rewriter.setInsertionPointAfter(innerWarp);
2346 // Insert a scf.yield op at the end of the new `ForOp` body that yields
2347 // the inner `WarpOp` results.
2348 if (!innerWarp.getResults().empty())
2349 scf::YieldOp::create(builder&: rewriter, location: forOp.getLoc(), results: innerWarp.getResults());
2350
2351 // Update the users of the new `WarpOp` results that were coming from the
2352 // original `ForOp` to the corresponding new `ForOp` result.
2353 for (auto [origIdx, newIdx] : forResultMapping)
2354 rewriter.replaceAllUsesExcept(from: newWarpOp.getResult(i: origIdx),
2355 to: newForOp.getResult(i: newIdx), exceptedUser: newForOp);
2356
2357 // The original `ForOp` was left inside `newWarpOp` with an empty body
2358 // region (its body block was moved into `innerWarp` by `mergeBlocks`).
2359 // Clear remaining uses and erase it to restore IR validity.
2360 for (OpResult result : forOp.getResults()) {
2361 if (forResultsMapped.test(Idx: result.getResultNumber()))
2362 rewriter.replaceAllUsesWith(
2363 from: result, to: forOp.getInitArgs()[result.getResultNumber()]);
2364 }
2365 rewriter.eraseOp(op: forOp);
2366
2367 // Update any users of escaping values that were forwarded to the
2368 // inner `WarpOp`. These values are now arguments of the inner `WarpOp`.
2369 newForOp.walk(callback: [&](Operation *op) {
2370 SmallVector<std::pair<unsigned, Value>> replacements;
2371 for (OpOperand &operand : op->getOpOperands()) {
2372 auto it = argIndexMapping.find(Val: operand.get());
2373 if (it == argIndexMapping.end())
2374 continue;
2375 replacements.emplace_back(
2376 Args: operand.getOperandNumber(),
2377 Args: innerWarp.getBodyRegion().getArgument(i: it->second));
2378 }
2379 if (!replacements.empty()) {
2380 rewriter.modifyOpInPlace(root: op, callable: [&]() {
2381 for (auto [idx, newVal] : replacements)
2382 op->setOperand(idx, value: newVal);
2383 });
2384 }
2385 });
2386
2387 // Finally, hoist out any now uniform code from the inner `WarpOp`.
2388 mlir::vector::moveScalarUniformCode(op: innerWarp);
2389 return success();
2390 }
2391
2392private:
2393 DistributionMapFn distributionMapFn;
2394};
2395
2396/// A pattern that extracts vector.reduction ops from a WarpExecuteOnLane0Op.
2397/// The vector is reduced in parallel. Currently limited to vector size
2398/// matching the warpOp size. E.g.:
2399/// ```
2400/// %r = gpu.warp_execute_on_lane_0(%laneid)[32] -> (f32) {
2401/// %0 = "some_def"() : () -> (vector<32xf32>)
2402/// %1 = vector.reduction "add", %0 : vector<32xf32> into f32
2403/// gpu.yield %1 : f32
2404/// }
2405/// ```
2406/// is lowered to:
2407/// ```
2408/// %0 = gpu.warp_execute_on_lane_0(%laneid)[32] -> (vector<1xf32>) {
2409/// %1 = "some_def"() : () -> (vector<32xf32>)
2410/// gpu.yield %1 : vector<32xf32>
2411/// }
2412/// %a = vector.extract %0[0] : f32 from vector<1xf32>
2413/// %r = ("warp.reduction %a")
2414/// ```
2415struct WarpOpReduction : public WarpDistributionPattern {
2416 WarpOpReduction(MLIRContext *context,
2417 DistributedReductionFn distributedReductionFn,
2418 PatternBenefit benefit = 1)
2419 : WarpDistributionPattern(context, benefit),
2420 distributedReductionFn(std::move(distributedReductionFn)) {}
2421
2422 LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
2423 PatternRewriter &rewriter) const override {
2424 OpOperand *yieldOperand =
2425 getWarpResult(warpOp, fn: llvm::IsaPred<vector::ReductionOp>);
2426 if (!yieldOperand)
2427 return failure();
2428
2429 auto reductionOp =
2430 cast<vector::ReductionOp>(Val: yieldOperand->get().getDefiningOp());
2431 auto vectorType = cast<VectorType>(Val: reductionOp.getVector().getType());
2432 // Only rank 1 vectors supported.
2433 if (vectorType.getRank() != 1)
2434 return rewriter.notifyMatchFailure(
2435 arg&: warpOp, msg: "Only rank 1 reductions can be distributed.");
2436 if (vectorType.isScalable())
2437 return rewriter.notifyMatchFailure(
2438 arg&: warpOp, msg: "Scalable reductions cannot be distributed.");
2439 // Only warp_size-sized vectors supported.
2440 if (vectorType.getShape()[0] % warpOp.getWarpSize() != 0)
2441 return rewriter.notifyMatchFailure(
2442 arg&: warpOp, msg: "Reduction vector dimension must match was size.");
2443 if (!reductionOp.getType().isIntOrFloat())
2444 return rewriter.notifyMatchFailure(
2445 arg&: warpOp, msg: "Reduction distribution currently only supports floats and "
2446 "integer types.");
2447
2448 int64_t numElements = vectorType.getShape()[0] / warpOp.getWarpSize();
2449 // Return vector that will be reduced from the WarpExecuteOnLane0Op.
2450 unsigned operandIndex = yieldOperand->getOperandNumber();
2451 SmallVector<Value> yieldValues = {reductionOp.getVector()};
2452 SmallVector<Type> retTypes = {
2453 VectorType::get(shape: {numElements}, elementType: reductionOp.getType())};
2454 if (reductionOp.getAcc()) {
2455 yieldValues.push_back(Elt: reductionOp.getAcc());
2456 retTypes.push_back(Elt: reductionOp.getAcc().getType());
2457 }
2458 SmallVector<size_t> newRetIndices;
2459 WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
2460 rewriter, warpOp, newYieldedValues: yieldValues, newReturnTypes: retTypes, indices&: newRetIndices);
2461 rewriter.setInsertionPointAfter(newWarpOp);
2462
2463 // Obtain data to reduce for a single lane.
2464 Value laneValVec = newWarpOp.getResult(i: newRetIndices[0]);
2465 // Distribute and reduce across threads.
2466 Value fullReduce =
2467 distributedReductionFn(reductionOp.getLoc(), rewriter, laneValVec,
2468 reductionOp.getKind(), newWarpOp.getWarpSize());
2469 if (reductionOp.getAcc()) {
2470 fullReduce = vector::makeArithReduction(
2471 b&: rewriter, loc: reductionOp.getLoc(), kind: reductionOp.getKind(), v1: fullReduce,
2472 acc: newWarpOp.getResult(i: newRetIndices[1]));
2473 }
2474 rewriter.replaceAllUsesWith(from: newWarpOp.getResult(i: operandIndex), to: fullReduce);
2475 return success();
2476 }
2477
2478private:
2479 DistributedReductionFn distributedReductionFn;
2480};
2481} // namespace
2482
2483void mlir::vector::populateWarpExecuteOnLane0OpToScfForPattern(
2484 RewritePatternSet &patterns,
2485 const WarpExecuteOnLane0LoweringOptions &options, PatternBenefit benefit) {
2486 patterns.add<WarpOpToScfIfPattern>(arg: patterns.getContext(), args: options, args&: benefit);
2487}
2488
2489void mlir::vector::populateDistributeTransferWriteOpPatterns(
2490 RewritePatternSet &patterns, const DistributionMapFn &distributionMapFn,
2491 unsigned maxNumElementsToExtract, PatternBenefit benefit) {
2492 patterns.add<WarpOpTransferWrite>(arg: patterns.getContext(), args: distributionMapFn,
2493 args&: maxNumElementsToExtract, args&: benefit);
2494}
2495
2496void mlir::vector::populatePropagateWarpVectorDistributionPatterns(
2497 RewritePatternSet &patterns, const DistributionMapFn &distributionMapFn,
2498 const WarpShuffleFromIdxFn &warpShuffleFromIdxFn, PatternBenefit benefit,
2499 PatternBenefit readBenefit) {
2500 patterns.add<WarpOpTransferRead>(arg: patterns.getContext(), args&: readBenefit);
2501 patterns.add<WarpOpElementwise, WarpOpDeadResult, WarpOpBroadcast,
2502 WarpOpShapeCast, WarpOpExtract, WarpOpForwardOperand,
2503 WarpOpConstant, WarpOpInsertScalar, WarpOpInsert,
2504 WarpOpCreateMask<vector::CreateMaskOp>,
2505 WarpOpCreateMask<vector::ConstantMaskOp>,
2506 WarpOpExtractStridedSlice, WarpOpInsertStridedSlice, WarpOpStep>(
2507 arg: patterns.getContext(), args&: benefit);
2508 patterns.add<WarpOpExtractScalar>(arg: patterns.getContext(), args: warpShuffleFromIdxFn,
2509 args&: benefit);
2510 patterns.add<WarpOpScfForOp>(arg: patterns.getContext(), args: distributionMapFn,
2511 args&: benefit);
2512 patterns.add<WarpOpScfIfOp>(arg: patterns.getContext(), args: distributionMapFn,
2513 args&: benefit);
2514}
2515
2516void mlir::vector::populateDistributeReduction(
2517 RewritePatternSet &patterns,
2518 const DistributedReductionFn &distributedReductionFn,
2519 PatternBenefit benefit) {
2520 patterns.add<WarpOpReduction>(arg: patterns.getContext(), args: distributedReductionFn,
2521 args&: benefit);
2522}
2523
2524/// Helper to know if an op can be hoisted out of the region.
2525static bool canBeHoisted(Operation *op,
2526 function_ref<bool(Value)> definedOutside) {
2527 return llvm::all_of(Range: op->getOperands(), P: definedOutside) &&
2528 isMemoryEffectFree(op) && op->getNumRegions() == 0;
2529}
2530
2531void mlir::vector::moveScalarUniformCode(WarpExecuteOnLane0Op warpOp) {
2532 Block *body = warpOp.getBody();
2533
2534 // Keep track of the ops we want to hoist.
2535 llvm::SmallSetVector<Operation *, 8> opsToMove;
2536
2537 // Helper to check if a value is or will be defined outside of the region.
2538 auto isDefinedOutsideOfBody = [&](Value value) {
2539 auto *definingOp = value.getDefiningOp();
2540 return (definingOp && opsToMove.count(key: definingOp)) ||
2541 warpOp.isDefinedOutsideOfRegion(value);
2542 };
2543
2544 // Do not use walk here, as we do not want to go into nested regions and hoist
2545 // operations from there.
2546 for (auto &op : body->without_terminator()) {
2547 bool hasVectorResult = llvm::any_of(Range: op.getResults(), P: [](Value result) {
2548 return isa<VectorType>(Val: result.getType());
2549 });
2550 if (!hasVectorResult && canBeHoisted(op: &op, definedOutside: isDefinedOutsideOfBody))
2551 opsToMove.insert(X: &op);
2552 }
2553
2554 // Move all the ops marked as uniform outside of the region.
2555 for (Operation *op : opsToMove)
2556 op->moveBefore(existingOp: warpOp);
2557}
2558