1//===- CreateAsyncGroups.cpp - Create async device copies -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/NVGPU/Transforms/Transforms.h"
10
11#include "mlir/Dialect/Arith/IR/Arith.h"
12#include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
13#include "mlir/Dialect/NVGPU/Transforms/Utils.h"
14#include "mlir/Dialect/Vector/IR/VectorOps.h"
15#include "mlir/IR/BuiltinAttributes.h"
16#include "mlir/IR/BuiltinTypes.h"
17
18using namespace mlir;
19
20/// Return "true" if the given vector transfer op is contiguous and suitable
21/// for replacement with an async copy.
22template <typename OpTy>
23static bool isContiguousXferOp(OpTy op) {
24 return op.getPermutationMap().isMinorIdentity() && op.isDimInBounds(0) &&
25 op.hasPureBufferSemantics() &&
26 cast<MemRefType>(nvgpu::getMemrefOperand(op).getType())
27 .isLastDimUnitStride();
28}
29
30/// Return "true" if the given op is a contiguous and suitable
31/// vector.transfer_write or vector.store op.
32static bool isContiguousStore(Operation *write) {
33 if (auto transferWrite = dyn_cast<vector::TransferWriteOp>(Val: write))
34 return isContiguousXferOp(op: transferWrite) && !transferWrite.getMask();
35 // vector.store are always contiguous.
36 return isa<vector::StoreOp>(Val: write);
37}
38
39/// Return "true" if the given op is a contiguous and suitable
40/// vector.transfer_read or vector.load op.
41static bool isContiguousRead(Operation *read) {
42 if (auto transferRead = dyn_cast<vector::TransferReadOp>(Val: read))
43 return isContiguousXferOp(op: transferRead);
44 // vector.load are always contiguous.
45 return isa<vector::LoadOp>(Val: read);
46}
47
48namespace {
49/// A vector.create_mask op and extract position.
50struct TransferMask {
51 vector::CreateMaskOp createMaskOp;
52 SmallVector<int64_t> extractPosition;
53};
54} // namespace
55
56/// If the given vector load op has a mask that is defined by
57/// vector.create_mask, return that op.
58static FailureOr<TransferMask> getMaskOp(Operation *loadOp) {
59 auto transferRead = dyn_cast<vector::TransferReadOp>(Val: loadOp);
60 if (!transferRead || !transferRead.getMask())
61 return TransferMask{.createMaskOp: {}, .extractPosition: {}};
62 assert(transferRead.getMask().getType().getRank() == 1 &&
63 "expected 1-D mask");
64
65 // Case 1: Mask is the result of a vector.create_mask.
66 if (auto maskOp =
67 transferRead.getMask().getDefiningOp<vector::CreateMaskOp>())
68 return TransferMask{.createMaskOp: maskOp, .extractPosition: {}};
69
70 // Case 2: Mask is the result of a vector.extract(vector.create_mask).
71 if (auto extractOp =
72 transferRead.getMask().getDefiningOp<vector::ExtractOp>())
73 if (auto maskOp =
74 extractOp.getSource().getDefiningOp<vector::CreateMaskOp>())
75 return TransferMask{.createMaskOp: maskOp,
76 .extractPosition: SmallVector<int64_t>(extractOp.getStaticPosition())};
77
78 // All other cases: not supported.
79 return failure();
80}
81
82/// Build an SSA value that represents the number of read elements.
83static Value buildNumReadElements(OpBuilder &b, Location loc,
84 Operation *readOp) {
85 FailureOr<TransferMask> transferMask = getMaskOp(loadOp: readOp);
86 assert(succeeded(transferMask) && "invalid transfer mask");
87
88 // No mask => no num_read_elements.
89 if (!transferMask->createMaskOp)
90 return Value();
91
92 // vector.create_mask treats a negative size as 0 and a size larger than the
93 // dimension as the dimension size. The async copy must not read more
94 // elements than it writes, so clamp the size of the "ones" segment in the
95 // same way.
96 auto clampToDimSize = [&](Value size) -> Value {
97 int64_t dimSize =
98 transferMask->createMaskOp.getVectorType().getShape().back();
99 if (std::optional<int64_t> cst = getConstantIntValue(ofr: size))
100 return arith::ConstantIndexOp::create(
101 builder&: b, location: loc, value: std::clamp(val: *cst, lo: int64_t(0), hi: dimSize));
102 Value zero = arith::ConstantIndexOp::create(builder&: b, location: loc, value: 0);
103 Value dim = arith::ConstantIndexOp::create(builder&: b, location: loc, value: dimSize);
104 Value nonNegative = arith::MaxSIOp::create(builder&: b, location: loc, lhs: size, rhs: zero);
105 return arith::MinSIOp::create(builder&: b, location: loc, lhs: nonNegative, rhs: dim);
106 };
107
108 // No extract: return size of "ones" segment in the mask.
109 if (transferMask->extractPosition.empty()) {
110 assert(transferMask->createMaskOp.getNumOperands() == 1 &&
111 "expected single operand");
112 return clampToDimSize(transferMask->createMaskOp.getOperand(i: 0));
113 }
114
115 // vector.extract(vector.create_mask).
116 // If extract_pos < num_ones, take number of elements from the least
117 // significant dimension. (Do this for all dimensions and bit-AND the
118 // conditions.)
119 assert(transferMask->createMaskOp.getVectorType().getRank() -
120 transferMask->extractPosition.size() ==
121 1 &&
122 "expected N-D -> (N-1)-D extract");
123 Value cond;
124 // Note: There is one more `sz` than `pos`. The loop end with the last `pos`.
125 for (auto [pos, sz] : llvm::zip(t&: transferMask->extractPosition,
126 u: transferMask->createMaskOp->getOperands())) {
127 Value cmp =
128 arith::CmpIOp::create(builder&: b, location: loc, predicate: arith::CmpIPredicate::slt,
129 lhs: arith::ConstantIndexOp::create(builder&: b, location: loc, value: pos), rhs: sz);
130 if (!cond) {
131 cond = cmp;
132 continue;
133 }
134 cond = arith::AndIOp::create(builder&: b, location: loc, lhs: cmp, rhs: cond);
135 }
136 return arith::SelectOp::create(
137 builder&: b, location: loc, condition: cond,
138 true_value: clampToDimSize(transferMask->createMaskOp->getOperands().back()),
139 false_value: arith::ConstantIndexOp::create(builder&: b, location: loc, value: 0));
140}
141
142/// Return "true" if the conversion to async copy is supported by "async copy".
143static bool resultsInSupportedAsyncCopy(MemRefType memrefType,
144 VectorType vecType) {
145 assert(vecType.getRank() == 1 && "expected 1-D vector");
146 constexpr int64_t kSupportedCpAsyncAlignmentsInBytes[3] = {4, 8, 16};
147
148 // Condition 1: the copy size must be supported.
149 bool supportedCopySize = false;
150 int64_t numElements = vecType.getNumElements();
151 Type elementType = vecType.getElementType();
152 for (int64_t alignmentInBytes : kSupportedCpAsyncAlignmentsInBytes) {
153 if (alignmentInBytes * 8 ==
154 numElements * elementType.getIntOrFloatBitWidth()) {
155 supportedCopySize = true;
156 break;
157 }
158 }
159 if (!supportedCopySize)
160 return false;
161
162 // TODO: Condition 2: the alignments must be supported. For cp.async the
163 // NVIDIA doc (section 6.4.1) says: "The address must be naturally aligned to
164 // a multiple of the access size. If an address is not properly aligned, the
165 // resulting behavior is undefined.".
166 return true;
167}
168
169void nvgpu::createAsyncGroups(RewriterBase &rewriter, Operation *op,
170 bool bypassL1) {
171 llvm::SmallSetVector<Operation *, 16> copyToSharedMem;
172
173 // Look for all the copy that can be converted to async copy ops.
174 op->walk(callback: [&](Operation *writeOp) {
175 // Look for contiguous 1D vector store into shared memory.
176 if (!isContiguousStore(write: writeOp))
177 return;
178 Value vectorVal = nvgpu::getValueStored(op: writeOp);
179 if (cast<VectorType>(Val: vectorVal.getType()).getRank() != 1)
180 return;
181 Value storeBase = nvgpu::getMemrefOperand(op: writeOp);
182 if (!nvgpu::NVGPUDialect::hasSharedMemoryAddressSpace(
183 type: cast<MemRefType>(Val: storeBase.getType())))
184 return;
185
186 // The stored vector must originate from a contiguous 1D vector load.
187 Operation *readOp = vectorVal.getDefiningOp();
188 if (readOp == nullptr || !isContiguousRead(read: readOp))
189 return;
190 Value loadBase = nvgpu::getMemrefOperand(op: readOp);
191 // Should be reading from global memory (not shared memory).
192 if (nvgpu::NVGPUDialect::hasSharedMemoryAddressSpace(
193 type: cast<MemRefType>(Val: loadBase.getType())))
194 return;
195
196 // Look for compatible mask and padding.
197 if (auto transferRead = dyn_cast<vector::TransferReadOp>(Val: readOp)) {
198 if (Value mask = transferRead.getMask()) {
199 if (getConstantIntValue(ofr: transferRead.getPadding()) ==
200 static_cast<int64_t>(0))
201 return;
202 if (failed(Result: getMaskOp(loadOp: readOp)))
203 return;
204 }
205 }
206
207 // Check whether both accesses are supported before we emit: this is
208 // necessary to ensure the correctness of DeviceAsyncCopyOp.
209 VectorType vecType = cast<VectorType>(Val: vectorVal.getType());
210
211 if (!resultsInSupportedAsyncCopy(memrefType: cast<MemRefType>(Val: loadBase.getType()),
212 vecType) ||
213 !resultsInSupportedAsyncCopy(memrefType: cast<MemRefType>(Val: storeBase.getType()),
214 vecType))
215 return;
216
217 copyToSharedMem.insert(X: writeOp);
218 return;
219 });
220
221 while (!copyToSharedMem.empty()) {
222 // Start a group with the first write.
223 SmallVector<Operation *> group;
224 Operation *writeOp = *copyToSharedMem.begin();
225 copyToSharedMem.remove(X: writeOp);
226 group.push_back(Elt: writeOp);
227 Operation *nextNode = writeOp;
228
229 // Look in the next nodes for more copies to add to the same group.
230 while ((nextNode = nextNode->getNextNode())) {
231 // Ignore ops without side effects.
232 auto memInterface = dyn_cast<MemoryEffectOpInterface>(Val: nextNode);
233 if (memInterface && memInterface.hasNoEffect() &&
234 !nextNode->hasTrait<OpTrait::HasRecursiveMemoryEffects>())
235 continue;
236 // Ignore read from a different address space.
237 if (isa<vector::TransferReadOp, vector::LoadOp>(Val: nextNode)) {
238 Operation *readOp = nextNode;
239 Value memrefOperand = nvgpu::getMemrefOperand(op: readOp);
240 if (!nvgpu::NVGPUDialect::hasSharedMemoryAddressSpace(
241 type: cast<MemRefType>(Val: memrefOperand.getType()))) {
242 continue;
243 }
244 }
245 if (copyToSharedMem.count(key: nextNode)) {
246 // Found another copy, add it to the group.
247 copyToSharedMem.remove(X: nextNode);
248 group.push_back(Elt: nextNode);
249 continue;
250 }
251 // If the op is something else stop the accumulating op in the group.
252 break;
253 }
254
255 // Emit the group.
256 SmallVector<Value> tokens;
257 for (Operation *writeOp : group) {
258 rewriter.setInsertionPoint(writeOp);
259 Value vectorVal = nvgpu::getValueStored(op: writeOp);
260 auto vectorType = cast<VectorType>(Val: vectorVal.getType());
261 int64_t numElements = vectorType.getNumElements();
262 Operation *readOp = vectorVal.getDefiningOp();
263 Value storeBase = nvgpu::getMemrefOperand(op: writeOp);
264 Value loadBase = nvgpu::getMemrefOperand(op: readOp);
265 Value numReadElements =
266 buildNumReadElements(b&: rewriter, loc: writeOp->getLoc(), readOp);
267 auto dstMemref = cast<MemRefType>(Val: storeBase.getType());
268 int64_t sizeInBytes =
269 (dstMemref.getElementTypeBitWidth() * numElements) / 8;
270 // bypass_l1 only possible with 16 byte transfer.
271 Value token = nvgpu::DeviceAsyncCopyOp::create(
272 builder&: rewriter, location: writeOp->getLoc(),
273 asyncToken: nvgpu::DeviceAsyncTokenType::get(ctx: op->getContext()),
274 /*dst=*/storeBase, /*dstIndices=*/nvgpu::getIndices(op: writeOp),
275 /*src=*/loadBase,
276 /*srcIndices=*/nvgpu::getIndices(op: readOp),
277 /*dstElements=*/rewriter.getIndexAttr(value: numElements),
278 /*srcElements=*/numReadElements,
279 /*bypassL1=*/bypassL1 && sizeInBytes == 16 ? rewriter.getUnitAttr()
280 : UnitAttr());
281 tokens.push_back(Elt: token);
282 }
283
284 // Create the group and wait for it right after.
285 Value groupToken = nvgpu::DeviceAsyncCreateGroupOp::create(
286 builder&: rewriter, location: op->getLoc(),
287 asyncToken: nvgpu::DeviceAsyncTokenType::get(ctx: op->getContext()), inputTokens: tokens);
288 nvgpu::DeviceAsyncWaitOp::create(builder&: rewriter, location: op->getLoc(), asyncDependencies: groupToken,
289 numGroups: nullptr);
290 // Clean up old stores.
291 for (Operation *writeOp : group)
292 rewriter.eraseOp(op: writeOp);
293 }
294}
295