1//===- NVGPUTransformOps.cpp - Implementation of NVGPU transform ops ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/NVGPU/TransformOps/NVGPUTransformOps.h"
10
11#include "mlir/Analysis/SliceAnalysis.h"
12#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
13#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
14#include "mlir/Conversion/NVGPUToNVVM/NVGPUToNVVM.h"
15#include "mlir/Dialect/Affine/IR/AffineOps.h"
16#include "mlir/Dialect/Arith/IR/Arith.h"
17#include "mlir/Dialect/Arith/Utils/Utils.h"
18#include "mlir/Dialect/GPU/IR/GPUDialect.h"
19#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
20#include "mlir/Dialect/LLVMIR/NVVMDialectDecl.h"
21#include "mlir/Dialect/Linalg/IR/Linalg.h"
22#include "mlir/Dialect/MemRef/IR/MemRef.h"
23#include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
24#include "mlir/Dialect/NVGPU/Transforms/Transforms.h"
25#include "mlir/Dialect/SCF/IR/SCF.h"
26#include "mlir/Dialect/SCF/Transforms/Transforms.h"
27#include "mlir/Dialect/Utils/IndexingUtils.h"
28#include "mlir/Dialect/Utils/StaticValueUtils.h"
29#include "mlir/Dialect/Vector/IR/VectorOps.h"
30#include "mlir/IR/AffineExpr.h"
31#include "mlir/IR/BuiltinTypes.h"
32#include "mlir/IR/Value.h"
33#include "llvm/ADT/ArrayRef.h"
34
35using namespace mlir;
36using namespace mlir::linalg;
37using namespace mlir::nvgpu;
38using namespace mlir::NVVM;
39using namespace mlir::transform;
40
41#define DEBUG_TYPE "nvgpu-transforms"
42
43//===----------------------------------------------------------------------===//
44// Apply...ConversionPatternsOp
45//===----------------------------------------------------------------------===//
46
47void ApplyNVGPUToNVVMConversionPatternsOp::populatePatterns(
48 TypeConverter &typeConverter, RewritePatternSet &patterns) {
49 auto &llvmTypeConverter = static_cast<LLVMTypeConverter &>(typeConverter);
50 /// device-side async tokens cannot be materialized in nvvm. We just
51 /// convert them to a dummy i32 type in order to easily drop them during
52 /// conversion.
53 nvgpu::populateCommonGPUTypeAndAttributeConversions(typeConverter&: llvmTypeConverter);
54 llvmTypeConverter.addConversion(callback: [&](DeviceAsyncTokenType type) -> Type {
55 return llvmTypeConverter.convertType(
56 t: IntegerType::get(context: type.getContext(), width: 32));
57 });
58 llvmTypeConverter.addConversion(callback: [&](MBarrierTokenType type) -> Type {
59 return llvmTypeConverter.convertType(
60 t: IntegerType::get(context: type.getContext(), width: 64));
61 });
62 llvmTypeConverter.addConversion(callback: [&](WarpgroupAccumulatorType type) -> Type {
63 Type elemType = type.getFragmented().getElementType();
64 int64_t sizeM = type.getFragmented().getDimSize(idx: 0);
65 int64_t sizeN = type.getFragmented().getDimSize(idx: 1);
66
67 unsigned numMembers;
68 if (elemType.isF32() || elemType.isInteger(width: 32))
69 numMembers = sizeN / 2;
70 else if (elemType.isF16())
71 numMembers = sizeN / 4;
72 else
73 llvm_unreachable("unsupported type for warpgroup accumulator");
74
75 SmallVector<Type> innerStructBody;
76 for (unsigned i = 0; i < numMembers; i++)
77 innerStructBody.push_back(Elt: elemType);
78 auto innerStructType =
79 LLVM::LLVMStructType::getLiteral(context: type.getContext(), types: innerStructBody);
80
81 SmallVector<Type> structBody;
82 for (int i = 0; i < sizeM; i += kWgmmaSizeM)
83 structBody.push_back(Elt: innerStructType);
84
85 auto convertedType =
86 LLVM::LLVMStructType::getLiteral(context: type.getContext(), types: structBody);
87 return llvmTypeConverter.convertType(t: convertedType);
88 });
89 llvmTypeConverter.addConversion(callback: [&](MBarrierGroupType type) -> Type {
90 return llvmTypeConverter.convertType(
91 t: getMBarrierMemrefType(context: type.getContext(), barrierType: type));
92 });
93 llvmTypeConverter.addConversion(
94 callback: [&](WarpgroupMatrixDescriptorType type) -> Type {
95 return llvmTypeConverter.convertType(
96 t: IntegerType::get(context: type.getContext(), width: 64));
97 });
98 llvmTypeConverter.addConversion(callback: [&](TensorMapDescriptorType type) -> Type {
99 return LLVM::LLVMPointerType::get(context: type.getContext());
100 });
101 populateNVGPUToNVVMConversionPatterns(converter: llvmTypeConverter, patterns);
102}
103
104LogicalResult ApplyNVGPUToNVVMConversionPatternsOp::verifyTypeConverter(
105 TypeConverterBuilderOpInterface builder) {
106 if (builder.getTypeConverterType() != "LLVMTypeConverter")
107 return emitOpError(message: "expected LLVMTypeConverter");
108 return success();
109}
110
111//===---------------------------------------------------------------------===//
112// CreateAsyncGroupsOp
113//===---------------------------------------------------------------------===//
114
115void CreateAsyncGroupsOp::getEffects(
116 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
117 consumesHandle(handles: getTargetMutable(), effects);
118 producesHandle(handles: getOperation()->getOpResults(), effects);
119 modifiesPayload(effects);
120}
121
122DiagnosedSilenceableFailure
123CreateAsyncGroupsOp::applyToOne(TransformRewriter &rewriter, Operation *target,
124 ApplyToEachResultList &results,
125 TransformState &state) {
126 createAsyncGroups(rewriter, op: target, bypassL1: getBypassL1());
127 results.push_back(op: target);
128 return DiagnosedSilenceableFailure::success();
129}
130
131//===----------------------------------------------------------------------===//
132// PipelineSharedMemoryCopiesOp
133//===----------------------------------------------------------------------===//
134
135/// Returns true if the given type has the default memory space.
136static bool hasDefaultMemorySpace(BaseMemRefType type) {
137 return !type.getMemorySpace() || type.getMemorySpaceAsInt() == 0;
138}
139
140/// Returns true if the given type has the shared (workgroup) memory space.
141static bool hasSharedMemorySpace(BaseMemRefType type) {
142 auto space =
143 dyn_cast_if_present<gpu::AddressSpaceAttr>(Val: type.getMemorySpace());
144 return space &&
145 space.getValue() == gpu::GPUDialect::getWorkgroupAddressSpace();
146}
147
148/// Returns the value produced by a load from the default memory space. Returns
149/// null if the operation is not such a load.
150static Value getValueLoadedFromGlobal(Operation *op) {
151 // TODO: consider an interface or leveraging the memory effects interface.
152 auto load = dyn_cast<vector::TransferReadOp>(Val: op);
153 if (!load)
154 return nullptr;
155
156 auto loadType = dyn_cast<MemRefType>(Val: load.getBase().getType());
157 if (!loadType || !hasDefaultMemorySpace(type: loadType))
158 return nullptr;
159 return load;
160}
161
162/// Returns true if the operation is storing the given value into shared memory.
163static bool isStoreToShared(Operation *op, Value v) {
164 // TOD: consider an interface or leveraging the memory effects interface.
165 auto store = dyn_cast<vector::TransferWriteOp>(Val: op);
166 if (!store || store.getVector() != v)
167 return false;
168
169 auto storeType = dyn_cast<MemRefType>(Val: store.getBase().getType());
170 return storeType || hasSharedMemorySpace(type: storeType);
171}
172
173/// Returns true if the operation is a load from the default memory space the
174/// result of which is only stored into the shared memory space.
175static bool isLoadFromGlobalStoredToShared(Operation *op) {
176 Value loaded = getValueLoadedFromGlobal(op);
177 if (!loaded || !loaded.hasOneUse())
178 return false;
179
180 return isStoreToShared(op: *loaded.getUsers().begin(), v: loaded);
181}
182
183/// Populate `ops` with the set of operations that belong to the stage 0 of the
184/// pipelined version of the given loop when pipelining copies to shared memory.
185/// Specifically, this collects:
186///
187/// 1. all loads from global memory, both sync and async;
188/// 2. the barriers for async loads.
189///
190/// In particular, barriers are omitted if they do not dominate at least one
191/// async load for which there is not yet a barrier.
192static LogicalResult
193collectStage0PipeliningOps(scf::ForOp forOp,
194 llvm::SmallPtrSet<Operation *, 16> &ops) {
195
196 llvm::SmallPtrSet<Operation *, 4> barriers;
197 for (Operation &op : *forOp.getBody()) {
198 // Bail on nested ops for now.
199 if (op.getNumRegions() > 0)
200 return failure();
201
202 if (isa<gpu::BarrierOp>(Val: op)) {
203 barriers.insert(Ptr: &op);
204 continue;
205 }
206
207 if (isa<DeviceAsyncCopyOp, DeviceAsyncCreateGroupOp>(Val: op)) {
208 ops.insert(Ptr: &op);
209 ops.insert(I: std::make_move_iterator(i: barriers.begin()),
210 E: std::make_move_iterator(i: barriers.end()));
211 assert(barriers.empty() &&
212 "expected to have moved the barriers into another set");
213 continue;
214 }
215
216 if (isLoadFromGlobalStoredToShared(op: &op)) {
217 ops.insert(Ptr: &op);
218 continue;
219 }
220 }
221
222 return success();
223}
224
225/// Hook for the loop pipeliner that sets the "num groups in flight" attribute
226/// of async wait operations corresponding to pipelined shared memory copies.
227// TODO: this currently assumes that there are no groups that could be in flight
228// in the existing code.
229static void
230setAsyncWaitGroupsInFlight(OpBuilder &builder, Operation *op,
231 scf::PipeliningOption::PipelinerPart part,
232 unsigned iteration, unsigned depth) {
233 // Based on the order of copies within the loop we need to set the number
234 // of copies in flight, unless it is already set.
235 auto waitOp = dyn_cast<DeviceAsyncWaitOp>(Val: op);
236 if (!waitOp || waitOp.getNumGroups())
237 return;
238
239 int numGroupInFlight = 0;
240 if (part == scf::PipeliningOption::PipelinerPart::Kernel ||
241 part == scf::PipeliningOption::PipelinerPart::Prologue) {
242 numGroupInFlight = depth - 1;
243 } else {
244 // By construction there should be no wait op in the prologue as all the
245 // wait should be in the last stage.
246 assert(part == scf::PipeliningOption::PipelinerPart::Epilogue);
247 // Based on the schedule we pick we know how many groups are in flight for
248 // each iteration of the epilogue.
249 numGroupInFlight = depth - 1 - iteration;
250 }
251 waitOp.setNumGroups(numGroupInFlight);
252}
253
254/// Hook for the loop pipeliner that populates `ops` with the stage information
255/// as follows:
256///
257/// - operations in `stage0Ops` (typically loads from global memory and
258/// related barriers) are at stage 0;
259/// - operations in the backward slice of any stage0Ops are all at stage 0;
260/// - other operations are at stage `depth`;
261/// - the internal order of the pipelined loop has ops at stage `depth` first,
262/// then those at stage 0, with relative order within each group preserved.
263///
264static void getPipelineStages(
265 scf::ForOp forOp,
266 std::vector<std::pair<Operation *, unsigned>> &opsWithPipelineStages,
267 unsigned depth, llvm::SmallPtrSetImpl<Operation *> &stage0Ops) {
268 SetVector<Operation *> dependencies;
269 BackwardSliceOptions options([&](Operation *visited) {
270 return visited->getBlock() == forOp.getBody();
271 });
272 options.inclusive = true;
273 for (Operation &op : forOp.getBody()->getOperations()) {
274 if (stage0Ops.contains(Ptr: &op)) {
275 LogicalResult result = getBackwardSlice(op: &op, backwardSlice: &dependencies, options);
276 assert(result.succeeded() && "expected a backward slice");
277 (void)result;
278 }
279 }
280
281 for (Operation &op : forOp.getBody()->getOperations()) {
282 if (!dependencies.contains(key: &op) && !isa<scf::YieldOp>(Val: op))
283 opsWithPipelineStages.emplace_back(args: &op, args&: depth);
284 }
285 for (Operation &op : forOp.getBody()->getOperations()) {
286 if (dependencies.contains(key: &op))
287 opsWithPipelineStages.emplace_back(args: &op, args: 0);
288 }
289}
290
291/// Hook for the loop pipeliner. Replaces op with a predicated version and
292/// returns the resulting operation. Returns the original op if the predication
293/// isn't necessary for the given op. Returns null if predication is needed but
294/// not supported.
295static Operation *replaceOpWithPredicatedOp(RewriterBase &rewriter,
296 Operation *op, Value predicate) {
297 // Some operations may be fine to execute "speculatively" more times than the
298 // original number of iterations, in particular side-effect free operations
299 // and barriers, even if they cannot be predicated.
300 if (isMemoryEffectFree(op) ||
301 isa<gpu::BarrierOp, DeviceAsyncCreateGroupOp, DeviceAsyncWaitOp>(Val: op)) {
302 return op;
303 }
304
305 // Otherwise, only async copies can currently be predicated.
306 auto asyncCopyOp = dyn_cast<DeviceAsyncCopyOp>(Val: op);
307 if (!asyncCopyOp)
308 return nullptr;
309
310 // Create srcElement Value based on `predicate`. The next lines generate
311 // the following code:
312 //
313 // srcElement = (pred) ? prevSrcElements : 0;
314 //
315 Location loc = asyncCopyOp->getLoc();
316 Value dstElements = arith::ConstantOp::create(
317 builder&: rewriter, location: loc, value: asyncCopyOp.getDstElementsAttr());
318 Value originalSrcElement =
319 asyncCopyOp.getSrcElements() ? asyncCopyOp.getSrcElements() : dstElements;
320 Value c0Index = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
321 auto srcElements = arith::SelectOp::create(builder&: rewriter, location: loc, condition: predicate,
322 true_value: originalSrcElement, false_value: c0Index);
323 auto asyncCopyZeroFillOp = DeviceAsyncCopyOp::create(
324 builder&: rewriter, location: loc, asyncToken: DeviceAsyncTokenType::get(ctx: asyncCopyOp.getContext()),
325 dst: asyncCopyOp.getDst(), dstIndices: asyncCopyOp.getDstIndices(), src: asyncCopyOp.getSrc(),
326 srcIndices: asyncCopyOp.getSrcIndices(), dstElements: asyncCopyOp.getDstElements(), srcElements,
327 bypassL1: UnitAttr());
328 rewriter.replaceOp(op: asyncCopyOp, newOp: asyncCopyZeroFillOp);
329 return asyncCopyZeroFillOp;
330}
331
332/// Applies loop pipelining with the given depth to the given loop so that
333/// copies into the shared memory are pipelined. Doesn't affect other loops.
334/// Returns a pair containing the error state and the pipelined op, the latter
335/// being null in case of any failure. The error state contains a definite error
336/// if the IR has been modified and a silenceable error otherwise.
337static std::tuple<DiagnosedSilenceableFailure, scf::ForOp>
338pipelineForSharedCopies(RewriterBase &rewriter, scf::ForOp forOp, int64_t depth,
339 bool epiloguePeeling) {
340 llvm::SmallPtrSet<Operation *, 16> stage0Ops;
341 if (failed(Result: collectStage0PipeliningOps(forOp, ops&: stage0Ops))) {
342 return std::make_tuple(
343 args: emitSilenceableFailure(op: forOp, message: "cannot find stage 0 ops for pipelining"),
344 args: scf::ForOp());
345 }
346 if (stage0Ops.empty()) {
347 return std::make_tuple(
348 args: emitSilenceableFailure(op: forOp, message: "no shared memory copy"), args: scf::ForOp());
349 }
350
351 scf::PipeliningOption options;
352 unsigned maxDepth = depth;
353 auto setAnnotation = [&](Operation *op,
354 scf::PipeliningOption::PipelinerPart part,
355 unsigned iteration) {
356 return setAsyncWaitGroupsInFlight(builder&: rewriter, op, part, iteration, depth: maxDepth);
357 };
358 options.getScheduleFn =
359 [&](scf::ForOp schedulingFor,
360 std::vector<std::pair<Operation *, unsigned>> &ops) {
361 if (schedulingFor != forOp)
362 return;
363 return getPipelineStages(forOp, opsWithPipelineStages&: ops, depth: maxDepth, stage0Ops);
364 };
365 options.annotateFn = setAnnotation;
366 if (!epiloguePeeling) {
367 options.peelEpilogue = false;
368 options.predicateFn = replaceOpWithPredicatedOp;
369 }
370
371 OpBuilder::InsertionGuard guard(rewriter);
372 rewriter.setInsertionPoint(forOp);
373 bool modifiedIR;
374 FailureOr<scf::ForOp> maybePipelined =
375 pipelineForLoop(rewriter, forOp, options, modifiedIR: &modifiedIR);
376 if (succeeded(Result: maybePipelined)) {
377 return std::make_tuple(args: DiagnosedSilenceableFailure::success(),
378 args&: *maybePipelined);
379 }
380 return std::make_tuple(
381 args: modifiedIR
382 ? DiagnosedSilenceableFailure::definiteFailure()
383 : emitSilenceableFailure(op: forOp, message: "pipelining preconditions failed"),
384 args: scf::ForOp());
385}
386
387DiagnosedSilenceableFailure PipelineSharedMemoryCopiesOp::applyToOne(
388 TransformRewriter &rewriter, scf::ForOp forOp,
389 ApplyToEachResultList &results, TransformState &state) {
390 auto [diag, pipelined] = pipelineForSharedCopies(
391 rewriter, forOp, depth: static_cast<int64_t>(getDepth()), epiloguePeeling: getPeelEpilogue());
392 if (diag.succeeded()) {
393 results.push_back(op: pipelined);
394 return DiagnosedSilenceableFailure::success();
395 }
396 if (diag.isDefiniteFailure()) {
397 auto diag = emitDefiniteFailure(message: "irreversible pipelining failure");
398 if (!getPeelEpilogue()) {
399 diag.attachNote(loc: forOp->getLoc()) << "couldn't predicate?";
400 diag.attachNote(loc: getLoc()) << "try setting " << getPeelEpilogueAttrName();
401 }
402 return diag;
403 }
404
405 return std::move(diag);
406}
407
408//===----------------------------------------------------------------------===//
409// RewriteMatmulAsMmaSyncOp
410//===----------------------------------------------------------------------===//
411
412/// Helper struct to encode a pair of row/column indexings in the form of
413/// affine expressions.
414struct RowColIndexing : private std::pair<AffineExpr, AffineExpr> {
415 RowColIndexing(AffineExpr row, AffineExpr col)
416 : std::pair<AffineExpr, AffineExpr>(row, col) {}
417
418 AffineExpr row() const { return first; };
419 AffineExpr col() const { return second; };
420
421 void print(llvm::raw_ostream &os) const {
422 os << "- indexing: " << first << ", " << second;
423 }
424};
425
426/// Helper struct to provide a simple mapping from matmul operations to the
427/// corresponding mma.sync operation. This is constrained to the case where the
428/// matmul matches the mma.sync operation 1-1.
429struct MmaSyncBuilder {
430 MmaSyncBuilder(OpBuilder &b, Location loc, OpFoldResult laneId)
431 : b(b), loc(loc), laneId(laneId) {}
432
433 using IndexCalculator =
434 std::function<SmallVector<RowColIndexing>(MLIRContext *)>;
435
436 /// Create the mma.sync operation corresponding to `linalgOp` along with all
437 /// the supporting load/store and vector operations.
438 FailureOr<Operation *> buildMmaSync(LinalgOp linalgOp);
439
440private:
441 struct MmaSyncInfo {
442 std::tuple<IndexCalculator, IndexCalculator, IndexCalculator> indexFns;
443 std::tuple<SmallVector<int64_t>, SmallVector<int64_t>, SmallVector<int64_t>>
444 vectorShapes;
445 SmallVector<int64_t> mmaShape;
446 bool tf32Enabled;
447 };
448
449 /// Return the specific index calculator for the given `linalgOp` or failure
450 /// if the op is not supported. This is the toplevel switch that should just
451 /// be Tablegen'd in the future.
452 FailureOr<MmaSyncInfo> getIndexCalculators(ArrayRef<int64_t> opShape,
453 TypeRange elementalTypes);
454
455 //===--------------------------------------------------------------------===//
456 // Instruction-specific row, column indexing expression builders.
457 // These should all be declaratively specified via Tablegen in the future.
458 // The Tablegen specification should be as straightforward as possible to
459 // only model the existing size and type combinations.
460 //===--------------------------------------------------------------------===//
461 //
462 // TODO: Tablegen all this.
463 //===--------------------------------------------------------------------===//
464 // m16n8k4 tf32 case.
465 //===--------------------------------------------------------------------===//
466 /// From the NVIDIA doc:
467 /// groupID = %laneid >> 2
468 /// threadIDInGroup = %laneid % 4
469 /// row = groupID for a0
470 /// groupID + 8 for a1
471 /// col = threadIDInGroup
472 static SmallVector<RowColIndexing> m16n8k4tf32Lhs(MLIRContext *ctx) {
473 auto dim = getAffineDimExpr(position: 0, context: ctx);
474 AffineExpr groupID = dim.floorDiv(v: 4);
475 AffineExpr threadIDInGroup = dim % 4;
476 return {RowColIndexing{groupID, threadIDInGroup},
477 RowColIndexing{groupID + 8, threadIDInGroup}};
478 }
479
480 /// From the NVIDIA doc:
481 /// groupID = %laneid >> 2
482 /// threadIDInGroup = %laneid % 4
483 /// row = threadIDInGroup
484 /// col = groupID
485 static SmallVector<RowColIndexing> m16n8k4tf32Rhs(MLIRContext *ctx) {
486 auto dim = getAffineDimExpr(position: 0, context: ctx);
487 AffineExpr groupID = dim.floorDiv(v: 4);
488 AffineExpr threadIDInGroup = dim % 4;
489 return {RowColIndexing{threadIDInGroup, groupID}};
490 }
491
492 /// From the NVIDIA doc:
493 /// groupID = %laneid >> 2
494 /// threadIDInGroup = %laneid % 4
495 /// row = groupID for c0 and c1
496 /// groupID + 8 for c2 and c3
497 /// col = (threadIDInGroup * 2) + (i & 0x1) for ci where i = {0,..,3}
498 static SmallVector<RowColIndexing> m16n8k4tf32Res(MLIRContext *ctx) {
499 auto dim = getAffineDimExpr(position: 0, context: ctx);
500 AffineExpr groupID = dim.floorDiv(v: 4);
501 AffineExpr threadIDInGroup = dim % 4;
502 return {RowColIndexing{groupID, threadIDInGroup * 2 + 0},
503 RowColIndexing{groupID, threadIDInGroup * 2 + 1},
504 RowColIndexing{groupID + 8, threadIDInGroup * 2 + 0},
505 RowColIndexing{groupID + 8, threadIDInGroup * 2 + 1}};
506 }
507
508 //===--------------------------------------------------------------------===//
509 // m16n8k16 f16 case.
510 //===--------------------------------------------------------------------===//
511 /// From the NVIDIA doc:
512 /// groupID = %laneid >> 2
513 /// threadIDInGroup = %laneid % 4
514 ///
515 /// row = groupID for ai where 0 <= i < 2 || 4 <= i < 6
516 /// groupID + 8 Otherwise
517 ///
518 /// col = (threadIDInGroup * 2) + (i & 0x1) for ai where i < 4
519 /// (threadIDInGroup * 2) + (i & 0x1) + 8 for ai where i >= 4
520 static SmallVector<RowColIndexing> m16n8k16f16Lhs(MLIRContext *ctx) {
521 auto dim = getAffineDimExpr(position: 0, context: ctx);
522 AffineExpr groupID = dim.floorDiv(v: 4);
523 AffineExpr threadIDInGroup = dim % 4;
524 // clang-format off
525 return {
526 RowColIndexing{groupID, threadIDInGroup * 2 + 0}, // i == 0
527 RowColIndexing{groupID, threadIDInGroup * 2 + 1}, // i == 1
528 RowColIndexing{groupID + 8, threadIDInGroup * 2 + 0}, // i == 2
529 RowColIndexing{groupID + 8, threadIDInGroup * 2 + 1}, // i == 3
530 RowColIndexing{groupID, threadIDInGroup * 2 + 0 + 8}, // i == 4
531 RowColIndexing{groupID, threadIDInGroup * 2 + 1 + 8}, // i == 5
532 RowColIndexing{groupID + 8, threadIDInGroup * 2 + 0 + 8}, // i == 6
533 RowColIndexing{groupID + 8, threadIDInGroup * 2 + 1 + 8} // i == 7
534 };
535 // clang-format on
536 }
537
538 /// From the NVIDIA doc:
539 /// groupID = %laneid >> 2
540 /// threadIDInGroup = %laneid % 4
541 ///
542 /// row = (threadIDInGroup * 2) + (i & 0x1) for bi where i < 2
543 /// (threadIDInGroup * 2) + (i & 0x1) + 8 for bi where i >= 2
544 ///
545 /// col = groupID
546 static SmallVector<RowColIndexing> m16n8k16f16Rhs(MLIRContext *ctx) {
547 auto dim = getAffineDimExpr(position: 0, context: ctx);
548 AffineExpr groupID = dim.floorDiv(v: 4);
549 AffineExpr threadIDInGroup = dim % 4;
550 // clang-format off
551 return {
552 RowColIndexing{threadIDInGroup * 2 + 0, groupID}, // i == 0
553 RowColIndexing{threadIDInGroup * 2 + 1, groupID}, // i == 1
554 RowColIndexing{threadIDInGroup * 2 + 0 + 8, groupID}, // i == 2
555 RowColIndexing{threadIDInGroup * 2 + 1 + 8, groupID} // i == 3
556 };
557 // clang-format on
558 }
559
560 /// From the NVIDIA doc:
561 /// groupID = %laneid >> 2
562 /// threadIDInGroup = %laneid % 4
563 ///
564 /// row = groupID for ci where i < 2
565 /// groupID + 8 for ci where i >= 2
566 ///
567 /// col = (threadIDInGroup * 2) + (i & 0x1) for ci where i = {0,..,3}
568 static SmallVector<RowColIndexing> m16n8k16f16Res(MLIRContext *ctx) {
569 auto dim = getAffineDimExpr(position: 0, context: ctx);
570 AffineExpr groupID = dim.floorDiv(v: 4);
571 AffineExpr threadIDInGroup = dim % 4;
572 // clang-format off
573 return {
574 RowColIndexing{groupID, threadIDInGroup * 2 + 0}, // i == 0
575 RowColIndexing{groupID, threadIDInGroup * 2 + 1}, // i == 1
576 RowColIndexing{groupID + 8, threadIDInGroup * 2 + 0}, // i == 2
577 RowColIndexing{groupID + 8, threadIDInGroup * 2 + 1} // i == 3
578 };
579 // clang-format on
580 }
581
582 //===--------------------------------------------------------------------===//
583 /// Helper functions to create customizable load and stores operations. The
584 /// specific shapes of each MMA instruction are passed via the
585 /// IndexCalculator callback.
586 //===--------------------------------------------------------------------===//
587 /// Build a list of memref.load operations indexed at `(row, col)` indices
588 /// that make sense for a particular MMA instruction and specified via the
589 /// IndexCalculator callback.
590 SmallVector<Value> buildMemRefLoads(OpBuilder &b, Location loc,
591 OpFoldResult laneId, Value memref,
592 const IndexCalculator &indexFn);
593
594 /// Perform a distributed load of a vector operand of `vectorShape` for a
595 /// particular MMA instruction whose `(row, col)` indices are specified via
596 /// the IndexCalculator callback. Each `laneId` loads the subportion of the
597 /// data that makes sense for the particular MMA operation.
598 /// The `vectorShape` matches existing NVGPU dialect op specification but
599 /// could also be flattened in the future if needed for simplification.
600 Value buildMmaSyncMemRefLoadOperand(OpBuilder &b, Location loc,
601 OpFoldResult laneId, Value memref,
602 IndexCalculator indexFn,
603 ArrayRef<int64_t> vectorShape);
604
605 /// Build a list of memref.store operations indexed at `(row, col)` indices
606 /// that make sense for a particular MMA instruction and specified via the
607 /// IndexCalculator callback.
608 SmallVector<Operation *> buildMemRefStores(OpBuilder &b, Location loc,
609 ValueRange toStore,
610 OpFoldResult laneId, Value memref,
611 const IndexCalculator &indexFn);
612
613 /// Perform a distributed store of a vector operand of `vectorShape` for a
614 /// particular MMA instruction whose `(row, col)` indices are specified via
615 /// the IndexCalculator callback. Each `laneId` loads the subportion of the
616 /// data that makes sense for the particular MMA operation.
617 /// The `vectorShape` matches existing NVGPU dialect op specification but
618 /// could also be flattened in the future if needed for simplification.
619 SmallVector<Operation *> buildMmaSyncMemRefStoreOperand(
620 OpBuilder &b, Location loc, Value vectorToStore, OpFoldResult laneId,
621 Value memref, IndexCalculator indexFn, ArrayRef<int64_t> vectorShape);
622
623 OpBuilder &b;
624 Location loc;
625 OpFoldResult laneId;
626};
627
628//===--------------------------------------------------------------------===//
629/// Helper functions to create customizable load and stores operations. The
630/// specific shapes of each MMA instruction are passed via the
631/// IndexCalculator callback.
632//===--------------------------------------------------------------------===//
633
634template <typename ApplyFn, typename ReduceFn>
635static void foreachIndividualVectorElement(Value vector, ApplyFn applyFn,
636 ReduceFn reduceFn) {
637 VectorType vectorType = cast<VectorType>(Val: vector.getType());
638 auto vectorShape = vectorType.getShape();
639 auto strides = computeStrides(sizes: vectorShape);
640 for (int64_t idx = 0, e = vectorShape[0] * strides[0]; idx < e; ++idx) {
641 auto indices = delinearize(linearIndex: idx, strides);
642 reduceFn(applyFn(vector, idx, indices), idx, indices);
643 }
644}
645
646SmallVector<Value>
647MmaSyncBuilder::buildMemRefLoads(OpBuilder &b, Location loc,
648 OpFoldResult laneId, Value memref,
649 const IndexCalculator &indexFn) {
650 auto aff = [&](AffineExpr e) {
651 return affine::makeComposedFoldedAffineApply(b, loc, expr: e, operands: laneId);
652 };
653 SmallVector<Value> res;
654 SmallVector<RowColIndexing> indexings = indexFn(b.getContext());
655 for (auto indexing : indexings) {
656 Value row = getValueOrCreateConstantIndexOp(b, loc, ofr: aff(indexing.row()));
657 Value col = getValueOrCreateConstantIndexOp(b, loc, ofr: aff(indexing.col()));
658 auto load = memref::LoadOp::create(builder&: b, location: loc, memref, indices: ValueRange{row, col});
659 res.push_back(Elt: load);
660 }
661 return res;
662}
663
664Value MmaSyncBuilder::buildMmaSyncMemRefLoadOperand(
665 OpBuilder &b, Location loc, OpFoldResult laneId, Value memref,
666 IndexCalculator indexFn, ArrayRef<int64_t> vectorShape) {
667 auto loads = buildMemRefLoads(b, loc, laneId, memref, indexFn);
668
669 Type elementType = getElementTypeOrSelf(type: memref.getType());
670 auto vt = VectorType::get(shape: vectorShape, elementType);
671 Value res = vector::BroadcastOp::create(builder&: b, location: loc, result: vt, source: loads[0]);
672 foreachIndividualVectorElement(
673 vector: res,
674 /*applyFn=*/
675 [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) {
676 return loads[linearIdx];
677 },
678 /*reduceFn=*/
679 [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) {
680 res = vector::InsertOp::create(builder&: b, location: loc, valueToStore: v, dest: res, position: indices);
681 });
682
683 return res;
684}
685
686SmallVector<Operation *> MmaSyncBuilder::buildMemRefStores(
687 OpBuilder &b, Location loc, ValueRange toStore, OpFoldResult laneId,
688 Value memref, const IndexCalculator &indexFn) {
689 auto aff = [&](AffineExpr e) {
690 return affine::makeComposedFoldedAffineApply(b, loc, expr: e, operands: laneId);
691 };
692 SmallVector<Operation *> res;
693 for (auto [indexing, val] :
694 llvm::zip_equal(t: indexFn(b.getContext()), u&: toStore)) {
695 Value row = getValueOrCreateConstantIndexOp(b, loc, ofr: aff(indexing.row()));
696 Value col = getValueOrCreateConstantIndexOp(b, loc, ofr: aff(indexing.col()));
697 Operation *store =
698 memref::StoreOp::create(builder&: b, location: loc, valueToStore: val, memref, indices: ValueRange{row, col});
699 res.push_back(Elt: store);
700 }
701 return res;
702}
703
704SmallVector<Operation *> MmaSyncBuilder::buildMmaSyncMemRefStoreOperand(
705 OpBuilder &b, Location loc, Value vectorToStore, OpFoldResult laneId,
706 Value memref, IndexCalculator indexFn, ArrayRef<int64_t> vectorShape) {
707 SmallVector<Value> toStore;
708 toStore.reserve(N: 32);
709 foreachIndividualVectorElement(
710 vector: vectorToStore,
711 /*applyFn=*/
712 [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) {
713 return vector::ExtractOp::create(builder&: b, location: loc, source: vectorToStore, position: indices);
714 },
715 /*reduceFn=*/
716 [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) {
717 toStore.push_back(Elt: v);
718 });
719 return buildMemRefStores(b, loc, toStore, laneId, memref, indexFn);
720}
721
722static std::tuple<SmallVector<int64_t>, SmallVector<int64_t>,
723 SmallVector<int64_t>>
724makeVectorShapes(ArrayRef<int64_t> lhs, ArrayRef<int64_t> rhs,
725 ArrayRef<int64_t> res) {
726 SmallVector<int64_t> vlhs(lhs);
727 SmallVector<int64_t> vrhs(rhs);
728 SmallVector<int64_t> vres(res);
729 return std::make_tuple(args&: vlhs, args&: vrhs, args&: vres);
730}
731
732FailureOr<MmaSyncBuilder::MmaSyncInfo>
733MmaSyncBuilder::getIndexCalculators(ArrayRef<int64_t> opShape,
734 TypeRange elementalTypes) {
735 // TODO: Tablegen all this.
736 Type f16 = b.getF16Type();
737 Type f32 = b.getF32Type();
738 if (opShape == ArrayRef<int64_t>{16, 8, 4} &&
739 elementalTypes == TypeRange{f32, f32, f32}) {
740 return MmaSyncInfo{.indexFns: std::make_tuple(args: &MmaSyncBuilder::m16n8k4tf32Lhs,
741 args: &MmaSyncBuilder::m16n8k4tf32Rhs,
742 args: &MmaSyncBuilder::m16n8k4tf32Res),
743 .vectorShapes: makeVectorShapes(lhs: {2, 1}, rhs: {1, 1}, res: {2, 2}),
744 .mmaShape: SmallVector<int64_t>{opShape},
745 /*tf32Enabled=*/true};
746 }
747 // This is the version with f16 accumulation.
748 // TODO: version with f32 accumulation.
749 if (opShape == ArrayRef<int64_t>{16, 8, 16} &&
750 elementalTypes == TypeRange{f16, f16, f16}) {
751 return MmaSyncInfo{.indexFns: std::make_tuple(args: &MmaSyncBuilder::m16n8k16f16Lhs,
752 args: &MmaSyncBuilder::m16n8k16f16Rhs,
753 args: &MmaSyncBuilder::m16n8k16f16Res),
754 .vectorShapes: makeVectorShapes(lhs: {4, 2}, rhs: {2, 2}, res: {2, 2}),
755 .mmaShape: SmallVector<int64_t>{opShape},
756 /*tf32Enabled=*/false};
757 }
758 return failure();
759}
760
761FailureOr<Operation *> MmaSyncBuilder::buildMmaSync(LinalgOp linalgOp) {
762 Value lhsMemRef = linalgOp.getDpsInputOperand(i: 0)->get();
763 Value rhsMemRef = linalgOp.getDpsInputOperand(i: 1)->get();
764 Value resMemRef = linalgOp.getDpsInitOperand(i: 0)->get();
765 assert(cast<MemRefType>(lhsMemRef.getType()).getRank() == 2 &&
766 "expected lhs to be a 2D memref");
767 assert(cast<MemRefType>(rhsMemRef.getType()).getRank() == 2 &&
768 "expected rhs to be a 2D memref");
769 assert(cast<MemRefType>(resMemRef.getType()).getRank() == 2 &&
770 "expected res to be a 2D memref");
771
772 int64_t m = cast<MemRefType>(Val: lhsMemRef.getType()).getShape()[0];
773 int64_t n = cast<MemRefType>(Val: rhsMemRef.getType()).getShape()[1];
774 int64_t k = cast<MemRefType>(Val: lhsMemRef.getType()).getShape()[1];
775 Type lhsType = getElementTypeOrSelf(type: lhsMemRef.getType());
776 Type rhsType = getElementTypeOrSelf(type: rhsMemRef.getType());
777 Type resType = getElementTypeOrSelf(type: resMemRef.getType());
778
779 FailureOr<MmaSyncInfo> maybeInfo =
780 getIndexCalculators(opShape: {m, n, k}, elementalTypes: {lhsType, rhsType, resType});
781 if (failed(Result: maybeInfo))
782 return failure();
783
784 const MmaSyncInfo &info = *maybeInfo;
785 auto [lhsIndexFn, rhsIndexFn, resIndexFn] = info.indexFns;
786 auto [lhsShape, rhsShape, resShape] = info.vectorShapes;
787 Value lhs = buildMmaSyncMemRefLoadOperand(b, loc, laneId, memref: lhsMemRef,
788 indexFn: lhsIndexFn, vectorShape: lhsShape);
789 Value rhs = buildMmaSyncMemRefLoadOperand(b, loc, laneId, memref: rhsMemRef,
790 indexFn: rhsIndexFn, vectorShape: rhsShape);
791 Value res = buildMmaSyncMemRefLoadOperand(b, loc, laneId, memref: resMemRef,
792 indexFn: resIndexFn, vectorShape: resShape);
793 res =
794 MmaSyncOp::create(builder&: b, location: loc, matrixA: lhs, matrixB: rhs, matrixC: res, mmaShape: info.mmaShape, tf32Enabled: info.tf32Enabled);
795 buildMmaSyncMemRefStoreOperand(b, loc, vectorToStore: res, laneId, memref: resMemRef, indexFn: resIndexFn,
796 vectorShape: resShape);
797 return res.getDefiningOp();
798}
799
800DiagnosedSilenceableFailure RewriteMatmulAsMmaSyncOp::applyToOne(
801 TransformRewriter &rewriter, LinalgOp linalgOp,
802 ApplyToEachResultList &results, TransformState &state) {
803 bool fail = true;
804 // TODO: more robust detection of matmulOp, with transposes etc.
805 if (isa_and_nonnull<linalg::MatmulOp>(Val: linalgOp.getOperation())) {
806 // Check to not let go the matmul with extended semantic, through this
807 // transform.
808 if (linalgOp.hasUserDefinedMaps()) {
809 return emitSilenceableError()
810 << "only matmul ops with non-extended semantics are supported";
811 }
812 Location loc = linalgOp.getLoc();
813 // TODO: more robust computation of laneId, for now assume a single warp.
814 Value laneId = gpu::ThreadIdOp::create(
815 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::x);
816 if (succeeded(Result: MmaSyncBuilder(rewriter, loc, laneId).buildMmaSync(linalgOp)))
817 fail = false;
818 }
819
820 if (fail) {
821 DiagnosedSilenceableFailure diag = emitSilenceableError()
822 << "unsupported target op: " << linalgOp;
823 diag.attachNote(loc: linalgOp->getLoc()) << "target op";
824 return diag;
825 }
826
827 rewriter.eraseOp(op: linalgOp);
828 return DiagnosedSilenceableFailure::success();
829}
830
831//===----------------------------------------------------------------------===//
832// Hopper builders.
833//===----------------------------------------------------------------------===//
834
835/// Helper to create the base Hopper-specific operations that are reused in
836/// various other places.
837struct HopperBuilder {
838 HopperBuilder(RewriterBase &rewriter, Location loc)
839 : rewriter(rewriter), loc(loc) {}
840
841 TypedValue<MBarrierGroupType>
842 buildAndInitBarrierInSharedMemory(OpFoldResult numThreads);
843
844 /// Create tma descriptor op to initiate transfer from global to shared
845 /// memory. This must be done before the launch op, on the host.
846 TypedValue<TensorMapDescriptorType>
847 buildGlobalMemRefDescriptor(TypedValue<MemRefType> memref,
848 gpu::LaunchOp launchOp);
849
850 /// Build a tma load from global memory to shared memory using `barrier` to
851 /// synchronize. Return the number of bytes that will be transferred.
852 OpFoldResult buildTmaAsyncLoad(TypedValue<TensorMapDescriptorType> globalDesc,
853 TypedValue<MemRefType> sharedMemref,
854 TypedValue<MBarrierGroupType> barrier,
855 SmallVectorImpl<Operation *> &loadOps);
856 void buildBarrierArriveTx(TypedValue<MBarrierGroupType> barrier,
857 ArrayRef<OpFoldResult> sizes);
858
859 /// If threadIdx.x == 0 does TMA request + wait, else just wait.
860 /// Return the operation that performs the transfer on thread0.
861 // TODO: In the future, don't hardcode to thread 0 but elect a leader.
862 SmallVector<Operation *> buildPredicateLoadsOnThread0(
863 ArrayRef<TypedValue<TensorMapDescriptorType>> globalDescriptors,
864 ArrayRef<TypedValue<MemRefType>> sharedMemBuffers,
865 TypedValue<MBarrierGroupType> barrier);
866
867 void buildTryWaitParity(TypedValue<MBarrierGroupType> barrier);
868
869 RewriterBase &rewriter;
870 Location loc;
871};
872
873SmallVector<Operation *> HopperBuilder::buildPredicateLoadsOnThread0(
874 ArrayRef<TypedValue<TensorMapDescriptorType>> globalDescriptors,
875 ArrayRef<TypedValue<MemRefType>> sharedMemBuffers,
876 TypedValue<MBarrierGroupType> barrier) {
877 SmallVector<Operation *> loadOps;
878 Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
879 Value tidx = gpu::ThreadIdOp::create(builder&: rewriter, location: loc, dimension: gpu::Dimension::x);
880 Value cond = arith::CmpIOp::create(builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq,
881 lhs: tidx, rhs: zero);
882 // clang-format off
883 scf::IfOp::create(builder&: rewriter,
884 /*location=*/loc,
885 /*conditional=*/cond,
886 /*thenBuilder=*/
887 [&](OpBuilder &lb, Location loc) {
888 SmallVector<OpFoldResult> sizes;
889 sizes.reserve(N: globalDescriptors.size());
890 for (auto [desc, shmem] : llvm::zip_equal(
891 t&: globalDescriptors, u&: sharedMemBuffers)) {
892 OpFoldResult sz = buildTmaAsyncLoad(globalDesc: desc, sharedMemref: shmem, barrier, loadOps);
893 sizes.push_back(Elt: sz);
894 }
895 // TODO: Note that cutlass predeclares the barrier arrive tx before the tma.async.load.
896 // This may or may not have perf implications.
897 buildBarrierArriveTx(barrier, sizes);
898 scf::YieldOp::create(builder&: rewriter, location: loc);
899 },
900 /*elseBuilder=*/
901 [&](OpBuilder &lb, Location loc) {
902 // TODO: is this for no-thread divergence?
903 // Should we just yield the size and hoist?
904 buildBarrierArriveTx(barrier, sizes: getAsIndexOpFoldResult(ctx: rewriter.getContext(), val: 0));
905 scf::YieldOp::create(builder&: rewriter, location: loc);
906 });
907 // clang-format on
908 return loadOps;
909}
910
911static Attribute getSharedAddressSpaceAttribute(OpBuilder &b) {
912 return gpu::AddressSpaceAttr::get(
913 context: b.getContext(), value: gpu::GPUDialect::getWorkgroupAddressSpace());
914 // return b.getI64IntegerAttr(static_cast<int64_t>(kSharedMemorySpace));
915}
916
917TypedValue<MBarrierGroupType>
918HopperBuilder::buildAndInitBarrierInSharedMemory(OpFoldResult numThreads) {
919 auto sharedMemorySpace = getSharedAddressSpaceAttribute(b&: rewriter);
920 Value barrier = MBarrierCreateOp::create(
921 builder&: rewriter, location: loc,
922 barriers: MBarrierGroupType::get(context: rewriter.getContext(), memorySpace: sharedMemorySpace));
923 Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
924 nvgpu::MBarrierInitOp::create(
925 builder&: rewriter, location: loc, barriers: barrier,
926 count: getValueOrCreateConstantIndexOp(b&: rewriter, loc, ofr: numThreads), mbarId: zero,
927 predicate: Value());
928 gpu::BarrierOp::create(builder&: rewriter, location: loc);
929 return cast<TypedValue<MBarrierGroupType>>(Val&: barrier);
930}
931
932TypedValue<TensorMapDescriptorType>
933HopperBuilder::buildGlobalMemRefDescriptor(TypedValue<MemRefType> memref,
934 gpu::LaunchOp launchOp) {
935 OpBuilder::InsertionGuard guard(rewriter);
936 rewriter.setInsertionPoint(launchOp);
937 Value unrankedMemRef = memref::CastOp::create(
938 builder&: rewriter, location: loc,
939 dest: UnrankedMemRefType::get(elementType: memref.getType().getElementType(),
940 memorySpace: memref.getType().getMemorySpace()),
941 source: memref);
942 SmallVector<OpFoldResult> mixedSizes =
943 memref::getMixedSizes(builder&: rewriter, loc, value: memref);
944 SmallVector<Value> sizes =
945 getValueOrCreateConstantIndexOp(b&: rewriter, loc, valueOrAttrVec: mixedSizes);
946
947 auto sharedMemorySpace = getSharedAddressSpaceAttribute(b&: rewriter);
948 Value desc = TmaCreateDescriptorOp::create(
949 builder&: rewriter, location: loc,
950 tensorMap: TensorMapDescriptorType::get(context: rewriter.getContext(),
951 tensor: MemRefType::Builder(memref.getType())
952 .setMemorySpace(sharedMemorySpace),
953 swizzle: TensorMapSwizzleKind::SWIZZLE_NONE,
954 l2promo: TensorMapL2PromoKind::L2PROMO_NONE,
955 oob: TensorMapOOBKind::OOB_ZERO,
956 interleave: TensorMapInterleaveKind::INTERLEAVE_NONE),
957 tensor: unrankedMemRef, boxDimensions: sizes);
958 return cast<TypedValue<TensorMapDescriptorType>>(Val&: desc);
959}
960
961OpFoldResult
962HopperBuilder::buildTmaAsyncLoad(TypedValue<TensorMapDescriptorType> globalDesc,
963 TypedValue<MemRefType> sharedMemref,
964 TypedValue<MBarrierGroupType> barrier,
965 SmallVectorImpl<Operation *> &loadOps) {
966 MLIRContext *ctx = rewriter.getContext();
967 Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
968 Operation *loadOp =
969 TmaAsyncLoadOp::create(builder&: rewriter, location: loc, dst: sharedMemref, barriers: barrier, tensorMapDescriptor: globalDesc,
970 coordinates: ValueRange{zero, zero}, mbarId: zero, multicastMask: Value(), predicate: Value());
971 loadOps.push_back(Elt: loadOp);
972 auto mixedSizes = memref::getMixedSizes(builder&: rewriter, loc, value: sharedMemref);
973 SmallVector<AffineExpr> symbols(mixedSizes.size());
974 bindSymbolsList(ctx, exprs: llvm::MutableArrayRef{symbols});
975 AffineExpr prodExprInBytes =
976 computeProduct(ctx, basis: symbols) *
977 (sharedMemref.getType().getElementTypeBitWidth() / 8);
978 auto res = affine::makeComposedFoldedAffineApply(b&: rewriter, loc,
979 expr: prodExprInBytes, operands: mixedSizes);
980 return res;
981}
982
983void HopperBuilder::buildBarrierArriveTx(TypedValue<MBarrierGroupType> barrier,
984 ArrayRef<OpFoldResult> mixedSizes) {
985 assert(!mixedSizes.empty() && "expecte non-empty sizes");
986 MLIRContext *ctx = rewriter.getContext();
987 SmallVector<AffineExpr> symbols(mixedSizes.size());
988 bindSymbolsList(ctx, exprs: llvm::MutableArrayRef{symbols});
989 AffineExpr sumExpr = computeSum(ctx, basis: symbols);
990 OpFoldResult size =
991 affine::makeComposedFoldedAffineApply(b&: rewriter, loc, expr: sumExpr, operands: mixedSizes);
992 Value sizeVal = getValueOrCreateConstantIndexOp(b&: rewriter, loc, ofr: size);
993 Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
994 nvgpu::MBarrierArriveExpectTxOp::create(builder&: rewriter, location: loc, barriers: barrier, txcount: sizeVal, mbarId: zero,
995 predicate: Value());
996}
997
998void HopperBuilder::buildTryWaitParity(TypedValue<MBarrierGroupType> barrier) {
999 Type i1 = rewriter.getI1Type();
1000 Value parity = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i1, value: 0);
1001 // 10M is an arbitrary, not too small or too big number to specify the number
1002 // of ticks before retry.
1003 // TODO: hoist this in a default dialect constant.
1004 Value ticksBeforeRetry =
1005 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 10000000);
1006 Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
1007 nvgpu::MBarrierTryWaitParityOp::create(builder&: rewriter, location: loc, barriers: barrier, phaseParity: parity,
1008 ticks: ticksBeforeRetry, mbarId: zero);
1009}
1010
1011//===----------------------------------------------------------------------===//
1012// RewriteCopyAsTmaOp
1013//===----------------------------------------------------------------------===//
1014
1015/// Helper to create the tma operations corresponding to `linalg::CopyOp`.
1016struct CopyBuilder : public HopperBuilder {
1017 CopyBuilder(RewriterBase &rewriter, Location loc)
1018 : HopperBuilder(rewriter, loc) {}
1019
1020 SmallVector<Operation *> rewrite(ArrayRef<Operation *> copyOps);
1021};
1022
1023SmallVector<Operation *> CopyBuilder::rewrite(ArrayRef<Operation *> copyOps) {
1024 MLIRContext *ctx = rewriter.getContext();
1025 if (copyOps.empty())
1026 return SmallVector<Operation *>();
1027
1028 auto launchOp = copyOps.front()->getParentOfType<gpu::LaunchOp>();
1029 assert(launchOp && "expected launch op");
1030
1031 // 1. Init a barrier object in shared memory.
1032 OpBuilder::InsertionGuard g(rewriter);
1033 rewriter.setInsertionPoint(copyOps.front());
1034 AffineExpr bx, by, bz;
1035 bindSymbols(ctx, exprs&: bx, exprs&: by, exprs&: bz);
1036 AffineExpr prod = computeProduct(ctx, basis: ArrayRef<AffineExpr>{bx, by, bz});
1037 OpFoldResult numThreads = affine::makeComposedFoldedAffineApply(
1038 b&: rewriter, loc, expr: prod,
1039 operands: ArrayRef<OpFoldResult>{launchOp.getBlockSizeX(), launchOp.getBlockSizeY(),
1040 launchOp.getBlockSizeZ()});
1041
1042 TypedValue<MBarrierGroupType> barrier =
1043 buildAndInitBarrierInSharedMemory(numThreads);
1044
1045 SmallVector<TypedValue<MemRefType>> shmems;
1046 SmallVector<TypedValue<TensorMapDescriptorType>> globalDescs;
1047 for (Operation *op : copyOps) {
1048 auto copyOp = cast<linalg::CopyOp>(Val: op);
1049 auto inMemRef =
1050 cast<TypedValue<MemRefType>>(Val: copyOp.getDpsInputOperand(i: 0)->get());
1051 assert(inMemRef.getType().getRank() == 2 &&
1052 "expected in to be a 2D memref");
1053
1054 // 2. Build global memory descriptor.
1055 TypedValue<TensorMapDescriptorType> globalDesc =
1056 buildGlobalMemRefDescriptor(memref: inMemRef, launchOp);
1057 globalDescs.push_back(Elt: globalDesc);
1058
1059 // 3. Shared memory and descriptor for the tmp array.
1060 auto shmem =
1061 cast<TypedValue<MemRefType>>(Val: copyOp.getDpsInitOperand(i: 0)->get());
1062 shmems.push_back(Elt: shmem);
1063 }
1064
1065 // 4. Load in from global memory to shared memory using tma.
1066 OpBuilder::InsertionGuard g2(rewriter);
1067 rewriter.setInsertionPoint(copyOps.front());
1068 SmallVector<Operation *> results =
1069 buildPredicateLoadsOnThread0(globalDescriptors: globalDescs, sharedMemBuffers: shmems, barrier);
1070
1071 // 5. Spin-loop until data is ready.
1072 buildTryWaitParity(barrier);
1073
1074 // 6. Erase the ops that have now been rewritten.
1075 for (Operation *op : copyOps)
1076 rewriter.eraseOp(op);
1077
1078 return results;
1079}
1080
1081DiagnosedSilenceableFailure
1082RewriteCopyAsTmaOp::apply(TransformRewriter &rewriter,
1083 TransformResults &results, TransformState &state) {
1084 auto payloadOps = state.getPayloadOps(value: getTarget());
1085 gpu::LaunchOp commonLaunchOp;
1086 Operation *firstOp, *failingOp;
1087 if (llvm::any_of(Range&: payloadOps, P: [&](Operation *op) {
1088 if (!commonLaunchOp) {
1089 commonLaunchOp = op->getParentOfType<gpu::LaunchOp>();
1090 firstOp = op;
1091 }
1092 auto fail = !op->getParentOfType<gpu::LaunchOp>() ||
1093 commonLaunchOp != op->getParentOfType<gpu::LaunchOp>() ||
1094 !isa<linalg::CopyOp>(Val: op);
1095 if (fail)
1096 failingOp = op;
1097 return fail;
1098 })) {
1099 DiagnosedSilenceableFailure diag =
1100 emitSilenceableError()
1101 << "target ops must be linalg::CopyOp nested under a common "
1102 "gpu.LaunchOp to be rewritten because the tma descriptors need to "
1103 "be created on the host.\nBut got: "
1104 << *firstOp << "\nand " << *failingOp;
1105 return diag;
1106 }
1107
1108 // TODO: more robust detection of copy, with transposes etc.
1109 CopyBuilder(rewriter, getLoc()).rewrite(copyOps: llvm::to_vector(Range&: payloadOps));
1110
1111 return DiagnosedSilenceableFailure::success();
1112}
1113
1114//===----------------------------------------------------------------------===//
1115// Transform op registration
1116//===----------------------------------------------------------------------===//
1117
1118namespace {
1119class NVGPUTransformDialectExtension
1120 : public TransformDialectExtension<NVGPUTransformDialectExtension> {
1121public:
1122 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(NVGPUTransformDialectExtension)
1123
1124 NVGPUTransformDialectExtension() {
1125 declareGeneratedDialect<arith::ArithDialect>();
1126 declareGeneratedDialect<affine::AffineDialect>();
1127 declareGeneratedDialect<NVGPUDialect>();
1128 declareGeneratedDialect<NVVM::NVVMDialect>();
1129 declareGeneratedDialect<vector::VectorDialect>();
1130 registerTransformOps<
1131#define GET_OP_LIST
1132#include "mlir/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp.inc"
1133 >();
1134 }
1135};
1136} // namespace
1137
1138#define GET_OP_CLASSES
1139#include "mlir/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp.inc"
1140
1141void mlir::nvgpu::registerTransformDialectExtension(DialectRegistry &registry) {
1142 registry.addExtensions<NVGPUTransformDialectExtension>();
1143}
1144